def instantiate_processes(section,
local_bear_list,
global_bear_list,
job_count,
log_printer):
"""
Instantiate the number of processes that will run bears which will be
responsible for running bears in a multiprocessing environment.
:param section: The section the bears belong to.
:param local_bear_list: List of local bears belonging to the section.
:param global_bear_list: List of global bears belonging to the section.
:param job_count: Max number of processes to create.
:param log_printer: The log printer to warn to.
:return: A tuple containing a list of processes,
and the arguments passed to each process which are
the same for each object.
"""
filename_list = collect_files(
path_list(section.get('files', "")),
log_printer,
ignored_file_paths=path_list(section.get('ignore', "")),
limit_file_paths=path_list(section.get('limit_files', "")))
file_dict = get_file_dict(filename_list, log_printer)
manager = multiprocessing.Manager()
global_bear_queue = multiprocessing.Queue()
filename_queue = multiprocessing.Queue()
local_result_dict = manager.dict()
global_result_dict = manager.dict()
message_queue = multiprocessing.Queue()
control_queue = multiprocessing.Queue()
bear_runner_args = {"file_name_queue": filename_queue,
"local_bear_list": local_bear_list,
"global_bear_list": global_bear_list,
"global_bear_queue": global_bear_queue,
"file_dict": file_dict,
"local_result_dict": local_result_dict,
"global_result_dict": global_result_dict,
"message_queue": message_queue,
"control_queue": control_queue,
"timeout": 0.1}
local_bear_list[:], global_bear_list[:] = instantiate_bears(
section,
local_bear_list,
global_bear_list,
file_dict,
message_queue)
fill_queue(filename_queue, file_dict.keys())
fill_queue(global_bear_queue, range(len(global_bear_list)))
return ([multiprocessing.Process(target=run, kwargs=bear_runner_args)
for i in range(job_count)],
bear_runner_args)
def test_path_list(self):
abspath = os.path.abspath('.')
# Need to escape backslashes since we use list conversion
self.uut = Setting('key', '., ' + abspath.replace('\\', '\\\\'),
origin=os.path.join('test', 'somefile'))
self.assertEqual(path_list(self.uut),
[os.path.abspath(os.path.join('test', '.')), abspath])
self.uut = Setting('key', '., ' + abspath.replace('\\', '\\\\'),
origin=SourcePosition(
os.path.join('test', 'somefile')))
self.assertEqual(path_list(self.uut),
[os.path.abspath(os.path.join('test', '.')), abspath])
def test_path_list(self):
abspath = os.path.abspath(".")
# Need to escape backslashes since we use list conversion
self.uut = Setting("key", "., " + abspath.replace("\\", "\\\\"),
origin="test" + os.path.sep + "somefile")
self.assertEqual(path_list(self.uut),
[os.path.abspath(os.path.join("test", ".")), abspath])
def Analyze(self):
"""
This method analyzes the document and sends back the result
:return: The output is a list with an element for each section.
It contains:
- The name of the section
- Boolean which is true if all bears in the section executed
successfully
- List of results where each result is a list which contains:
(str)origin, (str)message, (str)file, (str)line_nr,
(str)severity
"""
retval = []
if self.path == "" or self.config_file == "":
return retval
args = ["--config=" + self.config_file]
log_printer = ListLogPrinter()
exitcode = 0
try:
yielded_results = False
(sections,
local_bears,
global_bears,
targets) = gather_configuration(fail_acquire_settings,
log_printer,
arg_list=args)
for section_name in sections:
section = sections[section_name]
if not section.is_enabled(targets):
continue
if any([fnmatch(self.path, file_pattern)
for file_pattern in path_list(section["files"])]):
section["files"].value = self.path
section_result = execute_section(
section=section,
global_bear_list=global_bears[section_name],
local_bear_list=local_bears[section_name],
print_results=lambda *args: True,
log_printer=log_printer,
file_diff_dict={})
yielded_results = yielded_results or section_result[0]
retval.append(
DbusDocument.results_to_dbus_struct(section_result,
section_name))
if yielded_results:
exitcode = 1
except BaseException as exception: # pylint: disable=broad-except
exitcode = exitcode or get_exitcode(exception, log_printer)
logs = [log.to_string_dict() for log in log_printer.logs]
return (exitcode, logs, retval)
def _instantiate_processes(self, job_count):
filename_list = collect_files(path_list(self.section.get('files', "")))
file_dict = self._get_file_dict(filename_list)
manager = multiprocessing.Manager()
global_bear_queue = multiprocessing.Queue()
filename_queue = multiprocessing.Queue()
local_result_dict = manager.dict()
global_result_dict = manager.dict()
message_queue = multiprocessing.Queue()
control_queue = multiprocessing.Queue()
barrier = Barrier(parties=job_count)
bear_runner_args = {"file_name_queue": filename_queue,
"local_bear_list": self.local_bear_list,
"global_bear_list": self.global_bear_list,
"global_bear_queue": global_bear_queue,
"file_dict": file_dict,
"local_result_dict": local_result_dict,
"global_result_dict": global_result_dict,
"message_queue": message_queue,
"control_queue": control_queue,
"barrier": barrier,
"TIMEOUT": 0.1}
self._instantiate_bears(file_dict,
message_queue)
self._fill_queue(filename_queue, filename_list)
self._fill_queue(global_bear_queue, range(len(self.global_bear_list)))
return ([BearRunner(**bear_runner_args) for i in range(job_count)],
bear_runner_args)
def bear_dirs(self):
bear_dirs = path_list(self.get("bear_dirs", ""))
for bear_dir in bear_dirs:
sys.path.append(bear_dir)
bear_dir_globs = [
os.path.join(glob_escape(bear_dir), "**")
for bear_dir in bear_dirs]
bear_dir_globs += [
os.path.join(glob_escape(bear_dir), "**")
for bear_dir in collect_registered_bears_dirs('coalabears')]
return bear_dir_globs
def bear_dirs(self):
bear_dirs = path_list(self.get("bear_dirs", ""))
for bear_dir in bear_dirs:
sys.path.append(bear_dir)
bear_dir_globs = [
os.path.join(glob_escape(bear_dir), "**") for bear_dir in bear_dirs
]
bear_dir_globs += [
os.path.join(glob_escape(bear_dir), "**")
for bear_dir in collect_registered_bears_dirs('coalabears')
]
return bear_dir_globs
def _fill_settings(self):
for section_name in self.sections:
section = self.sections[section_name]
section.retrieve_logging_objects()
bear_dirs = path_list(section.get("bear_dirs", ""))
bear_dirs.append(
os.path.join(StringConstants.coalib_bears_root, "**"))
bears = list(section.get("bears", ""))
local_bears = collect_bears(bear_dirs, bears, [BEAR_KIND.LOCAL])
global_bears = collect_bears(bear_dirs, bears, [BEAR_KIND.GLOBAL])
filler = SectionFiller(section)
all_bears = copy.deepcopy(local_bears)
all_bears.extend(global_bears)
filler.fill_section(all_bears)
self.local_bears[section_name] = local_bears
self.global_bears[section_name] = global_bears
def fill_settings(sections, acquire_settings, log_printer):
"""
Retrieves all bears and requests missing settings via the given
acquire_settings method.
:param sections: The sections to fill up, modified in place.
:param acquire_settings: The method to use for requesting settings. It will
get a parameter which is a dictionary with the
settings name as key and a list containing a
description in [0] and the names of the bears
who need this setting in all following indexes.
:param log_printer: The log printer to use for logging.
:return: A tuple containing (local_bears, global_bears),
each of them being a dictionary with the section
name as key and as value the bears as a list.
"""
local_bears = {}
global_bears = {}
for section_name, section in sections.items():
bear_dirs = path_list(section.get("bear_dirs", ""))
bear_dirs.append(os.path.join(Constants.coalib_bears_root,
"**"))
bears = list(section.get("bears", ""))
section_local_bears = collect_bears(bear_dirs,
bears,
[BEAR_KIND.LOCAL],
log_printer)
section_global_bears = collect_bears(bear_dirs,
bears,
[BEAR_KIND.GLOBAL],
log_printer)
all_bears = copy.deepcopy(section_local_bears)
all_bears.extend(section_global_bears)
fill_section(section, acquire_settings, log_printer, all_bears)
local_bears[section_name] = section_local_bears
global_bears[section_name] = section_global_bears
return local_bears, global_bears
def _fill_settings(self):
for section_name in self.sections:
section = self.sections[section_name]
section.retrieve_logging_objects()
bear_dirs = path_list(section.get("bear_dirs", ""))
bear_dirs.append(os.path.join(StringConstants.coalib_bears_root,
"**"))
bears = list(section.get("bears", ""))
local_bears = collect_bears(bear_dirs,
bears,
[BEAR_KIND.LOCAL])
global_bears = collect_bears(bear_dirs,
bears,
[BEAR_KIND.GLOBAL])
filler = SectionFiller(section)
all_bears = copy.deepcopy(local_bears)
all_bears.extend(global_bears)
filler.fill_section(all_bears)
self.local_bears[section_name] = local_bears
self.global_bears[section_name] = global_bears
def bear_dirs(self):
bear_dirs = path_list(self.get("bear_dirs", ""))
bear_dirs.append(os.path.join(Constants.coalib_bears_root, "**"))
bear_dirs += collect_registered_bears_dirs('coalabears')
return bear_dirs
def bear_dirs(self):
bear_dirs = path_list(self.get("bear_dirs", ""))
bear_dirs.append(os.path.join(Constants.coalib_bears_root, "**"))
return bear_dirs
def Analyze(self):
"""
This method analyzes the document and sends back the result
:return: The output is structure which has 3 items:
- The exitcode from the analysis.
- List of logs from the analysis.
- List of information about each section that contains:
- The name of the section.
- Boolean which is true if all bears in the section
executed successfully.
- List of results where each result is a string
dictionary which contains:
id, origin, message, file, line_nr, severity
"""
retval = []
if self.path == "" or self.config_file == "":
return retval
args = ["--config=" + self.config_file]
log_printer = ListLogPrinter()
exitcode = 0
try:
yielded_results = False
(sections,
local_bears,
global_bears,
targets) = gather_configuration(fail_acquire_settings,
log_printer,
arg_list=args)
for section_name in sections:
section = sections[section_name]
if not section.is_enabled(targets):
continue
if any([fnmatch(self.path, file_pattern)
for file_pattern in path_list(section["files"])]):
section["files"].value = self.path
section_result = execute_section(
section=section,
global_bear_list=global_bears[section_name],
local_bear_list=local_bears[section_name],
print_results=lambda *args: True,
log_printer=log_printer)
yielded_results = yielded_results or section_result[0]
retval.append(
DbusDocument.results_to_dbus_struct(section_result,
section_name))
if yielded_results:
exitcode = 1
except BaseException as exception: # pylint: disable=broad-except
exitcode = exitcode or get_exitcode(exception, log_printer)
logs = [log.to_string_dict() for log in log_printer.logs]
return (exitcode, logs, retval)
def bear_dirs(self):
bear_dirs = path_list(self.get("bear_dirs", ""))
bear_dirs.append(os.path.join(Constants.coalib_bears_root, "**"))
bear_dirs += [os.path.join(bear_dir, "**") for bear_dir in collect_registered_bears_dirs("coalabears")]
return bear_dirs
def bear_dirs(self):
bear_dirs = path_list(self.get("bear_dirs", ""))
bear_dirs.append(os.path.join(Constants.coalib_bears_root, "**"))
bear_dirs += collect_registered_bears_dirs('coalabears')
return bear_dirs
def bear_dirs(self):
bear_dirs = path_list(self.get("bear_dirs", ""))
bear_dirs.append(os.path.join(Constants.coalib_bears_root, "**"))
return bear_dirs
def bear_dirs(self):
bear_dirs = path_list(self.get("bear_dirs", ""))
bear_dirs += [
os.path.join(bear_dir, "**")
for bear_dir in collect_registered_bears_dirs('coalabears')]
return bear_dirs
