def fill_settings(sections,
acquire_settings,
log_printer=None,
fill_section_method=fill_section,
**kwargs):
"""
Retrieves all bears and requests missing settings via the given
acquire_settings method.
This will retrieve all bears and their dependencies.
:param sections: The sections to fill up, modified in place.
:param acquire_settings: The method to use for requesting settings. It
will get a parameter which is a dictionary with
the settings name as key and a list containing
a description in [0] and the names of the bears
who need this setting in all following indexes.
:param log_printer: The log printer to use for logging.
:param fill_section_method: Method to be used to fill the section settings.
:param kwargs: Any other arguments for the fill_section_method
can be supplied via kwargs, which are passed
directly to the fill_section_method.
:return: A tuple containing (local_bears, global_bears),
each of them being a dictionary with the
section name as key and as value the bears as a
list.
"""
local_bears = {}
global_bears = {}
for section_name, section in sections.items():
bear_dirs = section.bear_dirs()
if getattr(section, 'aspects', None):
section_local_bears, section_global_bears = (
collect_bears_by_aspects(
section.aspects,
[BEAR_KIND.LOCAL, BEAR_KIND.GLOBAL]))
else:
bears = list(section.get('bears', ''))
section_local_bears, section_global_bears = collect_bears(
bear_dirs,
bears,
[BEAR_KIND.LOCAL, BEAR_KIND.GLOBAL])
section_local_bears = Dependencies.resolve(section_local_bears)
section_global_bears = Dependencies.resolve(section_global_bears)
all_bears = copy.deepcopy(section_local_bears)
all_bears.extend(section_global_bears)
fill_section_method(section,
acquire_settings,
None,
all_bears,
**kwargs)
local_bears[section_name] = section_local_bears
global_bears[section_name] = section_global_bears
return local_bears, global_bears
def __init__(self, section, local_bear_list, global_bear_list):
if not isinstance(section, Section):
raise TypeError("section has to be of type Section")
if not isinstance(local_bear_list, list):
raise TypeError("local_bear_list has to be of type list")
if not isinstance(global_bear_list, list):
raise TypeError("global_bear_list has to be of type list")
self.section = section
self.local_bear_list = Dependencies.resolve(local_bear_list)
self.global_bear_list = Dependencies.resolve(global_bear_list)
def __init__(self,
section,
local_bear_list,
global_bear_list):
if not isinstance(section, Section):
raise TypeError("section has to be of type Section")
if not isinstance(local_bear_list, list):
raise TypeError("local_bear_list has to be of type list")
if not isinstance(global_bear_list, list):
raise TypeError("global_bear_list has to be of type list")
self.section = section
self.local_bear_list = Dependencies.resolve(local_bear_list)
self.global_bear_list = Dependencies.resolve(global_bear_list)
def fill_settings(sections,
acquire_settings,
log_printer,
fill_section_method=fill_section,
**kwargs):
"""
Retrieves all bears and requests missing settings via the given
acquire_settings method.
This will retrieve all bears and their dependencies.
:param sections: The sections to fill up, modified in place.
:param acquire_settings: The method to use for requesting settings. It
will get a parameter which is a dictionary with
the settings name as key and a list containing
a description in [0] and the names of the bears
who need this setting in all following indexes.
:param log_printer: The log printer to use for logging.
:param fill_section_method: Method to be used to fill the section settings.
:param kwargs: Any other arguments for the fill_section_method
can be supplied via kwargs, which are passed
directly to the fill_section_method.
:return: A tuple containing (local_bears, global_bears),
each of them being a dictionary with the
section name as key and as value the bears as a
list.
"""
local_bears = {}
global_bears = {}
for section_name, section in sections.items():
bear_dirs = section.bear_dirs()
bears = list(section.get('bears', ''))
section_local_bears, section_global_bears = collect_bears(
bear_dirs, bears, [BEAR_KIND.LOCAL, BEAR_KIND.GLOBAL], log_printer)
section_local_bears = Dependencies.resolve(section_local_bears)
section_global_bears = Dependencies.resolve(section_global_bears)
all_bears = copy.deepcopy(section_local_bears)
all_bears.extend(section_global_bears)
fill_section_method(section, acquire_settings, log_printer, all_bears,
**kwargs)
local_bears[section_name] = section_local_bears
global_bears[section_name] = section_global_bears
return local_bears, global_bears
def fill_settings(sections, acquire_settings, log_printer):
"""
Retrieves all bears and requests missing settings via the given
acquire_settings method.
This will retrieve all bears and their dependencies.
:param sections: The sections to fill up, modified in place.
:param acquire_settings: The method to use for requesting settings. It will
get a parameter which is a dictionary with the
settings name as key and a list containing a
description in [0] and the names of the bears
who need this setting in all following indexes.
:param log_printer: The log printer to use for logging.
:return: A tuple containing (local_bears, global_bears),
each of them being a dictionary with the section
name as key and as value the bears as a list.
"""
local_bears = {}
global_bears = {}
for section_name, section in sections.items():
bear_dirs = section.bear_dirs()
bears = list(section.get("bears", ""))
section_local_bears, section_global_bears = collect_bears(
bear_dirs,
bears,
[BEAR_KIND.LOCAL, BEAR_KIND.GLOBAL],
log_printer)
section_local_bears = Dependencies.resolve(section_local_bears)
section_global_bears = Dependencies.resolve(section_global_bears)
all_bears = copy.deepcopy(section_local_bears)
all_bears.extend(section_global_bears)
fill_section(section, acquire_settings, log_printer, all_bears)
local_bears[section_name] = section_local_bears
global_bears[section_name] = section_global_bears
return local_bears, global_bears
def execute_section(section,
global_bear_list,
local_bear_list,
print_results,
log_printer):
"""
Executes the section with the given bears.
The execute_section method does the following things:
1. Prepare a Process
* Load files
* Create queues
2. Spawn up one or more Processes
3. Output results from the Processes
4. Join all processes
:param section: The section to execute.
:param global_bear_list: List of global bears belonging to the section.
:param local_bear_list: List of local bears belonging to the section.
:param print_results: Prints all given results appropriate to the
output medium.
:param log_printer: The log_printer to warn to.
:return: Tuple containing a bool (True if results were
yielded, False otherwise), a Manager.dict
containing all local results(filenames are key)
and a Manager.dict containing all global bear
results (bear names are key) as well as the
file dictionary.
"""
local_bear_list = Dependencies.resolve(local_bear_list)
global_bear_list = Dependencies.resolve(global_bear_list)
try:
running_processes = int(section['jobs'])
except ValueError:
log_printer.warn("Unable to convert setting 'jobs' into a number. "
"Falling back to CPU count.")
running_processes = get_cpu_count()
except IndexError:
running_processes = get_cpu_count()
processes, arg_dict = instantiate_processes(section,
local_bear_list,
global_bear_list,
running_processes,
log_printer)
logger_thread = LogPrinterThread(arg_dict["message_queue"],
log_printer)
# Start and join the logger thread along with the processes to run bears
processes.append(logger_thread)
for runner in processes:
runner.start()
try:
return (process_queues(processes,
arg_dict["control_queue"],
arg_dict["local_result_dict"],
arg_dict["global_result_dict"],
arg_dict["file_dict"],
print_results,
section,
log_printer),
arg_dict["local_result_dict"],
arg_dict["global_result_dict"],
arg_dict["file_dict"])
finally:
logger_thread.running = False
for runner in processes:
runner.join()
def test_resolvable_deps(self):
self.assertEqual(
Dependencies.resolve([ResolvableBear1, ResolvableBear2]),
[BearWithoutDeps, ResolvableBear1, ResolvableBear2])
def test_no_deps(self):
self.assertEqual(
len(Dependencies.resolve([BearWithoutDeps, BearWithoutDeps])), 1)
def execute_section(section, global_bear_list, local_bear_list, print_results,
cache, log_printer):
"""
Executes the section with the given bears.
The execute_section method does the following things:
1. Prepare a Process
- Load files
- Create queues
2. Spawn up one or more Processes
3. Output results from the Processes
4. Join all processes
:param section: The section to execute.
:param global_bear_list: List of global bears belonging to the section.
:param local_bear_list: List of local bears belonging to the section.
:param print_results: Prints all given results appropriate to the
output medium.
:param cache: An instance of ``misc.Caching.FileCache`` to use as
a file cache buffer.
:param log_printer: The log_printer to warn to.
:return: Tuple containing a bool (True if results were
yielded, False otherwise), a Manager.dict
containing all local results(filenames are key)
and a Manager.dict containing all global bear
results (bear names are key) as well as the
file dictionary.
"""
local_bear_list = Dependencies.resolve(local_bear_list)
global_bear_list = Dependencies.resolve(global_bear_list)
try:
running_processes = int(section['jobs'])
except ValueError:
log_printer.warn("Unable to convert setting 'jobs' into a number. "
"Falling back to CPU count.")
running_processes = get_cpu_count()
except IndexError:
running_processes = get_cpu_count()
processes, arg_dict = instantiate_processes(section, local_bear_list,
global_bear_list,
running_processes, cache,
log_printer)
logger_thread = LogPrinterThread(arg_dict["message_queue"], log_printer)
# Start and join the logger thread along with the processes to run bears
processes.append(logger_thread)
for runner in processes:
runner.start()
try:
return (process_queues(processes, arg_dict["control_queue"],
arg_dict["local_result_dict"],
arg_dict["global_result_dict"],
arg_dict["file_dict"], print_results, section,
cache,
log_printer), arg_dict["local_result_dict"],
arg_dict["global_result_dict"], arg_dict["file_dict"])
finally:
logger_thread.running = False
for runner in processes:
runner.join()
def test_resolvable_deps(self):
self.assertEqual(Dependencies.resolve([ResolvableBear1,
ResolvableBear2]),
[BearWithoutDeps, ResolvableBear1, ResolvableBear2])
def test_no_deps(self):
self.assertEqual(
len(Dependencies.resolve([BearWithoutDeps,
BearWithoutDeps])),
1)