Ejemplo n.º 1
0
def fill_settings(sections,
                  acquire_settings,
                  log_printer=None,
                  fill_section_method=fill_section,
                  **kwargs):
    """
    Retrieves all bears and requests missing settings via the given
    acquire_settings method.

    This will retrieve all bears and their dependencies.

    :param sections:            The sections to fill up, modified in place.
    :param acquire_settings:    The method to use for requesting settings. It
                                will get a parameter which is a dictionary with
                                the settings name as key and a list containing
                                a description in [0] and the names of the bears
                                who need this setting in all following indexes.
    :param log_printer:         The log printer to use for logging.
    :param fill_section_method: Method to be used to fill the section settings.
    :param kwargs:              Any other arguments for the fill_section_method
                                can be supplied via kwargs, which are passed
                                directly to the fill_section_method.
    :return:                    A tuple containing (local_bears, global_bears),
                                each of them being a dictionary with the
                                section name as key and as value the bears as a
                                list.
    """
    local_bears = {}
    global_bears = {}

    for section_name, section in sections.items():
        bear_dirs = section.bear_dirs()
        if getattr(section, 'aspects', None):
            section_local_bears, section_global_bears = (
                collect_bears_by_aspects(
                    section.aspects,
                    [BEAR_KIND.LOCAL, BEAR_KIND.GLOBAL]))
        else:
            bears = list(section.get('bears', ''))
            section_local_bears, section_global_bears = collect_bears(
                bear_dirs,
                bears,
                [BEAR_KIND.LOCAL, BEAR_KIND.GLOBAL])
        section_local_bears = Dependencies.resolve(section_local_bears)
        section_global_bears = Dependencies.resolve(section_global_bears)
        all_bears = copy.deepcopy(section_local_bears)
        all_bears.extend(section_global_bears)
        fill_section_method(section,
                            acquire_settings,
                            None,
                            all_bears,
                            **kwargs)

        local_bears[section_name] = section_local_bears
        global_bears[section_name] = section_global_bears

    return local_bears, global_bears
Ejemplo n.º 2
0
    def __init__(self, section, local_bear_list, global_bear_list):
        if not isinstance(section, Section):
            raise TypeError("section has to be of type Section")
        if not isinstance(local_bear_list, list):
            raise TypeError("local_bear_list has to be of type list")
        if not isinstance(global_bear_list, list):
            raise TypeError("global_bear_list has to be of type list")

        self.section = section
        self.local_bear_list = Dependencies.resolve(local_bear_list)
        self.global_bear_list = Dependencies.resolve(global_bear_list)
Ejemplo n.º 3
0
    def __init__(self,
                 section,
                 local_bear_list,
                 global_bear_list):
        if not isinstance(section, Section):
            raise TypeError("section has to be of type Section")
        if not isinstance(local_bear_list, list):
            raise TypeError("local_bear_list has to be of type list")
        if not isinstance(global_bear_list, list):
            raise TypeError("global_bear_list has to be of type list")

        self.section = section
        self.local_bear_list = Dependencies.resolve(local_bear_list)
        self.global_bear_list = Dependencies.resolve(global_bear_list)
Ejemplo n.º 4
0
def fill_settings(sections,
                  acquire_settings,
                  log_printer,
                  fill_section_method=fill_section,
                  **kwargs):
    """
    Retrieves all bears and requests missing settings via the given
    acquire_settings method.

    This will retrieve all bears and their dependencies.

    :param sections:            The sections to fill up, modified in place.
    :param acquire_settings:    The method to use for requesting settings. It
                                will get a parameter which is a dictionary with
                                the settings name as key and a list containing
                                a description in [0] and the names of the bears
                                who need this setting in all following indexes.
    :param log_printer:         The log printer to use for logging.
    :param fill_section_method: Method to be used to fill the section settings.
    :param kwargs:              Any other arguments for the fill_section_method
                                can be supplied via kwargs, which are passed
                                directly to the fill_section_method.
    :return:                    A tuple containing (local_bears, global_bears),
                                each of them being a dictionary with the
                                section name as key and as value the bears as a
                                list.
    """
    local_bears = {}
    global_bears = {}

    for section_name, section in sections.items():
        bear_dirs = section.bear_dirs()
        bears = list(section.get('bears', ''))
        section_local_bears, section_global_bears = collect_bears(
            bear_dirs, bears, [BEAR_KIND.LOCAL, BEAR_KIND.GLOBAL], log_printer)
        section_local_bears = Dependencies.resolve(section_local_bears)
        section_global_bears = Dependencies.resolve(section_global_bears)
        all_bears = copy.deepcopy(section_local_bears)
        all_bears.extend(section_global_bears)
        fill_section_method(section, acquire_settings, log_printer, all_bears,
                            **kwargs)

        local_bears[section_name] = section_local_bears
        global_bears[section_name] = section_global_bears

    return local_bears, global_bears
Ejemplo n.º 5
0
def fill_settings(sections, acquire_settings, log_printer):
    """
    Retrieves all bears and requests missing settings via the given
    acquire_settings method.

    This will retrieve all bears and their dependencies.

    :param sections:         The sections to fill up, modified in place.
    :param acquire_settings: The method to use for requesting settings. It will
                             get a parameter which is a dictionary with the
                             settings name as key and a list containing a
                             description in [0] and the names of the bears
                             who need this setting in all following indexes.
    :param log_printer:      The log printer to use for logging.
    :return:                 A tuple containing (local_bears, global_bears),
                             each of them being a dictionary with the section
                             name as key and as value the bears as a list.
    """
    local_bears = {}
    global_bears = {}

    for section_name, section in sections.items():
        bear_dirs = section.bear_dirs()
        bears = list(section.get("bears", ""))
        section_local_bears, section_global_bears = collect_bears(
            bear_dirs,
            bears,
            [BEAR_KIND.LOCAL, BEAR_KIND.GLOBAL],
            log_printer)
        section_local_bears = Dependencies.resolve(section_local_bears)
        section_global_bears = Dependencies.resolve(section_global_bears)
        all_bears = copy.deepcopy(section_local_bears)
        all_bears.extend(section_global_bears)
        fill_section(section, acquire_settings, log_printer, all_bears)

        local_bears[section_name] = section_local_bears
        global_bears[section_name] = section_global_bears

    return local_bears, global_bears
Ejemplo n.º 6
0
def execute_section(section,
                    global_bear_list,
                    local_bear_list,
                    print_results,
                    log_printer):
    """
    Executes the section with the given bears.

    The execute_section method does the following things:
    1. Prepare a Process
      * Load files
      * Create queues
    2. Spawn up one or more Processes
    3. Output results from the Processes
    4. Join all processes

    :param section:          The section to execute.
    :param global_bear_list: List of global bears belonging to the section.
    :param local_bear_list:  List of local bears belonging to the section.
    :param print_results:    Prints all given results appropriate to the
                             output medium.
    :param log_printer:      The log_printer to warn to.
    :return:                 Tuple containing a bool (True if results were
                             yielded, False otherwise), a Manager.dict
                             containing all local results(filenames are key)
                             and a Manager.dict containing all global bear
                             results (bear names are key) as well as the
                             file dictionary.
    """
    local_bear_list = Dependencies.resolve(local_bear_list)
    global_bear_list = Dependencies.resolve(global_bear_list)

    try:
        running_processes = int(section['jobs'])
    except ValueError:
        log_printer.warn("Unable to convert setting 'jobs' into a number. "
                         "Falling back to CPU count.")
        running_processes = get_cpu_count()
    except IndexError:
        running_processes = get_cpu_count()

    processes, arg_dict = instantiate_processes(section,
                                                local_bear_list,
                                                global_bear_list,
                                                running_processes,
                                                log_printer)

    logger_thread = LogPrinterThread(arg_dict["message_queue"],
                                     log_printer)
    # Start and join the logger thread along with the processes to run bears
    processes.append(logger_thread)

    for runner in processes:
        runner.start()

    try:
        return (process_queues(processes,
                               arg_dict["control_queue"],
                               arg_dict["local_result_dict"],
                               arg_dict["global_result_dict"],
                               arg_dict["file_dict"],
                               print_results,
                               section,
                               log_printer),
                arg_dict["local_result_dict"],
                arg_dict["global_result_dict"],
                arg_dict["file_dict"])
    finally:
        logger_thread.running = False

        for runner in processes:
            runner.join()
Ejemplo n.º 7
0
 def test_resolvable_deps(self):
     self.assertEqual(
         Dependencies.resolve([ResolvableBear1, ResolvableBear2]),
         [BearWithoutDeps, ResolvableBear1, ResolvableBear2])
Ejemplo n.º 8
0
 def test_no_deps(self):
     self.assertEqual(
         len(Dependencies.resolve([BearWithoutDeps, BearWithoutDeps])), 1)
Ejemplo n.º 9
0
def execute_section(section, global_bear_list, local_bear_list, print_results,
                    cache, log_printer):
    """
    Executes the section with the given bears.

    The execute_section method does the following things:

    1. Prepare a Process
       -  Load files
       -  Create queues
    2. Spawn up one or more Processes
    3. Output results from the Processes
    4. Join all processes

    :param section:          The section to execute.
    :param global_bear_list: List of global bears belonging to the section.
    :param local_bear_list:  List of local bears belonging to the section.
    :param print_results:    Prints all given results appropriate to the
                             output medium.
    :param cache:            An instance of ``misc.Caching.FileCache`` to use as
                             a file cache buffer.
    :param log_printer:      The log_printer to warn to.
    :return:                 Tuple containing a bool (True if results were
                             yielded, False otherwise), a Manager.dict
                             containing all local results(filenames are key)
                             and a Manager.dict containing all global bear
                             results (bear names are key) as well as the
                             file dictionary.
    """
    local_bear_list = Dependencies.resolve(local_bear_list)
    global_bear_list = Dependencies.resolve(global_bear_list)

    try:
        running_processes = int(section['jobs'])
    except ValueError:
        log_printer.warn("Unable to convert setting 'jobs' into a number. "
                         "Falling back to CPU count.")
        running_processes = get_cpu_count()
    except IndexError:
        running_processes = get_cpu_count()

    processes, arg_dict = instantiate_processes(section, local_bear_list,
                                                global_bear_list,
                                                running_processes, cache,
                                                log_printer)

    logger_thread = LogPrinterThread(arg_dict["message_queue"], log_printer)
    # Start and join the logger thread along with the processes to run bears
    processes.append(logger_thread)

    for runner in processes:
        runner.start()

    try:
        return (process_queues(processes, arg_dict["control_queue"],
                               arg_dict["local_result_dict"],
                               arg_dict["global_result_dict"],
                               arg_dict["file_dict"], print_results, section,
                               cache,
                               log_printer), arg_dict["local_result_dict"],
                arg_dict["global_result_dict"], arg_dict["file_dict"])
    finally:
        logger_thread.running = False

        for runner in processes:
            runner.join()
Ejemplo n.º 10
0
 def test_resolvable_deps(self):
     self.assertEqual(Dependencies.resolve([ResolvableBear1,
                                            ResolvableBear2]),
                      [BearWithoutDeps, ResolvableBear1, ResolvableBear2])
Ejemplo n.º 11
0
 def test_no_deps(self):
     self.assertEqual(
         len(Dependencies.resolve([BearWithoutDeps,
                                   BearWithoutDeps])),
         1)