def test_run(self): log_queue = queue.Queue() self.uut = LogPrinterThread(log_queue) log_queue.put(item='Sample message 1') log_queue.put(item='Sample message 2') log_queue.put(item='Sample message 3') self.assertEqual(self.uut.message_queue.qsize(), 3) with LogCapture() as capture: self.uut.start() while self.uut.message_queue.qsize() > 0: continue self.uut.running = False self.uut.join() capture.check(('root', 'INFO', 'Sample message 1'), ('root', 'INFO', 'Sample message 2'), ('root', 'INFO', 'Sample message 3'))
class LogPrinterThreadTest(unittest.TestCase): def test_run(self): log_printer = TestPrinter() log_queue = queue.Queue() self.uut = LogPrinterThread(log_queue, log_printer) log_queue.put(item="Sample message 1") log_queue.put(item="Sample message 2") log_queue.put(item="Sample message 3") self.assertEqual(self.uut.message_queue.qsize(), 3) with retrieve_stdout() as stdout: self.uut.start() while self.uut.message_queue.qsize() > 0: continue self.uut.running = False self.uut.join() self.assertEqual(stdout.getvalue(), "Sample message 1\nSample message 2\nSample " "message 3\n")
def test_run(self): log_printer = TestPrinter() log_queue = queue.Queue() self.uut = LogPrinterThread(log_queue, log_printer) log_queue.put(item="Sample message 1") log_queue.put(item="Sample message 2") log_queue.put(item="Sample message 3") self.assertEqual(self.uut.message_queue.qsize(), 3) with retrieve_stdout() as stdout: self.uut.start() while self.uut.message_queue.qsize() > 0: continue self.uut.running = False self.uut.join() self.assertEqual(stdout.getvalue(), "Sample message 1\nSample message 2\nSample " "message 3\n")
class LogPrinterThreadTest(unittest.TestCase): def test_run(self): log_printer = TestPrinter() log_queue = queue.Queue() self.uut = LogPrinterThread(log_queue, log_printer) log_queue.put(item='Sample message 1') log_queue.put(item='Sample message 2') log_queue.put(item='Sample message 3') self.assertEqual(self.uut.message_queue.qsize(), 3) with retrieve_stdout() as stdout: self.uut.start() while self.uut.message_queue.qsize() > 0: continue self.uut.running = False self.uut.join() self.assertEqual( stdout.getvalue(), 'Sample message 1\nSample message 2\nSample ' 'message 3\n')
def execute_section(section, global_bear_list, local_bear_list, print_results, log_printer): """ Executes the section with the given bears. The execute_section method does the following things: 1. Prepare a Process * Load files * Create queues 2. Spawn up one or more Processes 3. Output results from the Processes 4. Join all processes :param section: The section to execute. :param global_bear_list: List of global bears belonging to the section. :param local_bear_list: List of local bears belonging to the section. :param print_results: Prints all given results appropriate to the output medium. :param log_printer: The log_printer to warn to. :return: Tuple containing a bool (True if results were yielded, False otherwise), a Manager.dict containing all local results(filenames are key) and a Manager.dict containing all global bear results (bear names are key) as well as the file dictionary. """ local_bear_list = Dependencies.resolve(local_bear_list) global_bear_list = Dependencies.resolve(global_bear_list) try: running_processes = int(section['jobs']) except ValueError: log_printer.warn("Unable to convert setting 'jobs' into a number. " "Falling back to CPU count.") running_processes = get_cpu_count() except IndexError: running_processes = get_cpu_count() processes, arg_dict = instantiate_processes(section, local_bear_list, global_bear_list, running_processes, log_printer) logger_thread = LogPrinterThread(arg_dict["message_queue"], log_printer) # Start and join the logger thread along with the processes to run bears processes.append(logger_thread) for runner in processes: runner.start() try: return (process_queues(processes, arg_dict["control_queue"], arg_dict["local_result_dict"], arg_dict["global_result_dict"], arg_dict["file_dict"], print_results, section, log_printer), arg_dict["local_result_dict"], arg_dict["global_result_dict"], arg_dict["file_dict"]) finally: logger_thread.running = False for runner in processes: runner.join()
def execute_section(section, global_bear_list, local_bear_list, print_results, cache, log_printer, console_printer, debug=False): """ Executes the section with the given bears. The execute_section method does the following things: 1. Prepare a Process - Load files - Create queues 2. Spawn up one or more Processes 3. Output results from the Processes 4. Join all processes :param section: The section to execute. :param global_bear_list: List of global bears belonging to the section. Dependencies are already resolved. :param local_bear_list: List of local bears belonging to the section. Dependencies are already resolved. :param print_results: Prints all given results appropriate to the output medium. :param cache: An instance of ``misc.Caching.FileCache`` to use as a file cache buffer. :param log_printer: The log_printer to warn to. :param console_printer: Object to print messages on the console. :param debug: Bypass multiprocessing and run bears in debug mode, not catching any exceptions. :return: Tuple containing a bool (True if results were yielded, False otherwise), a Manager.dict containing all local results(filenames are key) and a Manager.dict containing all global bear results (bear names are key) as well as the file dictionary. """ if debug: running_processes = 1 else: try: running_processes = int(section['jobs']) except ValueError: log_printer.warn("Unable to convert setting 'jobs' into a number. " 'Falling back to CPU count.') running_processes = get_cpu_count() except IndexError: running_processes = get_cpu_count() bears = global_bear_list + local_bear_list use_raw_files = set(bear.USE_RAW_FILES for bear in bears) if len(use_raw_files) > 1: log_printer.err("Bears that uses raw files can't be mixed with " 'Bears that uses text files. Please move the following ' 'bears to their own section: ' + ', '.join(bear.name for bear in bears if not bear.USE_RAW_FILES)) return ((), {}, {}, {}) # use_raw_files is expected to be only one object. # The if statement is to ensure this doesn't fail when # it's running on an empty run use_raw_files = use_raw_files.pop() if len(use_raw_files) > 0 else False processes, arg_dict = instantiate_processes(section, local_bear_list, global_bear_list, running_processes, cache, log_printer, console_printer=console_printer, debug=debug, use_raw_files=use_raw_files) logger_thread = LogPrinterThread(arg_dict['message_queue'], log_printer) # Start and join the logger thread along with the processes to run bears if not debug: # in debug mode the logging messages are directly processed by the # message_queue processes.append(logger_thread) for runner in processes: runner.start() try: return (process_queues(processes, arg_dict['control_queue'], arg_dict['local_result_dict'], arg_dict['global_result_dict'], arg_dict['file_dict'], print_results, section, cache, log_printer, console_printer=console_printer, debug=debug), arg_dict['local_result_dict'], arg_dict['global_result_dict'], arg_dict['file_dict']) finally: if not debug: # in debug mode multiprocessing and logger_thread are disabled # ==> no need for following actions logger_thread.running = False for runner in processes: runner.join()
def execute_section(section, global_bear_list, local_bear_list, print_results, cache, log_printer): """ Executes the section with the given bears. The execute_section method does the following things: 1. Prepare a Process - Load files - Create queues 2. Spawn up one or more Processes 3. Output results from the Processes 4. Join all processes :param section: The section to execute. :param global_bear_list: List of global bears belonging to the section. :param local_bear_list: List of local bears belonging to the section. :param print_results: Prints all given results appropriate to the output medium. :param cache: An instance of ``misc.Caching.FileCache`` to use as a file cache buffer. :param log_printer: The log_printer to warn to. :return: Tuple containing a bool (True if results were yielded, False otherwise), a Manager.dict containing all local results(filenames are key) and a Manager.dict containing all global bear results (bear names are key) as well as the file dictionary. """ local_bear_list = Dependencies.resolve(local_bear_list) global_bear_list = Dependencies.resolve(global_bear_list) try: running_processes = int(section['jobs']) except ValueError: log_printer.warn("Unable to convert setting 'jobs' into a number. " "Falling back to CPU count.") running_processes = get_cpu_count() except IndexError: running_processes = get_cpu_count() processes, arg_dict = instantiate_processes(section, local_bear_list, global_bear_list, running_processes, cache, log_printer) logger_thread = LogPrinterThread(arg_dict["message_queue"], log_printer) # Start and join the logger thread along with the processes to run bears processes.append(logger_thread) for runner in processes: runner.start() try: return (process_queues(processes, arg_dict["control_queue"], arg_dict["local_result_dict"], arg_dict["global_result_dict"], arg_dict["file_dict"], print_results, section, cache, log_printer), arg_dict["local_result_dict"], arg_dict["global_result_dict"], arg_dict["file_dict"]) finally: logger_thread.running = False for runner in processes: runner.join()
def execute_section(section, global_bear_list, local_bear_list, print_results, cache, log_printer, console_printer, debug=False, apply_single=False): # type: (object, object, object, object, object, object, object, object, # object) -> object """ Executes the section with the given bears. The execute_section method does the following things: 1. Prepare a Process - Load files - Create queues 2. Spawn up one or more Processes 3. Output results from the Processes 4. Join all processes :param section: The section to execute. :param global_bear_list: List of global bears belonging to the section. Dependencies are already resolved. :param local_bear_list: List of local bears belonging to the section. Dependencies are already resolved. :param print_results: Prints all given results appropriate to the output medium. :param cache: An instance of ``misc.Caching.FileCache`` to use as a file cache buffer. :param log_printer: The log_printer to warn to. :param console_printer: Object to print messages on the console. :param debug: Bypass multiprocessing and run bears in debug mode, not catching any exceptions. :param apply_single: The action that should be applied for all results. If it's not selected, has a value of False. :return: Tuple containing a bool (True if results were yielded, False otherwise), a Manager.dict containing all local results(filenames are key) and a Manager.dict containing all global bear results (bear names are key) as well as the file dictionary. """ debug_bears = (False if 'debug_bears' not in section or ( section['debug_bears'].value == 'False') else typed_list(str)(section['debug_bears'])) if debug or debug_bears: running_processes = 1 else: try: running_processes = int(section['jobs']) except ValueError: logging.warning("Unable to convert setting 'jobs' into a number. " 'Falling back to CPU count.') running_processes = get_cpu_count() except IndexError: running_processes = get_cpu_count() bears = global_bear_list + local_bear_list use_raw_files = set(bear.USE_RAW_FILES for bear in bears) if len(use_raw_files) > 1: logging.error("Bears that uses raw files can't be mixed with Bears " 'that uses text files. Please move the following bears ' 'to their own section: ' + ', '.join(bear.name for bear in bears if not bear.USE_RAW_FILES)) return ((), {}, {}, {}) # use_raw_files is expected to be only one object. # The if statement is to ensure this doesn't fail when # it's running on an empty run use_raw_files = use_raw_files.pop() if len(use_raw_files) > 0 else False processes, arg_dict = instantiate_processes(section, local_bear_list, global_bear_list, running_processes, cache, None, console_printer=console_printer, debug=debug, use_raw_files=use_raw_files, debug_bears=debug_bears) logger_thread = LogPrinterThread(arg_dict['message_queue']) # Start and join the logger thread along with the processes to run bears if not (debug or debug_bears): # in debug mode the logging messages are directly processed by the # message_queue processes.append(logger_thread) for runner in processes: runner.start() try: return (process_queues(processes, arg_dict['control_queue'], arg_dict['local_result_dict'], arg_dict['global_result_dict'], arg_dict['file_dict'], print_results, section, cache, None, console_printer=console_printer, debug=debug, apply_single=apply_single, debug_bears=debug_bears), arg_dict['local_result_dict'], arg_dict['global_result_dict'], arg_dict['file_dict']) finally: if not (debug or debug_bears): # in debug mode multiprocessing and logger_thread are disabled # ==> no need for following actions logger_thread.running = False for runner in processes: runner.join()
def execute_section(section, global_bear_list, local_bear_list, print_results, cache, log_printer, console_printer, debug=False): """ Executes the section with the given bears. The execute_section method does the following things: 1. Prepare a Process - Load files - Create queues 2. Spawn up one or more Processes 3. Output results from the Processes 4. Join all processes :param section: The section to execute. :param global_bear_list: List of global bears belonging to the section. Dependencies are already resolved. :param local_bear_list: List of local bears belonging to the section. Dependencies are already resolved. :param print_results: Prints all given results appropriate to the output medium. :param cache: An instance of ``misc.Caching.FileCache`` to use as a file cache buffer. :param log_printer: The log_printer to warn to. :param console_printer: Object to print messages on the console. :param debug: Bypass multiprocessing and run bears in debug mode, not catching any exceptions. :return: Tuple containing a bool (True if results were yielded, False otherwise), a Manager.dict containing all local results(filenames are key) and a Manager.dict containing all global bear results (bear names are key) as well as the file dictionary. """ if debug: running_processes = 1 else: try: running_processes = int(section['jobs']) except ValueError: log_printer.warn("Unable to convert setting 'jobs' into a number. " 'Falling back to CPU count.') running_processes = get_cpu_count() except IndexError: running_processes = get_cpu_count() processes, arg_dict = instantiate_processes(section, local_bear_list, global_bear_list, running_processes, cache, log_printer, console_printer=console_printer, debug=debug) logger_thread = LogPrinterThread(arg_dict['message_queue'], log_printer) # Start and join the logger thread along with the processes to run bears if not debug: # in debug mode the logging messages are directly processed by the # message_queue processes.append(logger_thread) for runner in processes: runner.start() try: return (process_queues(processes, arg_dict['control_queue'], arg_dict['local_result_dict'], arg_dict['global_result_dict'], arg_dict['file_dict'], print_results, section, cache, log_printer, console_printer=console_printer, debug=debug), arg_dict['local_result_dict'], arg_dict['global_result_dict'], arg_dict['file_dict']) finally: if not debug: # in debug mode multiprocessing and logger_thread are disabled # ==> no need for following actions logger_thread.running = False for runner in processes: runner.join()