def __init__(self, debug_output=False, enable_profiling=False, profiling_directory=None, profiling_sample_rate=1000, profiling_type=u'all'): """Initializes an engine object. Args: debug_output (Optional[bool]): True if debug output should be enabled. enable_profiling (Optional[bool]): True if profiling should be enabled. profiling_directory (Optional[str]): path to the directory where the profiling sample files should be stored. profiling_sample_rate (Optional[int]): profiling sample rate. Contains the number of event sources processed. profiling_type (Optional[str]): type of profiling. Supported types are: * 'memory' to profile memory usage; * 'parsers' to profile CPU time consumed by individual parsers; * 'processing' to profile CPU time consumed by different parts of the processing; * 'serializers' to profile CPU time consumed by individual serializers. """ super(BaseEngine, self).__init__() self._abort = False self._debug_output = debug_output self._enable_profiling = enable_profiling self._processing_status = processing_status.ProcessingStatus() self._profiling_directory = profiling_directory self._profiling_sample_rate = profiling_sample_rate self._profiling_type = profiling_type self.knowledge_base = knowledge_base.KnowledgeBase()
def testPrintExtractionStatusUpdateLinear(self): """Tests the PrintExtractionStatusUpdateLinear function.""" output_writer = test_lib.TestOutputWriter() test_view = status_view.StatusView(output_writer, 'test_tool') test_view.SetSourceInformation('/test/source/path', dfvfs_definitions.SOURCE_TYPE_DIRECTORY) process_status = processing_status.ProcessingStatus() process_status.UpdateForemanStatus('f_identifier', 'f_status', 123, 0, 'f_test_file', 1, 29, 3, 456, 5, 6, 9, 10, 7, 8) test_view._PrintExtractionStatusUpdateLinear(process_status) output = output_writer.ReadOutput() self.assertEqual(output, '') process_status.UpdateWorkerStatus('w_identifier', 'w_status', 123, 0, 'w_test_file', 1, 2, 3, 4, 5, 6, 9, 10, 7, 8) test_view._PrintExtractionStatusUpdateLinear(process_status) expected_output = ('w_identifier (PID: 123) - events produced: 4 - ' 'file: w_test_file - running: True\n') output = output_writer.ReadOutput() self.assertEqual(output, expected_output)
def __init__(self): """Initializes an engine.""" super(BaseEngine, self).__init__() self._abort = False self._processing_status = processing_status.ProcessingStatus() self.knowledge_base = knowledge_base.KnowledgeBase()
def testPrintProcessingTime(self): """Tests the _PrintProcessingTime function.""" output_writer = test_lib.TestOutputWriter() process_status = processing_status.ProcessingStatus() test_view = status_view.StatusView(output_writer, 'test_tool') test_view.SetSourceInformation('/test/source/path', dfvfs_definitions.SOURCE_TYPE_DIRECTORY) process_status.start_time = 0 test_view._PrintProcessingTime(process_status) expected_output = 'Processing time\t\t: 00:00:00\n' output = output_writer.ReadOutput() self.assertEqual(output, expected_output) self._mocked_time = 12 * 60 * 60 + 31 * 60 + 15 test_view._PrintProcessingTime(process_status) expected_output = 'Processing time\t\t: 12:31:15\n' output = output_writer.ReadOutput() self.assertEqual(output, expected_output) self._mocked_time = 24 * 60 * 60 test_view._PrintProcessingTime(process_status) expected_output = 'Processing time\t\t: 1 day, 00:00:00\n' output = output_writer.ReadOutput() self.assertEqual(output, expected_output) self._mocked_time = 5 * 24 * 60 * 60 + 5 * 60 * 60 + 61 test_view._PrintProcessingTime(process_status) expected_output = 'Processing time\t\t: 5 days, 05:01:01\n' output = output_writer.ReadOutput() self.assertEqual(output, expected_output)
def testFormatProcessingTime(self): """Tests the _FormatProcessingTime function.""" output_writer = test_lib.TestOutputWriter() process_status = processing_status.ProcessingStatus() test_view = status_view.StatusView(output_writer, 'test_tool') test_view.SetSourceInformation('/test/source/path', dfvfs_definitions.SOURCE_TYPE_DIRECTORY) process_status.start_time = 0 processing_time = test_view._FormatProcessingTime(process_status) self.assertEqual(processing_time, '00:00:00') self._mocked_time = 12 * 60 * 60 + 31 * 60 + 15 processing_time = test_view._FormatProcessingTime(process_status) self.assertEqual(processing_time, '12:31:15') self._mocked_time = 24 * 60 * 60 processing_time = test_view._FormatProcessingTime(process_status) self.assertEqual(processing_time, '1 day, 00:00:00') self._mocked_time = 5 * 24 * 60 * 60 + 5 * 60 * 60 + 61 processing_time = test_view._FormatProcessingTime(process_status) self.assertEqual(processing_time, '5 days, 05:01:01')
def testPrintExtractionStatusUpdateLinear(self): """Tests the PrintExtractionStatusUpdateLinear function.""" output_writer = test_lib.TestOutputWriter() test_view = status_view.StatusView(output_writer, u'test_tool') test_view.SetSourceInformation(u'/test/source/path', dfvfs_definitions.SOURCE_TYPE_DIRECTORY) process_status = processing_status.ProcessingStatus() process_status.UpdateForemanStatus(u'f_identifier', u'f_status', 123, 0, u'f_test_file', 1, 29, 3, 456, 5, 6, 7, 8, 9, 10) test_view._PrintExtractionStatusUpdateLinear(process_status) string = output_writer.ReadOutput() expected_lines = [b''] self.assertEqual(string.split(b'\n'), expected_lines) process_status.UpdateWorkerStatus(u'w_identifier', u'w_status', 123, 0, u'w_test_file', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) test_view._PrintExtractionStatusUpdateLinear(process_status) string = output_writer.ReadOutput() expected_string = (u'w_identifier (PID: 123) - events produced: 4 - ' u'file: w_test_file - running: True\n') self.assertEqual(string, expected_string)
def testUpdateProcessStatus(self): """Tests the _UpdateProcessStatus function.""" process_status = processing_status.ProcessStatus() status = processing_status.ProcessingStatus() status._UpdateProcessStatus(process_status, 'test', 'Idle', 12345, 2000000, 'test process', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
def __init__(self): """Initializes an engine.""" super(BaseEngine, self).__init__() self._abort = False self._guppy_memory_profiler = None self._memory_profiler = None self._name = 'Main' self._processing_status = processing_status.ProcessingStatus() self._processing_profiler = None self._serializers_profiler = None self._storage_profiler = None self._task_queue_profiler = None self.knowledge_base = knowledge_base.KnowledgeBase()
def __init__(self, path_spec_queue, event_object_queue, parse_error_queue): """Initialize the engine object. Args: path_spec_queue: the path specification queue object (instance of Queue). event_object_queue: the event object queue object (instance of Queue). parse_error_queue: the parser error queue object (instance of Queue). """ self._enable_debug_output = False self._enable_profiling = False self._event_object_queue = event_object_queue self._path_spec_queue = path_spec_queue self._parse_error_queue = parse_error_queue self._processing_status = processing_status.ProcessingStatus() self._profiling_sample_rate = 1000 self._profiling_type = u'all' self.knowledge_base = knowledge_base.KnowledgeBase()
def testPrintStatusUpdateStream(self): """Tests the PrintStatusUpdateStream function.""" input_reader = tools.StdinInputReader(encoding=u'ascii') output_writer = test_lib.TestOutputWriter() status_view_tool = TestStatusViewTool( input_reader=input_reader, output_writer=output_writer) status_view_tool._PrintStatusHeader() process_status = processing_status.ProcessingStatus() process_status.UpdateForemanStatus( u'f_identifier', u'f_status', 123, 0, u'f_test_file', 1, 29, 3, 456, 5, 6, 7, 8, 9, 10) status_view_tool._PrintStatusUpdateStream(process_status) string = output_writer.ReadOutput() expected_lines = [ b'Source path\t: /test/source/path', b'Source type\t: TESTSOURCE', b'', b''] self.assertEqual(string.split(b'\n'), expected_lines) process_status.UpdateWorkerStatus( u'w_identifier', u'w_status', 123, 0, u'w_test_file', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) status_view_tool._PrintStatusUpdateStream(process_status) string = output_writer.ReadOutput() expected_string = ( u'w_identifier (PID: 123) - events produced: 4 - ' u'file: w_test_file - running: True\n') self.assertEqual(string, expected_string)
def testPrintExtractionStatusUpdateWindow(self): """Tests the _PrintExtractionStatusUpdateWindow function.""" output_writer = test_lib.TestOutputWriter() test_view = status_view.StatusView(output_writer, 'test_tool') test_view.SetSourceInformation('/test/source/path', dfvfs_definitions.SOURCE_TYPE_DIRECTORY) process_status = processing_status.ProcessingStatus() process_status.UpdateForemanStatus('f_identifier', 'f_status', 123, 0, 'f_test_file', 1, 29, 3, 456, 5, 6, 9, 10, 7, 8) test_view._PrintExtractionStatusUpdateWindow(process_status) table_header = ('Identifier ' 'PID ' 'Status ' 'Memory ' 'Sources ' 'Events ' 'File') if not sys.platform.startswith('win'): table_header = '\x1b[1m{0:s}\x1b[0m'.format(table_header) expected_output = [ 'plaso - test_tool version {0:s}'.format(plaso.__version__), '', 'Source path\t\t: /test/source/path', 'Source type\t\t: directory', 'Processing time\t\t: 00:00:00', '', table_header, ('f_identifier ' '123 ' 'f_status ' '0 B ' '29 (29) ' '456 (456) ' 'f_test_file'), '', '' ] output = output_writer.ReadOutput() self._CheckOutput(output, expected_output) process_status.UpdateWorkerStatus('w_identifier', 'w_status', 123, 0, 'w_test_file', 1, 2, 3, 4, 5, 6, 9, 10, 7, 8) test_view._PrintExtractionStatusUpdateWindow(process_status) expected_output = [ 'plaso - test_tool version {0:s}'.format(plaso.__version__), '', 'Source path\t\t: /test/source/path', 'Source type\t\t: directory', 'Processing time\t\t: 00:00:00', '', table_header, ('f_identifier ' '123 ' 'f_status ' '0 B ' '29 (29) ' '456 (456) ' 'f_test_file'), ('w_identifier ' '123 ' 'w_status ' '0 B ' '2 (2) ' '4 (4) ' 'w_test_file'), '', '' ] output = output_writer.ReadOutput() self._CheckOutput(output, expected_output)
def testPrintStatusUpdate(self): """Tests the PrintStatusUpdate function.""" input_reader = tools.StdinInputReader(encoding=u'ascii') output_writer = test_lib.TestOutputWriter() status_view_tool = TestStatusViewTool( input_reader=input_reader, output_writer=output_writer) status_view_tool._PrintStatusHeader() process_status = processing_status.ProcessingStatus() process_status.UpdateForemanStatus( u'f_identifier', u'f_status', 123, 0, u'f_test_file', 1, 29, 3, 456, 5, 6, 7, 8, 9, 10) status_view_tool._PrintStatusUpdate(process_status) string = output_writer.ReadOutput() plaso_version = plaso.GetVersion() table_header = ( b'Identifier\tPID\tStatus\t\tMemory\t\tSources\t\tEvents\t\tFile') if not sys.platform.startswith(u'win'): table_header = b'\x1b[1m{0:s}\x1b[0m'.format(table_header) expected_lines = [ b'Source path\t: /test/source/path', b'Source type\t: TESTSOURCE', b'', b'plaso - version {0:s}'.format(plaso_version), b'', b'Source path\t: /test/source/path', b'Source type\t: TESTSOURCE', b'', table_header, (b'f_identifier\t123\tf_status\t0 B\t\t29 (29)\t\t456 (456)\t' b'f_test_file'), b'', b''] self.assertEqual(string.split(b'\n'), expected_lines) process_status.UpdateWorkerStatus( u'w_identifier', u'w_status', 123, 0, u'w_test_file', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) status_view_tool._PrintStatusUpdate(process_status) string = output_writer.ReadOutput() expected_lines = [ b'plaso - version {0:s}'.format(plaso_version), b'', b'Source path\t: /test/source/path', b'Source type\t: TESTSOURCE', b'', table_header, (b'f_identifier\t123\tf_status\t0 B\t\t29 (29)\t\t456 (456)\t' b'f_test_file'), b'w_identifier\t123\tw_status\t0 B\t\t2 (2)\t\t4 (4)\t\tw_test_file', b'', b''] self.assertEqual(string.split(b'\n'), expected_lines)
def testUpdateTasksStatus(self): """Tests the UpdateTasksStatus function.""" task_status = processing_status.TasksStatus() status = processing_status.ProcessingStatus() status.UpdateTasksStatus(task_status)
def testUpdateForemanStatus(self): """Tests the UpdateForemanStatus function.""" status = processing_status.ProcessingStatus() status.UpdateForemanStatus('test', 'Idle', 12345, 2000000, 'test process', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
def testWorkersStatus(self): """Tests the workers_status property.""" status = processing_status.ProcessingStatus() self.assertEqual(status.workers_status, [])
def testPrintExtractionStatusUpdateWindow(self): """Tests the _PrintExtractionStatusUpdateWindow function.""" output_writer = test_lib.TestOutputWriter() test_view = status_view.StatusView(output_writer, u'test_tool') test_view.SetSourceInformation(u'/test/source/path', dfvfs_definitions.SOURCE_TYPE_DIRECTORY) process_status = processing_status.ProcessingStatus() process_status.UpdateForemanStatus(u'f_identifier', u'f_status', 123, 0, u'f_test_file', 1, 29, 3, 456, 5, 6, 7, 8, 9, 10) test_view._PrintExtractionStatusUpdateWindow(process_status) string = output_writer.ReadOutput() table_header = (b'Identifier ' b'PID ' b'Status ' b'Memory ' b'Sources ' b'Events ' b'File') if not sys.platform.startswith(u'win'): table_header = b'\x1b[1m{0:s}\x1b[0m'.format(table_header) expected_lines = [ b'plaso - test_tool version {0:s}'.format(plaso.__version__), b'', b'Source path\t: /test/source/path', b'Source type\t: directory', b'', table_header, (b'f_identifier ' b'123 ' b'f_status ' b'0 B ' b'29 (29) ' b'456 (456) ' b'f_test_file'), b'', b'' ] self.assertEqual(string.split(b'\n'), expected_lines) process_status.UpdateWorkerStatus(u'w_identifier', u'w_status', 123, 0, u'w_test_file', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) test_view._PrintExtractionStatusUpdateWindow(process_status) string = output_writer.ReadOutput() expected_lines = [ b'plaso - test_tool version {0:s}'.format(plaso.__version__), b'', b'Source path\t: /test/source/path', b'Source type\t: directory', b'', table_header, (b'f_identifier ' b'123 ' b'f_status ' b'0 B ' b'29 (29) ' b'456 (456) ' b'f_test_file'), (b'w_identifier ' b'123 ' b'w_status ' b'0 B ' b'2 (2) ' b'4 (4) ' b'w_test_file'), b'', b'' ] self.assertEqual(string.split(b'\n'), expected_lines)