def __init__(self,
debug_output=False,
enable_profiling=False,
profiling_directory=None,
profiling_sample_rate=1000,
profiling_type=u'all'):
"""Initializes an engine object.
Args:
debug_output (Optional[bool]): True if debug output should be enabled.
enable_profiling (Optional[bool]): True if profiling should be enabled.
profiling_directory (Optional[str]): path to the directory where
the profiling sample files should be stored.
profiling_sample_rate (Optional[int]): profiling sample rate.
Contains the number of event sources processed.
profiling_type (Optional[str]): type of profiling.
Supported types are:
* 'memory' to profile memory usage;
* 'parsers' to profile CPU time consumed by individual parsers;
* 'processing' to profile CPU time consumed by different parts of
the processing;
* 'serializers' to profile CPU time consumed by individual
serializers.
"""
super(BaseEngine, self).__init__()
self._abort = False
self._debug_output = debug_output
self._enable_profiling = enable_profiling
self._processing_status = processing_status.ProcessingStatus()
self._profiling_directory = profiling_directory
self._profiling_sample_rate = profiling_sample_rate
self._profiling_type = profiling_type
self.knowledge_base = knowledge_base.KnowledgeBase()
def testPrintExtractionStatusUpdateLinear(self):
"""Tests the PrintExtractionStatusUpdateLinear function."""
output_writer = test_lib.TestOutputWriter()
test_view = status_view.StatusView(output_writer, 'test_tool')
test_view.SetSourceInformation('/test/source/path',
dfvfs_definitions.SOURCE_TYPE_DIRECTORY)
process_status = processing_status.ProcessingStatus()
process_status.UpdateForemanStatus('f_identifier', 'f_status', 123, 0,
'f_test_file', 1, 29, 3, 456, 5, 6,
9, 10, 7, 8)
test_view._PrintExtractionStatusUpdateLinear(process_status)
output = output_writer.ReadOutput()
self.assertEqual(output, '')
process_status.UpdateWorkerStatus('w_identifier', 'w_status', 123, 0,
'w_test_file', 1, 2, 3, 4, 5, 6, 9,
10, 7, 8)
test_view._PrintExtractionStatusUpdateLinear(process_status)
expected_output = ('w_identifier (PID: 123) - events produced: 4 - '
'file: w_test_file - running: True\n')
output = output_writer.ReadOutput()
self.assertEqual(output, expected_output)
def __init__(self):
"""Initializes an engine."""
super(BaseEngine, self).__init__()
self._abort = False
self._processing_status = processing_status.ProcessingStatus()
self.knowledge_base = knowledge_base.KnowledgeBase()
def testPrintProcessingTime(self):
"""Tests the _PrintProcessingTime function."""
output_writer = test_lib.TestOutputWriter()
process_status = processing_status.ProcessingStatus()
test_view = status_view.StatusView(output_writer, 'test_tool')
test_view.SetSourceInformation('/test/source/path',
dfvfs_definitions.SOURCE_TYPE_DIRECTORY)
process_status.start_time = 0
test_view._PrintProcessingTime(process_status)
expected_output = 'Processing time\t\t: 00:00:00\n'
output = output_writer.ReadOutput()
self.assertEqual(output, expected_output)
self._mocked_time = 12 * 60 * 60 + 31 * 60 + 15
test_view._PrintProcessingTime(process_status)
expected_output = 'Processing time\t\t: 12:31:15\n'
output = output_writer.ReadOutput()
self.assertEqual(output, expected_output)
self._mocked_time = 24 * 60 * 60
test_view._PrintProcessingTime(process_status)
expected_output = 'Processing time\t\t: 1 day, 00:00:00\n'
output = output_writer.ReadOutput()
self.assertEqual(output, expected_output)
self._mocked_time = 5 * 24 * 60 * 60 + 5 * 60 * 60 + 61
test_view._PrintProcessingTime(process_status)
expected_output = 'Processing time\t\t: 5 days, 05:01:01\n'
output = output_writer.ReadOutput()
self.assertEqual(output, expected_output)
def testFormatProcessingTime(self):
"""Tests the _FormatProcessingTime function."""
output_writer = test_lib.TestOutputWriter()
process_status = processing_status.ProcessingStatus()
test_view = status_view.StatusView(output_writer, 'test_tool')
test_view.SetSourceInformation('/test/source/path',
dfvfs_definitions.SOURCE_TYPE_DIRECTORY)
process_status.start_time = 0
processing_time = test_view._FormatProcessingTime(process_status)
self.assertEqual(processing_time, '00:00:00')
self._mocked_time = 12 * 60 * 60 + 31 * 60 + 15
processing_time = test_view._FormatProcessingTime(process_status)
self.assertEqual(processing_time, '12:31:15')
self._mocked_time = 24 * 60 * 60
processing_time = test_view._FormatProcessingTime(process_status)
self.assertEqual(processing_time, '1 day, 00:00:00')
self._mocked_time = 5 * 24 * 60 * 60 + 5 * 60 * 60 + 61
processing_time = test_view._FormatProcessingTime(process_status)
self.assertEqual(processing_time, '5 days, 05:01:01')
def testPrintExtractionStatusUpdateLinear(self):
"""Tests the PrintExtractionStatusUpdateLinear function."""
output_writer = test_lib.TestOutputWriter()
test_view = status_view.StatusView(output_writer, u'test_tool')
test_view.SetSourceInformation(u'/test/source/path',
dfvfs_definitions.SOURCE_TYPE_DIRECTORY)
process_status = processing_status.ProcessingStatus()
process_status.UpdateForemanStatus(u'f_identifier', u'f_status', 123,
0, u'f_test_file', 1, 29, 3, 456, 5,
6, 7, 8, 9, 10)
test_view._PrintExtractionStatusUpdateLinear(process_status)
string = output_writer.ReadOutput()
expected_lines = [b'']
self.assertEqual(string.split(b'\n'), expected_lines)
process_status.UpdateWorkerStatus(u'w_identifier', u'w_status', 123, 0,
u'w_test_file', 1, 2, 3, 4, 5, 6, 7,
8, 9, 10)
test_view._PrintExtractionStatusUpdateLinear(process_status)
string = output_writer.ReadOutput()
expected_string = (u'w_identifier (PID: 123) - events produced: 4 - '
u'file: w_test_file - running: True\n')
self.assertEqual(string, expected_string)
def testUpdateProcessStatus(self):
"""Tests the _UpdateProcessStatus function."""
process_status = processing_status.ProcessStatus()
status = processing_status.ProcessingStatus()
status._UpdateProcessStatus(process_status, 'test', 'Idle', 12345,
2000000, 'test process', 0, 0, 0, 0, 0, 0,
0, 0, 0, 0)
def __init__(self):
"""Initializes an engine."""
super(BaseEngine, self).__init__()
self._abort = False
self._guppy_memory_profiler = None
self._memory_profiler = None
self._name = 'Main'
self._processing_status = processing_status.ProcessingStatus()
self._processing_profiler = None
self._serializers_profiler = None
self._storage_profiler = None
self._task_queue_profiler = None
self.knowledge_base = knowledge_base.KnowledgeBase()
def __init__(self, path_spec_queue, event_object_queue, parse_error_queue):
"""Initialize the engine object.
Args:
path_spec_queue: the path specification queue object (instance of Queue).
event_object_queue: the event object queue object (instance of Queue).
parse_error_queue: the parser error queue object (instance of Queue).
"""
self._enable_debug_output = False
self._enable_profiling = False
self._event_object_queue = event_object_queue
self._path_spec_queue = path_spec_queue
self._parse_error_queue = parse_error_queue
self._processing_status = processing_status.ProcessingStatus()
self._profiling_sample_rate = 1000
self._profiling_type = u'all'
self.knowledge_base = knowledge_base.KnowledgeBase()
def testPrintStatusUpdateStream(self):
"""Tests the PrintStatusUpdateStream function."""
input_reader = tools.StdinInputReader(encoding=u'ascii')
output_writer = test_lib.TestOutputWriter()
status_view_tool = TestStatusViewTool(
input_reader=input_reader, output_writer=output_writer)
status_view_tool._PrintStatusHeader()
process_status = processing_status.ProcessingStatus()
process_status.UpdateForemanStatus(
u'f_identifier', u'f_status', 123, 0,
u'f_test_file', 1, 29, 3, 456, 5, 6, 7,
8, 9, 10)
status_view_tool._PrintStatusUpdateStream(process_status)
string = output_writer.ReadOutput()
expected_lines = [
b'Source path\t: /test/source/path',
b'Source type\t: TESTSOURCE',
b'',
b'']
self.assertEqual(string.split(b'\n'), expected_lines)
process_status.UpdateWorkerStatus(
u'w_identifier', u'w_status', 123, 0,
u'w_test_file', 1, 2, 3, 4, 5, 6, 7, 8, 9,
10)
status_view_tool._PrintStatusUpdateStream(process_status)
string = output_writer.ReadOutput()
expected_string = (
u'w_identifier (PID: 123) - events produced: 4 - '
u'file: w_test_file - running: True\n')
self.assertEqual(string, expected_string)
def testPrintExtractionStatusUpdateWindow(self):
"""Tests the _PrintExtractionStatusUpdateWindow function."""
output_writer = test_lib.TestOutputWriter()
test_view = status_view.StatusView(output_writer, 'test_tool')
test_view.SetSourceInformation('/test/source/path',
dfvfs_definitions.SOURCE_TYPE_DIRECTORY)
process_status = processing_status.ProcessingStatus()
process_status.UpdateForemanStatus('f_identifier', 'f_status', 123, 0,
'f_test_file', 1, 29, 3, 456, 5, 6,
9, 10, 7, 8)
test_view._PrintExtractionStatusUpdateWindow(process_status)
table_header = ('Identifier '
'PID '
'Status '
'Memory '
'Sources '
'Events '
'File')
if not sys.platform.startswith('win'):
table_header = '\x1b[1m{0:s}\x1b[0m'.format(table_header)
expected_output = [
'plaso - test_tool version {0:s}'.format(plaso.__version__), '',
'Source path\t\t: /test/source/path', 'Source type\t\t: directory',
'Processing time\t\t: 00:00:00', '', table_header,
('f_identifier '
'123 '
'f_status '
'0 B '
'29 (29) '
'456 (456) '
'f_test_file'), '', ''
]
output = output_writer.ReadOutput()
self._CheckOutput(output, expected_output)
process_status.UpdateWorkerStatus('w_identifier', 'w_status', 123, 0,
'w_test_file', 1, 2, 3, 4, 5, 6, 9,
10, 7, 8)
test_view._PrintExtractionStatusUpdateWindow(process_status)
expected_output = [
'plaso - test_tool version {0:s}'.format(plaso.__version__), '',
'Source path\t\t: /test/source/path', 'Source type\t\t: directory',
'Processing time\t\t: 00:00:00', '', table_header,
('f_identifier '
'123 '
'f_status '
'0 B '
'29 (29) '
'456 (456) '
'f_test_file'),
('w_identifier '
'123 '
'w_status '
'0 B '
'2 (2) '
'4 (4) '
'w_test_file'), '', ''
]
output = output_writer.ReadOutput()
self._CheckOutput(output, expected_output)
def testPrintStatusUpdate(self):
"""Tests the PrintStatusUpdate function."""
input_reader = tools.StdinInputReader(encoding=u'ascii')
output_writer = test_lib.TestOutputWriter()
status_view_tool = TestStatusViewTool(
input_reader=input_reader, output_writer=output_writer)
status_view_tool._PrintStatusHeader()
process_status = processing_status.ProcessingStatus()
process_status.UpdateForemanStatus(
u'f_identifier', u'f_status', 123, 0,
u'f_test_file', 1, 29, 3, 456, 5, 6, 7,
8, 9, 10)
status_view_tool._PrintStatusUpdate(process_status)
string = output_writer.ReadOutput()
plaso_version = plaso.GetVersion()
table_header = (
b'Identifier\tPID\tStatus\t\tMemory\t\tSources\t\tEvents\t\tFile')
if not sys.platform.startswith(u'win'):
table_header = b'\x1b[1m{0:s}\x1b[0m'.format(table_header)
expected_lines = [
b'Source path\t: /test/source/path',
b'Source type\t: TESTSOURCE',
b'',
b'plaso - version {0:s}'.format(plaso_version),
b'',
b'Source path\t: /test/source/path',
b'Source type\t: TESTSOURCE',
b'',
table_header,
(b'f_identifier\t123\tf_status\t0 B\t\t29 (29)\t\t456 (456)\t'
b'f_test_file'),
b'',
b'']
self.assertEqual(string.split(b'\n'), expected_lines)
process_status.UpdateWorkerStatus(
u'w_identifier', u'w_status', 123, 0,
u'w_test_file', 1, 2, 3, 4, 5, 6, 7, 8, 9,
10)
status_view_tool._PrintStatusUpdate(process_status)
string = output_writer.ReadOutput()
expected_lines = [
b'plaso - version {0:s}'.format(plaso_version),
b'',
b'Source path\t: /test/source/path',
b'Source type\t: TESTSOURCE',
b'',
table_header,
(b'f_identifier\t123\tf_status\t0 B\t\t29 (29)\t\t456 (456)\t'
b'f_test_file'),
b'w_identifier\t123\tw_status\t0 B\t\t2 (2)\t\t4 (4)\t\tw_test_file',
b'',
b'']
self.assertEqual(string.split(b'\n'), expected_lines)
def testUpdateTasksStatus(self):
"""Tests the UpdateTasksStatus function."""
task_status = processing_status.TasksStatus()
status = processing_status.ProcessingStatus()
status.UpdateTasksStatus(task_status)
def testUpdateForemanStatus(self):
"""Tests the UpdateForemanStatus function."""
status = processing_status.ProcessingStatus()
status.UpdateForemanStatus('test', 'Idle', 12345, 2000000,
'test process', 0, 0, 0, 0, 0, 0, 0, 0, 0,
0)
def testWorkersStatus(self):
"""Tests the workers_status property."""
status = processing_status.ProcessingStatus()
self.assertEqual(status.workers_status, [])
def testPrintExtractionStatusUpdateWindow(self):
"""Tests the _PrintExtractionStatusUpdateWindow function."""
output_writer = test_lib.TestOutputWriter()
test_view = status_view.StatusView(output_writer, u'test_tool')
test_view.SetSourceInformation(u'/test/source/path',
dfvfs_definitions.SOURCE_TYPE_DIRECTORY)
process_status = processing_status.ProcessingStatus()
process_status.UpdateForemanStatus(u'f_identifier', u'f_status', 123,
0, u'f_test_file', 1, 29, 3, 456, 5,
6, 7, 8, 9, 10)
test_view._PrintExtractionStatusUpdateWindow(process_status)
string = output_writer.ReadOutput()
table_header = (b'Identifier '
b'PID '
b'Status '
b'Memory '
b'Sources '
b'Events '
b'File')
if not sys.platform.startswith(u'win'):
table_header = b'\x1b[1m{0:s}\x1b[0m'.format(table_header)
expected_lines = [
b'plaso - test_tool version {0:s}'.format(plaso.__version__), b'',
b'Source path\t: /test/source/path', b'Source type\t: directory',
b'', table_header,
(b'f_identifier '
b'123 '
b'f_status '
b'0 B '
b'29 (29) '
b'456 (456) '
b'f_test_file'), b'', b''
]
self.assertEqual(string.split(b'\n'), expected_lines)
process_status.UpdateWorkerStatus(u'w_identifier', u'w_status', 123, 0,
u'w_test_file', 1, 2, 3, 4, 5, 6, 7,
8, 9, 10)
test_view._PrintExtractionStatusUpdateWindow(process_status)
string = output_writer.ReadOutput()
expected_lines = [
b'plaso - test_tool version {0:s}'.format(plaso.__version__), b'',
b'Source path\t: /test/source/path', b'Source type\t: directory',
b'', table_header,
(b'f_identifier '
b'123 '
b'f_status '
b'0 B '
b'29 (29) '
b'456 (456) '
b'f_test_file'),
(b'w_identifier '
b'123 '
b'w_status '
b'0 B '
b'2 (2) '
b'4 (4) '
b'w_test_file'), b'', b''
]
self.assertEqual(string.split(b'\n'), expected_lines)
