def testViperLookup(self):
"""Tests for the Viper analysis plugin."""
event_queue = single_process.SingleProcessQueue()
knowledge_base = self._SetUpKnowledgeBase()
# Fill the incoming queue with events.
test_queue_producer = plaso_queue.ItemQueueProducer(event_queue)
events = [
self._CreateTestEventObject(test_event)
for test_event in self.TEST_EVENTS
]
test_queue_producer.ProduceItems(events)
# Set up the plugin.
analysis_plugin = viper.ViperAnalysisPlugin(event_queue)
analysis_plugin.SetProtocol(u'http')
analysis_plugin.SetHost(u'localhost')
# Run the analysis plugin.
analysis_report_queue_consumer = self._RunAnalysisPlugin(
analysis_plugin, knowledge_base)
analysis_reports = self._GetAnalysisReportsFromQueue(
analysis_report_queue_consumer)
self.assertEqual(len(analysis_reports), 1)
report = analysis_reports[0]
tags = report.GetTags()
self.assertEqual(len(tags), 1)
tag = tags[0]
self.assertEqual(tag.event_uuid, u'8')
expected_labels = [
u'viper_present', u'viper_project_default', u'viper_tag_rat',
u'viper_tag_darkcomet'
]
self.assertEqual(tag.labels, expected_labels)
def _ParseFile(self, parser_object, path, knowledge_base_object):
"""Parses a file using the parser object.
Args:
parser_object: the parser object.
path: the path of the file to parse.
knowledge_base_object: the knowledge base object (instance of
KnowledgeBase).
Returns:
An event object queue object (instance of Queue).
"""
event_queue = single_process.SingleProcessQueue()
event_queue_producer = plaso_queue.ItemQueueProducer(event_queue)
parse_error_queue = single_process.SingleProcessQueue()
parser_mediator = parsers_mediator.ParserMediator(
event_queue_producer, parse_error_queue, knowledge_base_object)
path_spec = path_spec_factory.Factory.NewPathSpec(
definitions.TYPE_INDICATOR_OS, location=path)
file_entry = path_spec_resolver.Resolver.OpenFileEntry(path_spec)
parser_mediator.SetFileEntry(file_entry)
file_object = file_entry.GetFileObject()
try:
parser_object.Parse(parser_mediator, file_object)
finally:
file_object.close()
return event_queue
def testUniqueDomainExtraction(self):
"""Tests for the unique domains plugin."""
event_queue = single_process.SingleProcessQueue()
knowledge_base = self._SetUpKnowledgeBase()
# Fill the incoming queue with events.
test_queue_producer = plaso_queue.ItemQueueProducer(event_queue)
event_objects = [
self._CreateTestEventObject(test_event)
for test_event in self._EVENT_DICTS
]
test_queue_producer.ProduceItems(event_objects)
# Set up the plugin.
analysis_plugin = unique_domains_visited.UniqueDomainsVisitedPlugin(
event_queue)
analysis_report_queue_consumer = self._RunAnalysisPlugin(
analysis_plugin, knowledge_base)
analysis_reports = self._GetAnalysisReportsFromQueue(
analysis_report_queue_consumer)
self.assertEqual(len(analysis_reports), 1)
report_text = analysis_reports[0].GetString()
for event_object in self._EVENT_DICTS:
self.assertIn(event_object.get(u'domain', u''), report_text)
def testVirusTotalLookup(self):
"""Tests for the VirusTotal analysis plugin."""
event_queue = single_process.SingleProcessQueue()
knowledge_base = self._SetUpKnowledgeBase()
# Fill the incoming queue with events.
test_queue_producer = plaso_queue.ItemQueueProducer(event_queue)
events = [
self._CreateTestEventObject(test_event)
for test_event in self.TEST_EVENTS
]
test_queue_producer.ProduceItems(events)
analysis_plugin = virustotal.VirusTotalAnalysisPlugin(event_queue)
analysis_plugin.SetAPIKey(self.FAKE_API_KEY)
# Run the analysis plugin.
analysis_report_queue_consumer = self._RunAnalysisPlugin(
analysis_plugin, knowledge_base)
analysis_reports = self._GetAnalysisReportsFromQueue(
analysis_report_queue_consumer)
self.assertEqual(len(analysis_reports), 1)
report = analysis_reports[0]
tags = report.GetTags()
self.assertEqual(len(tags), 1)
tag = tags[0]
self.assertEqual(tag.event_uuid, u'8')
self.assertEqual(tag.labels[0], u'virustotal_detections_10')
def setUp(self):
"""Makes preparations before running an individual test."""
knowledge_base = self._SetUpKnowledgeBase()
analysis_report_queue = single_process.SingleProcessQueue()
analysis_report_queue_producer = plaso_queue.ItemQueueProducer(
analysis_report_queue)
self._analysis_mediator = mediator.AnalysisMediator(
analysis_report_queue_producer, knowledge_base)
def _StartAnalysisPlugins(
self, storage_file_path, analysis_plugins, pre_obj,
analysis_queue_port=None, analysis_report_incoming_queue=None,
command_line_arguments=None):
"""Start all the analysis plugin.
Args:
storage_file_path: string containing the path of the storage file.
analysis_plugins: list of analysis plugin objects (instance of
AnalysisPlugin) that should be started.
pre_obj: The preprocessor object (instance of PreprocessObject).
analysis_queue_port: optional TCP port that the ZeroMQ analysis report
queues should use.
analysis_report_incoming_queue: optional queue (instance of Queue) that
reports should to pushed to, when ZeroMQ
is not in use.
command_line_arguments: optional string of the command line arguments or
None if not set.
"""
logging.info(u'Starting analysis plugins.')
self._SetAnalysisPluginProcessInformation(
storage_file_path, analysis_plugins, pre_obj,
command_line_arguments=command_line_arguments)
knowledge_base_object = knowledge_base.KnowledgeBase(pre_obj=pre_obj)
for analysis_plugin in analysis_plugins:
if self._use_zeromq:
analysis_plugin_output_queue = zeromq_queue.ZeroMQPushConnectQueue(
delay_open=True, port=analysis_queue_port)
else:
analysis_plugin_output_queue = analysis_report_incoming_queue
analysis_report_queue_producer = plaso_queue.ItemQueueProducer(
analysis_plugin_output_queue)
completion_event = multiprocessing.Event()
analysis_mediator_object = analysis_mediator.AnalysisMediator(
analysis_report_queue_producer, knowledge_base_object,
data_location=self._data_location,
completion_event=completion_event)
analysis_process = multiprocessing.Process(
name=u'Analysis {0:s}'.format(analysis_plugin.plugin_name),
target=analysis_plugin.RunPlugin,
args=(analysis_mediator_object,))
process_info = PsortAnalysisProcess(
completion_event, analysis_plugin, analysis_process)
self._analysis_process_info.append(process_info)
analysis_process.start()
logging.info(
u'Plugin: [{0:s}] started.'.format(analysis_plugin.plugin_name))
logging.info(u'Analysis plugins running')
def CreateParserMediator(self, event_queue=None):
"""Create a parser mediator object.
Args:
event_queue: an optional event queue object (instance of Queue).
Returns:
A parser mediator object (instance of parsers_mediator.ParserMediator).
"""
if event_queue is None:
event_queue = single_process.SingleProcessQueue()
event_queue_producer = plaso_queue.ItemQueueProducer(event_queue)
parse_error_queue = single_process.SingleProcessQueue()
parse_error_queue_producer = plaso_queue.ItemQueueProducer(
parse_error_queue)
return parsers_mediator.ParserMediator(event_queue_producer,
parse_error_queue_producer,
self.knowledge_base_object)
def _GetParserMediator(self,
event_queue,
parse_error_queue,
knowledge_base_values=None,
file_entry=None,
parser_chain=None):
"""Retrieves a parser mediator object.
Args:
event_queue: the event queue (instance of Queue).
parse_error_queue: the parse error queue (instance of Queue).
knowledge_base_values: optional dict containing the knowledge base
values.
file_entry: optional dfVFS file_entry object (instance of dfvfs.FileEntry)
being parsed.
parser_chain: Optional string containing the parsing chain up to this
point.
Returns:
A parser mediator object (instance of ParserMediator).
"""
event_queue_producer = plaso_queue.ItemQueueProducer(event_queue)
parse_error_queue_producer = plaso_queue.ItemQueueProducer(
parse_error_queue)
knowledge_base_object = knowledge_base.KnowledgeBase()
if knowledge_base_values:
for identifier, value in iter(knowledge_base_values.items()):
knowledge_base_object.SetValue(identifier, value)
new_mediator = mediator.ParserMediator(event_queue_producer,
parse_error_queue_producer,
knowledge_base_object)
if file_entry:
new_mediator.SetFileEntry(file_entry)
if parser_chain:
new_mediator.parser_chain = parser_chain
return new_mediator
def GetAnalysisPluginsAndEventQueues(self, analysis_plugins_string):
"""Return a list of analysis plugins and event queues.
Args:
analysis_plugins_string: comma separated string with names of analysis
plugins to load.
Returns:
A tuple of two lists, one containing list of analysis plugins
and the other a list of event queues.
"""
if not analysis_plugins_string:
return [], []
event_producers = []
# These are the queues analysis plugins will read from.
analysis_plugin_input_queues = []
analysis_plugins_list = [
name.strip() for name in analysis_plugins_string.split(u',')
]
for _ in range(0, len(analysis_plugins_list)):
if self._use_zeromq:
output_queue = zeromq_queue.ZeroMQPushBindQueue()
# Open the queue so it can bind to a random port, and we can get the
# port number to use in the input queue.
output_queue.Open()
queue_port = output_queue.port
input_queue = zeromq_queue.ZeroMQPullConnectQueue(
port=queue_port, delay_open=True)
analysis_plugin_input_queues.append(input_queue)
else:
input_queue = multi_process.MultiProcessingQueue(timeout=5)
analysis_plugin_input_queues.append(input_queue)
output_queue = input_queue
event_producers.append(plaso_queue.ItemQueueProducer(output_queue))
analysis_plugins = analysis_manager.AnalysisPluginManager.LoadPlugins(
analysis_plugins_list, analysis_plugin_input_queues)
analysis_plugins = list(analysis_plugins)
return analysis_plugins, event_producers
def testWinAnalyzePlugin(self):
"""Test the plugin against mock events."""
knowledge_base = self._SetUpKnowledgeBase(
knowledge_base_values={'users': self.WIN_USERS})
event_queue = single_process.SingleProcessQueue()
# Fill the incoming queue with events.
test_queue_producer = plaso_queue.ItemQueueProducer(event_queue)
test_queue_producer.ProduceItems(
[self._CreateTestEventObject(path) for path in self.WIN_PATHS])
# Initialize plugin.
analysis_plugin = TestChromeExtensionPlugin(event_queue)
# Run the analysis plugin.
analysis_report_queue_consumer = self._RunAnalysisPlugin(
analysis_plugin, knowledge_base)
analysis_reports = self._GetAnalysisReportsFromQueue(
analysis_report_queue_consumer)
self.assertEqual(len(analysis_reports), 1)
analysis_report = analysis_reports[0]
self.assertEqual(analysis_plugin._sep, u'\\')
# Due to the behavior of the join one additional empty string at the end
# is needed to create the last empty line.
expected_text = u'\n'.join([
u' == USER: dude ==',
u' Google Keep - notes and lists [hmjkmjkepdijhoojdojkdfohbdgmmhki]',
u'', u' == USER: frank ==',
u' Google Play Music [icppfcnhkcmnfdhfhphakoifcfokfdhg]',
u' YouTube [blpcfgokakmgnkcojhhkbfbldkacnbeo]', u'', u''
])
self.assertEqual(analysis_report.text, expected_text)
self.assertEqual(analysis_report.plugin_name, 'chrome_extension_test')
expected_keys = set([u'frank', u'dude'])
self.assertEqual(set(analysis_report.report_dict.keys()),
expected_keys)
def testMacAnalyzerPlugin(self):
"""Test the plugin against mock events."""
knowledge_base = self._SetUpKnowledgeBase(
knowledge_base_values={'users': self.MAC_USERS})
event_queue = single_process.SingleProcessQueue()
# Fill the incoming queue with events.
test_queue_producer = plaso_queue.ItemQueueProducer(event_queue)
test_queue_producer.ProduceItems(
[self._CreateTestEventObject(path) for path in self.MAC_PATHS])
# Initialize plugin.
analysis_plugin = TestChromeExtensionPlugin(event_queue)
# Run the analysis plugin.
analysis_report_queue_consumer = self._RunAnalysisPlugin(
analysis_plugin, knowledge_base)
analysis_reports = self._GetAnalysisReportsFromQueue(
analysis_report_queue_consumer)
self.assertEqual(len(analysis_reports), 1)
analysis_report = analysis_reports[0]
self.assertEqual(analysis_plugin._sep, u'/')
# Due to the behavior of the join one additional empty string at the end
# is needed to create the last empty line.
expected_text = u'\n'.join([
u' == USER: dude ==',
u' Google Drive [apdfllckaahabafndbhieahigkjlhalf]', u'',
u' == USER: frank ==',
u' Gmail [pjkljhegncpnkpknbcohdijeoejaedia]', u'', u''
])
self.assertEqual(analysis_report.text, expected_text)
self.assertEqual(analysis_report.plugin_name, 'chrome_extension_test')
expected_keys = set([u'frank', u'dude'])
self.assertEqual(set(analysis_report.report_dict.keys()),
expected_keys)
def testSyntheticKeysText(self):
"""Test the plugin against mock events."""
event_queue = single_process.SingleProcessQueue()
# Fill the incoming queue with events.
test_queue_producer = plaso_queue.ItemQueueProducer(event_queue)
events = [
self._CreateTestEventObject(service_event)
for service_event in self.SERVICE_EVENTS
]
test_queue_producer.ProduceItems(events)
# Initialize plugin.
analysis_plugin = windows_services.WindowsServicesPlugin(event_queue)
# Run the analysis plugin.
knowledge_base = self._SetUpKnowledgeBase()
analysis_report_queue_consumer = self._RunAnalysisPlugin(
analysis_plugin, knowledge_base)
analysis_reports = self._GetAnalysisReportsFromQueue(
analysis_report_queue_consumer)
self.assertEqual(len(analysis_reports), 1)
analysis_report = analysis_reports[0]
expected_text = (u'Listing Windows Services\n'
u'TestbDriver\n'
u'\tImage Path = C:\\Dell\\testdriver.sys\n'
u'\tService Type = File System Driver (0x2)\n'
u'\tStart Type = Auto Start (2)\n'
u'\tService Dll = \n'
u'\tObject Name = \n'
u'\tSources:\n'
u'\t\tC:\\WINDOWS\\system32\\SYSTEM:'
u'\\ControlSet001\\services\\TestbDriver\n'
u'\t\tC:\\WINDOWS\\system32\\SYSTEM:'
u'\\ControlSet003\\services\\TestbDriver\n\n')
self.assertEqual(expected_text, analysis_report.text)
self.assertEqual(analysis_report.plugin_name, 'windows_services')
def testStorageWriter(self):
"""Test the storage writer."""
event_objects = test_lib.CreateTestEventObjects()
# The storage writer is normally run in a separate thread.
# For the purpose of this test it has to be run in sequence,
# hence the call to WriteEventObjects after all the event objects
# have been queued up.
# TODO: add upper queue limit.
# A timeout is used to prevent the multi processing queue to close and
# stop blocking the current process.
test_queue = multi_process.MultiProcessingQueue(timeout=0.1)
test_queue_producer = plaso_queue.ItemQueueProducer(test_queue)
test_queue_producer.ProduceItems(event_objects)
test_queue_producer.SignalAbort()
preprocessing_object = event.PreprocessObject()
with shared_test_lib.TempDirectory() as temp_directory:
temp_file = os.path.join(temp_directory, u'plaso.db')
storage_writer = zip_file.ZIPStorageFileWriter(
test_queue, temp_file, preprocessing_object)
storage_writer.WriteEventObjects()
storage_file = zipfile.ZipFile(temp_file,
mode='r',
compression=zipfile.ZIP_DEFLATED)
expected_filename_list = [
u'information.dump', u'plaso_index.000001',
u'plaso_proto.000001', u'plaso_timestamps.000001',
u'serializer.txt'
]
filename_list = sorted(storage_file.namelist())
self.assertEqual(len(filename_list), 5)
self.assertEqual(filename_list, expected_filename_list)
def _RunAnalysisPlugin(self, analysis_plugin, knowledge_base_object):
"""Analyzes an event object queue using the plugin object.
Args:
analysis_plugin: the analysis plugin object (instance of AnalysisPlugin).
knowledge_base_object: the knowledge base object (instance of
KnowledgeBase).
Returns:
An event object queue object (instance of Queue).
"""
analysis_report_queue = single_process.SingleProcessQueue()
analysis_report_queue_consumer = TestAnalysisReportQueueConsumer(
analysis_report_queue)
analysis_report_queue_producer = plaso_queue.ItemQueueProducer(
analysis_report_queue)
analysis_mediator = mediator.AnalysisMediator(
analysis_report_queue_producer, knowledge_base_object)
analysis_plugin.RunPlugin(analysis_mediator)
return analysis_report_queue_consumer
def testTag(self):
"""Test that the tagging plugin successfully tags events."""
event_queue = single_process.SingleProcessQueue()
test_queue_producer = plaso_queue.ItemQueueProducer(event_queue)
event_objects = [
self._CreateTestEventObject(test_event)
for test_event in self._EVENT_DICTS
]
test_queue_producer.ProduceItems(event_objects)
analysis_plugin = tagging.TaggingPlugin(event_queue)
test_file = self._GetTestFilePath([self._TEST_TAG_FILE_NAME])
analysis_plugin.SetAndLoadTagFile(test_file)
# Run the plugin.
knowledge_base = self._SetUpKnowledgeBase()
analysis_report_queue_consumer = self._RunAnalysisPlugin(
analysis_plugin, knowledge_base)
analysis_reports = self._GetAnalysisReportsFromQueue(
analysis_report_queue_consumer)
self.assertEqual(len(analysis_reports), 1)
report = analysis_reports[0]
self.assertEqual(len(report.GetTags()), 2)
def testEvents(self):
"""Test the plugin against mock events."""
event_queue = single_process.SingleProcessQueue()
# Fill the incoming queue with events.
test_queue_producer = plaso_queue.ItemQueueProducer(event_queue)
event_objects = [
self._CreateTestEventObject(event_dict)
for event_dict in self._EVENT_DICTS
]
test_queue_producer.ProduceItems(event_objects)
# Initialize plugin.
analysis_plugin = file_hashes.FileHashesPlugin(event_queue)
# Run the analysis plugin.
knowledge_base = self._SetUpKnowledgeBase()
analysis_report_queue_consumer = self._RunAnalysisPlugin(
analysis_plugin, knowledge_base)
analysis_reports = self._GetAnalysisReportsFromQueue(
analysis_report_queue_consumer)
self.assertEqual(len(analysis_reports), 1)
analysis_report = analysis_reports[0]
expected_text = (
u'Listing file paths and hashes\n'
u'FAKE:/opt/2hash_file: alternate_test_hash=5 test_hash=4\n'
u'FAKE:/opt/dfvfs: test_hash=4\n'
u'FAKE:/opt/no_hash_file:\n'
u'FAKE:/var/testing directory with space/file.txt: test_hash=4\n'
u'FAKE:C:\\Windows\\a.file.txt: test_hash=4\n')
self.assertEqual(expected_text, analysis_report.text)
self.assertEqual(analysis_report.plugin_name, u'file_hashes')
