def test_mhtml_analysis(self): root = create_root_analysis(analysis_mode='test_groups') root.initialize_storage() shutil.copy(os.path.join('test_data', 'mhtml', 'Invoice_PDF.mht'), root.storage_dir) file_observable = root.add_observable(F_FILE, 'Invoice_PDF.mht') root.save() root.schedule() engine = TestEngine(pool_size_limit=1) engine.enable_alerting() engine.enable_module('analysis_module_mhtml', 'test_groups') engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=root.storage_dir) root.load() file_observable = root.get_observable(file_observable.id) self.assertIsNotNone(file_observable) from saq.modules.file_analysis import MHTMLAnalysis analysis = file_observable.get_analysis(MHTMLAnalysis) self.assertIsNotNone(analysis) # should have extracted a single file self.assertEquals(len(analysis.details), 1) self.assertEquals(len(analysis.get_observables_by_type(F_FILE)), 1)
def get_file(uuid, file_uuid_or_name): storage_dir = storage_dir_from_uuid(uuid) if saq.CONFIG['service_engine']['work_dir'] and not os.path.isdir(storage_dir): storage_dir = workload_storage_dir(uuid) root = RootAnalysis(storage_dir=storage_dir) root.load() # is this a UUID? try: validate_uuid(file_uuid_or_name) file_observable = root.get_observable(file_uuid_or_name) if file_observable is None: abort(Response("invalid file_uuid {}".format(file_uuid_or_name), 400)) except ValueError: file_observable = root.find_observable(lambda o: o.type == F_FILE and o.value == file_uuid_or_name) if file_observable is None: abort(Response("invalid file name {}".format(file_uuid_or_name), 400)) # NOTE we use an absolute path here because if we don't then # send_from_directory makes it relavive from the app root path # which is (/opt/ace/aceapi) target_path = os.path.join(saq.SAQ_HOME, root.storage_dir, file_observable.value) if not os.path.exists(target_path): abort(Response("file path {} does not exist".format(target_path), 400)) # XXX revisit how we save (name) files return send_from_directory(os.path.dirname(target_path), os.path.basename(target_path), as_attachment=True, attachment_filename=os.path.basename(target_path).encode().decode('latin-1', errors='ignore'))
def test_intel_analysis(self): if not saq.CONFIG['sip'].getboolean('enabled'): return root = create_root_analysis(analysis_mode=ANALYSIS_MODE_CORRELATION) root.initialize_storage() i = root.add_observable(F_INDICATOR, 'sip:{}'.format(self.test_indicator_id)) self.assertIsNotNone(i) root.save() root.schedule() engine = TestEngine(local_analysis_modes=[ANALYSIS_MODE_CORRELATION]) engine.enable_module('analysis_module_intel_analyzer', ANALYSIS_MODE_CORRELATION) engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=root.storage_dir) root.load() i = root.get_observable(i.id) self.assertIsNotNone(i) from saq.modules.intel import IntelAnalysis analysis = i.get_analysis(IntelAnalysis) self.assertIsNotNone(analysis) # what we get here should be the same as what we got when we inserted it self.assertEquals(analysis.details, self.test_indicator)
def test_submit_alert(self): # disable cleaup for analysis mode analysis saq.CONFIG['analysis_mode_analysis']['cleanup'] = 'no' self.start_api_server() root = create_root_analysis(analysis_mode=ANALYSIS_MODE_ANALYSIS) root.initialize_storage() url = root.add_observable(F_URL, TEST_URL) url.add_directive(DIRECTIVE_CRAWL) root.save() root.schedule() engine = TestEngine(analysis_pools={ ANALYSIS_MODE_ANALYSIS: 1, ANALYSIS_MODE_CLOUDPHISH: 1 }, local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CLOUDPHISH ]) engine.enable_module('analysis_module_cloudphish', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_cloudphish_request_analyzer', ANALYSIS_MODE_CLOUDPHISH) engine.enable_module('analysis_module_crawlphish', ANALYSIS_MODE_CLOUDPHISH) engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_CLOUDPHISH) engine.enable_module('analysis_module_detection', ANALYSIS_MODE_CLOUDPHISH) engine.start() # should see cloudphish module complete wait_for_log_count('analysis CloudphishAnalysis is completed', 1, 10) engine.controlled_stop() engine.wait() # check the results root = RootAnalysis(storage_dir=storage_dir_from_uuid(root.uuid)) root.load() url = root.get_observable(url.id) self.assertIsNotNone(url) # this url should now have 3 analysis objects attached to it (cloudphish, crawlphish and forced detection) self.assertEquals(len(url.analysis), 3) from saq.modules.cloudphish import CloudphishAnalysis cloudphish_analysis = url.get_analysis(CloudphishAnalysis) self.assertIsNotNone(cloudphish_analysis) self.assertEquals(cloudphish_analysis.analysis_result, SCAN_RESULT_ALERT) from saq.modules.url import CrawlphishAnalysisV2 crawlphish_analysis = url.get_analysis(CrawlphishAnalysisV2) self.assertIsNotNone(crawlphish_analysis)
def test_submit_timeout_with_alert(self, db, c): # any cloudphish submission we make can turn into an alert # here we test a cloudphish submission that quickly times out # followed by cloudphish alerting on the submission # set the timeouts really low saq.CONFIG['analysis_module_cloudphish']['frequency'] = '1' saq.CONFIG['analysis_module_cloudphish']['query_timeout'] = '1' # disable cleaup for analysis mode analysis saq.CONFIG['analysis_mode_analysis']['cleanup'] = 'no' self.start_api_server() root = create_root_analysis(analysis_mode=ANALYSIS_MODE_ANALYSIS) root.initialize_storage() url = root.add_observable(F_URL, TEST_URL) url.add_directive(DIRECTIVE_CRAWL) root.save() root.schedule() engine = TestEngine(analysis_pools={}, local_analysis_modes=[ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CLOUDPHISH, ANALYSIS_MODE_CORRELATION]) engine.enable_alerting() engine.enable_module('analysis_module_cloudphish', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_cloudphish_request_analyzer', ANALYSIS_MODE_CLOUDPHISH) engine.enable_module('analysis_module_crawlphish', ANALYSIS_MODE_CLOUDPHISH) engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_CLOUDPHISH) engine.enable_module('analysis_module_cloudphish_delayed_test', ANALYSIS_MODE_CLOUDPHISH) engine.start() # watch for the original analysis to time out wait_for_log_count('has timed out', 1, 10) # we should see cloudphish eventually complete and alert though engine.controlled_stop() engine.wait() # check the results root = RootAnalysis(storage_dir=root.storage_dir) root.load() url = root.get_observable(url.id) self.assertIsNotNone(url) # should see an error here from saq.modules.cloudphish import CloudphishAnalysis cloudphish_analysis = url.get_analysis(CloudphishAnalysis) self.assertIsNotNone(cloudphish_analysis) self.assertEquals(cloudphish_analysis.result, SCAN_RESULT_ERROR) # however we should have an alert generated c.execute("SELECT COUNT(*) FROM alerts") self.assertEquals(c.fetchone()[0], 1)
def test_hal9000_alert_no_disposition(self, db, c): # same as above except we end up alerting root = create_root_analysis(analysis_mode=ANALYSIS_MODE_ANALYSIS) root.initialize_storage() test_observable = root.add_observable(F_TEST, 'test') root.save() root.schedule() engine = TestEngine(local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION ]) engine.enable_alerting() engine.set_cleanup(ANALYSIS_MODE_ANALYSIS, False) engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_hal9000', [ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION ]) engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=storage_dir_from_uuid(root.uuid)) root.load() # make sure we alerted self.assertEquals(root.analysis_mode, ANALYSIS_MODE_CORRELATION) test_observable = root.get_observable(test_observable.id) self.assertIsNotNone(test_observable) analysis = test_observable.get_analysis(HAL9000Analysis) self.assertIsNotNone(analysis) # total count and mal count should both be 0 self.assertEquals(analysis.total_count, 0) self.assertEquals(analysis.mal_count, 0) # we should have a single entry in the database for this observable hal9000_id = _compute_hal9000_md5(test_observable) # since we have NOT set a disposition yet we should have nothing in the database about it c.execute("SELECT total_count, mal_count FROM observables WHERE id = UNHEX(%s)", (hal9000_id,)) result = c.fetchone() self.assertIsNone(result) # verify the correct state is kept state = root.state['hal9000'] self.assertTrue(STATE_KEY_ID_TRACKING in state) tracking = state[STATE_KEY_ID_TRACKING] self.assertTrue(hal9000_id in tracking) tracking_info = tracking[hal9000_id] self.assertTrue('id' in tracking_info) self.assertEquals(tracking_info['id'], test_observable.id) self.assertTrue(KEY_TOTAL_COUNT in tracking_info) self.assertTrue(KEY_MAL_COUNT in tracking_info) # we have not made any changes yet either self.assertIsNone(tracking_info[KEY_TOTAL_COUNT]) self.assertIsNone(tracking_info[KEY_MAL_COUNT])
def test_submit_double_alert(self, db, c): # in this scenario we alert both with the original submission # and with the cloudphish submission self.start_api_server() root = create_root_analysis(analysis_mode=ANALYSIS_MODE_ANALYSIS) root.initialize_storage() url = root.add_observable(F_URL, TEST_URL) url.add_directive(DIRECTIVE_CRAWL) root.save() root.schedule() engine = TestEngine(local_analysis_modes=[ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CLOUDPHISH, ANALYSIS_MODE_CORRELATION]) engine.enable_alerting() engine.enable_module('analysis_module_cloudphish', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_cloudphish_request_analyzer', ANALYSIS_MODE_CLOUDPHISH) engine.enable_module('analysis_module_crawlphish', ANALYSIS_MODE_CLOUDPHISH) engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_CLOUDPHISH) engine.start() # should see cloudphish module complete wait_for_log_count('analysis CloudphishAnalysis is completed', 1, 10) engine.controlled_stop() engine.wait() # check the results root = RootAnalysis(storage_dir=storage_dir_from_uuid(root.uuid)) root.load() url = root.get_observable(url.id) self.assertIsNotNone(url) # this url should now have 3 analysis objects attached to it (cloudphish, crawlphish and forced detection) self.assertEquals(len(url.analysis), 3) from saq.modules.cloudphish import CloudphishAnalysis cloudphish_analysis = url.get_analysis(CloudphishAnalysis) self.assertIsNotNone(cloudphish_analysis) self.assertEquals(cloudphish_analysis.analysis_result, SCAN_RESULT_ALERT) from saq.modules.url import CrawlphishAnalysisV2 crawlphish_analysis = url.get_analysis(CrawlphishAnalysisV2) self.assertIsNotNone(crawlphish_analysis) # there should be two alerts generated in the database c.execute("SELECT COUNT(*) FROM alerts") self.assertEquals(c.fetchone()[0], 2) # the cloudphish alert should have a reference back to the original alert self.assertEquals(cloudphish_analysis.context['c'], root.uuid)
def test_hal9000_no_alert(self, db, c): root = create_root_analysis(analysis_mode=ANALYSIS_MODE_ANALYSIS) root.initialize_storage() test_observable = root.add_observable(F_TEST, 'test') root.save() root.schedule() engine = TestEngine(local_analysis_modes=[ANALYSIS_MODE_ANALYSIS]) engine.set_cleanup(ANALYSIS_MODE_ANALYSIS, False) engine.enable_module('analysis_module_hal9000', ANALYSIS_MODE_ANALYSIS) engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=root.storage_dir) root.load() # make sure we did NOT alert self.assertEquals(root.analysis_mode, ANALYSIS_MODE_ANALYSIS) test_observable = root.get_observable(test_observable.id) self.assertIsNotNone(test_observable) analysis = test_observable.get_analysis(HAL9000Analysis) self.assertIsNotNone(analysis) # total count and mal count should both be 0 self.assertEquals(analysis.total_count, 0) self.assertEquals(analysis.mal_count, 0) # we should have a single entry in the database for this observable hal9000_id = _compute_hal9000_md5(test_observable) c.execute( "SELECT total_count, mal_count FROM observables WHERE id = UNHEX(%s)", (hal9000_id, )) result = c.fetchone() self.assertIsNotNone(result) self.assertEquals(result[0], 1) self.assertEquals(result[1], 0) # verify the correct state is kept state = root.state['hal9000'] self.assertTrue(STATE_KEY_ID_TRACKING in state) tracking = state[STATE_KEY_ID_TRACKING] self.assertTrue(hal9000_id in tracking) tracking_info = tracking[hal9000_id] self.assertTrue('id' in tracking_info) self.assertEquals(tracking_info['id'], test_observable.id) self.assertTrue(KEY_TOTAL_COUNT in tracking_info) self.assertTrue(KEY_MAL_COUNT in tracking_info) # since this doesn't become an alert we don't bother tracking the changes self.assertIsNone(tracking_info[KEY_TOTAL_COUNT]) self.assertIsNone(tracking_info[KEY_MAL_COUNT])
def test_download(self): # first create something to download root = create_root_analysis(uuid=str(uuid.uuid4())) root.initialize_storage() root.details = {'hello': 'world'} with open(os.path.join(root.storage_dir, 'test.dat'), 'w') as fp: fp.write('test') file_observable = root.add_observable(F_FILE, 'test.dat') root.save() # ask for a download result = self.client.get(url_for('engine.download', uuid=root.uuid)) # we should get back a tar file tar_path = os.path.join(saq.TEMP_DIR, 'download.tar') output_dir = os.path.join(saq.TEMP_DIR, 'download') try: with open(tar_path, 'wb') as fp: for chunk in result.response: fp.write(chunk) with tarfile.open(name=tar_path, mode='r|') as tar: tar.extractall(path=output_dir) root = RootAnalysis(storage_dir=output_dir) root.load() self.assertTrue('hello' in root.details) self.assertEquals('world', root.details['hello']) file_observable = root.get_observable(file_observable.id) self.assertTrue( os.path.exists( os.path.join(root.storage_dir, file_observable.value))) with open(os.path.join(root.storage_dir, file_observable.value), 'r') as fp: self.assertEquals(fp.read(), 'test') finally: try: os.remove(tar_path) except: pass try: shutil.rmtree(output_dir) except: pass
def test_submit_forced_download(self): # disable cleaup for analysis mode analysis saq.CONFIG['analysis_mode_analysis']['cleanup'] = 'no' self.start_api_server() root = create_root_analysis(analysis_mode=ANALYSIS_MODE_ANALYSIS) root.initialize_storage() url = root.add_observable(F_URL, TEST_URL) url.add_directive(DIRECTIVE_CRAWL) url.add_directive(DIRECTIVE_FORCE_DOWNLOAD) root.save() root.schedule() engine = TestEngine(analysis_pools={ANALYSIS_MODE_ANALYSIS: 1, ANALYSIS_MODE_CLOUDPHISH: 1}, local_analysis_modes=[ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CLOUDPHISH]) engine.enable_module('analysis_module_cloudphish', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_cloudphish_request_analyzer', ANALYSIS_MODE_CLOUDPHISH) engine.enable_module('analysis_module_crawlphish', ANALYSIS_MODE_CLOUDPHISH) engine.start() # should see cloudphish module complete wait_for_log_count('analysis CloudphishAnalysis is completed', 1, 10) engine.controlled_stop() engine.wait() # check the results root = RootAnalysis(storage_dir=root.storage_dir) root.load() url = root.get_observable(url.id) self.assertIsNotNone(url) # should only have 1 analysis attached to the url self.assertEquals(len(url.analysis), 1) from saq.modules.cloudphish import CloudphishAnalysis cloudphish_analysis = url.get_analysis(CloudphishAnalysis) self.assertIsNotNone(cloudphish_analysis) self.assertEquals(cloudphish_analysis.analysis_result, SCAN_RESULT_CLEAR) # however there should be a file attached self.assertEquals(len(cloudphish_analysis.observables), 1) self.assertEquals(cloudphish_analysis.observables[0].type, F_FILE) self.assertEquals(cloudphish_analysis.observables[0].value, 'Payment_Advice.pdf') self.assertTrue(os.path.exists(os.path.join(root.storage_dir, cloudphish_analysis.observables[0].value)))
def test_request_limit(self): # only allow one request saq.CONFIG['analysis_module_cloudphish'][ 'cloudphish_request_limit'] = '1' # don't clear the analysis saq.CONFIG['analysis_mode_analysis']['cleanup'] = 'no' self.start_api_server() root = create_root_analysis(analysis_mode=ANALYSIS_MODE_ANALYSIS) root.initialize_storage() url_1 = root.add_observable(F_URL, TEST_URL) url_2 = root.add_observable(F_URL, 'http://invalid_domain.local/some/path') root.save() root.schedule() engine = TestEngine(analysis_pools={}, local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CLOUDPHISH ]) engine.enable_module('analysis_module_cloudphish', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_cloudphish_request_analyzer', ANALYSIS_MODE_CLOUDPHISH) engine.enable_module('analysis_module_crawlphish', ANALYSIS_MODE_CLOUDPHISH) engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=root.storage_dir) root.load() url_1 = root.get_observable(url_1.id) url_2 = root.get_observable(url_2.id) from saq.modules.cloudphish import CloudphishAnalysis analysis_1 = url_1.get_analysis(CloudphishAnalysis) analysis_2 = url_2.get_analysis(CloudphishAnalysis) self.assertTrue( (isinstance(analysis_1, Analysis) and analysis_2 is False) or (analysis_1 is False and isinstance(analysis_2, Analysis))) self.assertEquals(log_count('reached cloudphish limit'), 1)
def test_bro_smtp_stream_analysis_no_end_command(self): import saq import saq.modules.email # test the same thing as test_bro_smtp_stream_analysis except we remove the > . . saq.CONFIG['analysis_mode_email']['cleanup'] = 'no' root = create_root_analysis(alert_type=ANALYSIS_TYPE_BRO_SMTP, analysis_mode=ANALYSIS_MODE_EMAIL) root.initialize_storage() root.details = { } shutil.copy(os.path.join('test_data', 'smtp_streams', 'CBmtfvapmTMqCEUw6.missing_end'), os.path.join(root.storage_dir, 'CBmtfvapmTMqCEUw6')) file_observable = root.add_observable(F_FILE, 'CBmtfvapmTMqCEUw6') file_observable.add_directive(DIRECTIVE_ORIGINAL_SMTP) file_observable.add_directive(DIRECTIVE_NO_SCAN) root.save() root.schedule() engine = TestEngine(local_analysis_modes=[ANALYSIS_MODE_EMAIL]) engine.enable_module('analysis_module_file_type', 'test_groups') engine.enable_module('analysis_module_email_analyzer', 'test_groups') engine.enable_module('analysis_module_bro_smtp_analyzer', 'test_groups') engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=root.storage_dir) root.load() file_observable = root.get_observable(file_observable.id) self.assertIsNotNone(file_observable) analysis = file_observable.get_analysis(saq.modules.email.BroSMTPStreamAnalysis) self.assertIsNotNone(analysis) self.assertEquals(len(analysis.get_observables_by_type(F_FILE)), 1) self.assertEquals(len(analysis.get_observables_by_type(F_EMAIL_ADDRESS)), 2) self.assertEquals(len(analysis.get_observables_by_type(F_IPV4)), 1) self.assertEquals(len(analysis.get_observables_by_type(F_EMAIL_CONVERSATION)), 1) self.assertTrue(saq.modules.email.KEY_CONNECTION_ID in analysis.details) self.assertTrue(saq.modules.email.KEY_SOURCE_IPV4 in analysis.details) self.assertTrue(saq.modules.email.KEY_SOURCE_PORT in analysis.details) self.assertTrue(saq.modules.email.KEY_ENV_MAIL_FROM in analysis.details) self.assertTrue(saq.modules.email.KEY_ENV_RCPT_TO in analysis.details) email_file = analysis.find_observable(lambda o: o.type == F_FILE) self.assertIsNotNone(email_file) self.assertEquals(email_file.value, 'email.rfc822') email_analysis = email_file.get_analysis(saq.modules.email.EmailAnalysis) self.assertIsNotNone(email_analysis)
def test_open_office_extraction(self): root = create_root_analysis() root.initialize_storage() shutil.copy('test_data/openoffice/demo.odt', root.storage_dir) _file = root.add_observable(F_FILE, 'demo.odt') root.save() root.schedule() engine = TestEngine() engine.enable_module('analysis_module_archive', 'test_groups') engine.enable_module('analysis_module_file_type', 'test_groups') engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=root.storage_dir) root.load() _file = root.get_observable(_file.id) self.assertIsNotNone(_file) analysis = _file.get_analysis('ArchiveAnalysis') self.assertIsNotNone(analysis) self.assertEquals(len(analysis.find_observables(F_FILE)), 12)
def test_hal9000_alert_mal_disposition(self, db, c): # same as above except we end up alerting and disposition as malicious root = create_root_analysis(analysis_mode=ANALYSIS_MODE_ANALYSIS) root.initialize_storage() test_observable = root.add_observable(F_TEST, 'test') root.save() root.schedule() engine = TestEngine(local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION, ANALYSIS_MODE_DISPOSITIONED ]) engine.set_cleanup(ANALYSIS_MODE_ANALYSIS, False) engine.enable_alerting() engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_hal9000', [ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION, ANALYSIS_MODE_DISPOSITIONED ]) engine.controlled_stop() engine.start() engine.wait() # set the disposition for the alert set_dispositions([root.uuid], DISPOSITION_DELIVERY, UNITTEST_USER_ID) # run the engine again so that is processes the alert in correlation mode with the disposition set engine = TestEngine(local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION, ANALYSIS_MODE_DISPOSITIONED ]) engine.enable_alerting() engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_hal9000', [ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION, ANALYSIS_MODE_DISPOSITIONED ]) engine.controlled_stop() engine.start() engine.wait() # storage dir changes when it turns into alert root = RootAnalysis(storage_dir=storage_dir_from_uuid(root.uuid)) root.load() # make sure we alerted self.assertEquals(root.analysis_mode, ANALYSIS_MODE_DISPOSITIONED) test_observable = root.get_observable(test_observable.id) self.assertIsNotNone(test_observable) analysis = test_observable.get_analysis(HAL9000Analysis) self.assertIsNotNone(analysis) # these should still both be 0 self.assertEquals(analysis.total_count, 0) self.assertEquals(analysis.mal_count, 0) # we should have a single entry in the database for this observable hal9000_id = _compute_hal9000_md5(test_observable) # with the disposition set we should have the corresponding values c.execute("SELECT total_count, mal_count FROM observables WHERE id = UNHEX(%s)", (hal9000_id,)) result = c.fetchone() db.commit() self.assertIsNotNone(result) self.assertEquals(result[0], 1) self.assertEquals(result[1], 1) # verify the correct state is kept state = root.state['hal9000'] self.assertTrue(STATE_KEY_ID_TRACKING in state) tracking = state[STATE_KEY_ID_TRACKING] self.assertTrue(hal9000_id in tracking) tracking_info = tracking[hal9000_id] self.assertTrue('id' in tracking_info) self.assertEquals(tracking_info['id'], test_observable.id) self.assertTrue(KEY_TOTAL_COUNT in tracking_info) self.assertTrue(KEY_MAL_COUNT in tracking_info) # we should be tracking the change we made in here self.assertIsNotNone(tracking_info[KEY_TOTAL_COUNT]) self.assertIsNotNone(tracking_info[KEY_MAL_COUNT]) # now we change it to FP set_dispositions([root.uuid], DISPOSITION_FALSE_POSITIVE, UNITTEST_USER_ID) # run the engine again so that is processes the alert in the new correlation mode with the disposition changed engine = TestEngine(local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION, ANALYSIS_MODE_DISPOSITIONED ]) engine.enable_alerting() engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_hal9000', [ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION, ANALYSIS_MODE_DISPOSITIONED ]) engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=storage_dir_from_uuid(root.uuid)) root.load() test_observable = root.get_observable(test_observable.id) self.assertIsNotNone(test_observable) analysis = test_observable.get_analysis(HAL9000Analysis) self.assertIsNotNone(analysis) # these should still both be 0 self.assertEquals(analysis.total_count, 0) self.assertEquals(analysis.mal_count, 0) # we should have a single entry in the database for this observable hal9000_id = _compute_hal9000_md5(test_observable) # with the disposition set we should have the corresponding values c.execute("SELECT total_count, mal_count FROM observables WHERE id = UNHEX(%s)", (hal9000_id,)) result = c.fetchone() db.commit() self.assertIsNotNone(result) self.assertEquals(result[0], 1) self.assertEquals(result[1], 0) # <-- should be 0 now that it's set to FP # verify the correct state is kept state = root.state['hal9000'] self.assertTrue(STATE_KEY_ID_TRACKING in state) tracking = state[STATE_KEY_ID_TRACKING] self.assertTrue(hal9000_id in tracking) tracking_info = tracking[hal9000_id] self.assertTrue('id' in tracking_info) self.assertEquals(tracking_info['id'], test_observable.id) self.assertTrue(KEY_TOTAL_COUNT in tracking_info) self.assertTrue(KEY_MAL_COUNT in tracking_info) # we should be tracking the change we made in here self.assertIsNotNone(tracking_info[KEY_TOTAL_COUNT]) self.assertIsNone(tracking_info[KEY_MAL_COUNT]) # finally we change it to ignore, which should entirely remove the counters (set them to 0 anyways) set_dispositions([root.uuid], DISPOSITION_IGNORE, UNITTEST_USER_ID) # run the engine again so that is processes the alert in the new correlation mode with the disposition changed engine = TestEngine(local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION, ANALYSIS_MODE_DISPOSITIONED ]) engine.enable_alerting() engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_hal9000', [ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION, ANALYSIS_MODE_DISPOSITIONED ]) engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=storage_dir_from_uuid(root.uuid)) root.load() test_observable = root.get_observable(test_observable.id) self.assertIsNotNone(test_observable) analysis = test_observable.get_analysis(HAL9000Analysis) self.assertIsNotNone(analysis) # these should still both be 0 self.assertEquals(analysis.total_count, 0) self.assertEquals(analysis.mal_count, 0) # we should have a single entry in the database for this observable hal9000_id = _compute_hal9000_md5(test_observable) # with the disposition set we should have the corresponding values c.execute("SELECT total_count, mal_count FROM observables WHERE id = UNHEX(%s)", (hal9000_id,)) result = c.fetchone() db.commit() self.assertIsNotNone(result) self.assertEquals(result[0], 0) # <-- now both should be set to 0 self.assertEquals(result[1], 0) # verify the correct state is kept state = root.state['hal9000'] self.assertTrue(STATE_KEY_ID_TRACKING in state) tracking = state[STATE_KEY_ID_TRACKING] self.assertTrue(hal9000_id in tracking) tracking_info = tracking[hal9000_id] self.assertTrue('id' in tracking_info) self.assertEquals(tracking_info['id'], test_observable.id) self.assertTrue(KEY_TOTAL_COUNT in tracking_info) self.assertTrue(KEY_MAL_COUNT in tracking_info) # we should be tracking the change we made in here self.assertIsNone(tracking_info[KEY_TOTAL_COUNT]) self.assertIsNone(tracking_info[KEY_MAL_COUNT])
def test_hal9000_alert_mal_disposition(self, db, c): # same as above except we end up alerting and disposition as malicious root = create_root_analysis(analysis_mode=ANALYSIS_MODE_ANALYSIS) root.initialize_storage() test_observable = root.add_observable(F_TEST, 'test') root.save() root.schedule() engine = TestEngine(local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION ]) engine.set_cleanup(ANALYSIS_MODE_ANALYSIS, False) engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module( 'analysis_module_hal9000', [ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION]) engine.controlled_stop() engine.start() engine.wait() # XXX - fix this when you implement the gui api # right now the set_disposition function in the API is what both sets the disposition # and re-inserts the alert back into the workload # set the disposition for the alert with get_db_connection() as ace_db: ace_c = ace_db.cursor() ace_c.execute( """ UPDATE alerts SET disposition = %s, disposition_user_id = %s, disposition_time = NOW(), owner_id = %s, owner_time = NOW() WHERE uuid = %s AND ( disposition IS NULL OR disposition != %s )""", (DISPOSITION_DELIVERY, UNITTEST_USER_ID, UNITTEST_USER_ID, root.uuid, DISPOSITION_DELIVERY)) ace_c.execute( """ INSERT INTO workload ( uuid, node_id, analysis_mode, insert_date, company_id, exclusive_uuid, storage_dir ) SELECT alerts.uuid, nodes.id, %s, NOW(), alerts.company_id, NULL, alerts.storage_dir FROM alerts JOIN nodes ON alerts.location = nodes.name WHERE uuid = %s""", (ANALYSIS_MODE_CORRELATION, root.uuid)) ace_db.commit() # run the engine again so that is processes the alert in correlation mode with the disposition set engine = TestEngine(local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION ]) engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module( 'analysis_module_hal9000', [ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION]) engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=root.storage_dir) root.load() # make sure we alerted self.assertEquals(root.analysis_mode, ANALYSIS_MODE_CORRELATION) test_observable = root.get_observable(test_observable.id) self.assertIsNotNone(test_observable) analysis = test_observable.get_analysis(HAL9000Analysis) self.assertIsNotNone(analysis) # these should still both be 0 self.assertEquals(analysis.total_count, 0) self.assertEquals(analysis.mal_count, 0) # we should have a single entry in the database for this observable hal9000_id = _compute_hal9000_md5(test_observable) # with the disposition set we should have the corresponding values c.execute( "SELECT total_count, mal_count FROM observables WHERE id = UNHEX(%s)", (hal9000_id, )) result = c.fetchone() db.commit() self.assertIsNotNone(result) self.assertEquals(result[0], 1) self.assertEquals(result[1], 1) # verify the correct state is kept state = root.state['hal9000'] self.assertTrue(STATE_KEY_ID_TRACKING in state) tracking = state[STATE_KEY_ID_TRACKING] self.assertTrue(hal9000_id in tracking) tracking_info = tracking[hal9000_id] self.assertTrue('id' in tracking_info) self.assertEquals(tracking_info['id'], test_observable.id) self.assertTrue(KEY_TOTAL_COUNT in tracking_info) self.assertTrue(KEY_MAL_COUNT in tracking_info) # we should be tracking the change we made in here self.assertIsNotNone(tracking_info[KEY_TOTAL_COUNT]) self.assertIsNotNone(tracking_info[KEY_MAL_COUNT]) # now we change it to FP with get_db_connection() as ace_db: ace_c = ace_db.cursor() ace_c.execute( """ UPDATE alerts SET disposition = %s, disposition_user_id = %s, disposition_time = NOW(), owner_id = %s, owner_time = NOW() WHERE uuid = %s AND ( disposition IS NULL OR disposition != %s )""", (DISPOSITION_FALSE_POSITIVE, UNITTEST_USER_ID, UNITTEST_USER_ID, root.uuid, DISPOSITION_FALSE_POSITIVE)) ace_c.execute( """ INSERT INTO workload ( uuid, node_id, analysis_mode, insert_date, company_id, exclusive_uuid, storage_dir ) SELECT alerts.uuid, nodes.id, %s, NOW(), alerts.company_id, NULL, alerts.storage_dir FROM alerts JOIN nodes ON alerts.location = nodes.name WHERE uuid = %s""", (ANALYSIS_MODE_CORRELATION, root.uuid)) ace_db.commit() # run the engine again so that is processes the alert in the new correlation mode with the disposition changed engine = TestEngine(local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION ]) engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module( 'analysis_module_hal9000', [ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION]) engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=root.storage_dir) root.load() test_observable = root.get_observable(test_observable.id) self.assertIsNotNone(test_observable) analysis = test_observable.get_analysis(HAL9000Analysis) self.assertIsNotNone(analysis) # these should still both be 0 self.assertEquals(analysis.total_count, 0) self.assertEquals(analysis.mal_count, 0) # we should have a single entry in the database for this observable hal9000_id = _compute_hal9000_md5(test_observable) # with the disposition set we should have the corresponding values c.execute( "SELECT total_count, mal_count FROM observables WHERE id = UNHEX(%s)", (hal9000_id, )) result = c.fetchone() db.commit() self.assertIsNotNone(result) self.assertEquals(result[0], 1) self.assertEquals(result[1], 0) # <-- should be 0 now that it's set to FP # verify the correct state is kept state = root.state['hal9000'] self.assertTrue(STATE_KEY_ID_TRACKING in state) tracking = state[STATE_KEY_ID_TRACKING] self.assertTrue(hal9000_id in tracking) tracking_info = tracking[hal9000_id] self.assertTrue('id' in tracking_info) self.assertEquals(tracking_info['id'], test_observable.id) self.assertTrue(KEY_TOTAL_COUNT in tracking_info) self.assertTrue(KEY_MAL_COUNT in tracking_info) # we should be tracking the change we made in here self.assertIsNotNone(tracking_info[KEY_TOTAL_COUNT]) self.assertIsNone(tracking_info[KEY_MAL_COUNT]) # finally we change it to ignore, which should entirely remove the counters (set them to 0 anyways) with get_db_connection() as ace_db: ace_c = ace_db.cursor() ace_c.execute( """ UPDATE alerts SET disposition = %s, disposition_user_id = %s, disposition_time = NOW(), owner_id = %s, owner_time = NOW() WHERE uuid = %s AND ( disposition IS NULL OR disposition != %s )""", (DISPOSITION_IGNORE, UNITTEST_USER_ID, UNITTEST_USER_ID, root.uuid, DISPOSITION_IGNORE)) ace_c.execute( """ INSERT INTO workload ( uuid, node_id, analysis_mode, insert_date, company_id, exclusive_uuid, storage_dir ) SELECT alerts.uuid, nodes.id, %s, NOW(), alerts.company_id, NULL, alerts.storage_dir FROM alerts JOIN nodes ON alerts.location = nodes.name WHERE uuid = %s""", (ANALYSIS_MODE_CORRELATION, root.uuid)) ace_db.commit() # run the engine again so that is processes the alert in the new correlation mode with the disposition changed engine = TestEngine(local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION ]) engine.enable_module('analysis_module_forced_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_detection', ANALYSIS_MODE_ANALYSIS) engine.enable_module( 'analysis_module_hal9000', [ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CORRELATION]) engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=root.storage_dir) root.load() test_observable = root.get_observable(test_observable.id) self.assertIsNotNone(test_observable) analysis = test_observable.get_analysis(HAL9000Analysis) self.assertIsNotNone(analysis) # these should still both be 0 self.assertEquals(analysis.total_count, 0) self.assertEquals(analysis.mal_count, 0) # we should have a single entry in the database for this observable hal9000_id = _compute_hal9000_md5(test_observable) # with the disposition set we should have the corresponding values c.execute( "SELECT total_count, mal_count FROM observables WHERE id = UNHEX(%s)", (hal9000_id, )) result = c.fetchone() db.commit() self.assertIsNotNone(result) self.assertEquals(result[0], 0) # <-- now both should be set to 0 self.assertEquals(result[1], 0) # verify the correct state is kept state = root.state['hal9000'] self.assertTrue(STATE_KEY_ID_TRACKING in state) tracking = state[STATE_KEY_ID_TRACKING] self.assertTrue(hal9000_id in tracking) tracking_info = tracking[hal9000_id] self.assertTrue('id' in tracking_info) self.assertEquals(tracking_info['id'], test_observable.id) self.assertTrue(KEY_TOTAL_COUNT in tracking_info) self.assertTrue(KEY_MAL_COUNT in tracking_info) # we should be tracking the change we made in here self.assertIsNone(tracking_info[KEY_TOTAL_COUNT]) self.assertIsNone(tracking_info[KEY_MAL_COUNT])
def test_cloudphish_tracking(self, db, c): from saq.modules.email import EmailAnalysis saq.CONFIG['analysis_mode_email']['cleanup'] = 'no' self.start_api_server() root = create_root_analysis(alert_type='mailbox', analysis_mode=ANALYSIS_MODE_EMAIL) root.initialize_storage() shutil.copy( os.path.join('test_data', 'emails', 'splunk_logging.email.rfc822'), os.path.join(root.storage_dir, 'email.rfc822')) file_observable = root.add_observable(F_FILE, 'email.rfc822') file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL) test_observable = root.add_observable(F_TEST, 'test_detection') test_observable.add_directive(DIRECTIVE_TRACKED) root.save() root.schedule() analysis_modes = [ ANALYSIS_MODE_EMAIL, ANALYSIS_MODE_CLOUDPHISH, ANALYSIS_MODE_CORRELATION ] analysis_modules = [ 'analysis_module_file_type', 'analysis_module_email_analyzer', 'analysis_module_mailbox_email_analyzer', 'analysis_module_cloudphish', 'analysis_module_cloudphish_request_analyzer', 'analysis_module_crawlphish', 'analysis_module_url_extraction', 'analysis_module_detection' ] engine = TestEngine(local_analysis_modes=analysis_modes) for module in analysis_modules: engine.enable_module(module, analysis_modes) # we only enable the BasicTestAnalyzer for the cloudphish mode so that cloudphish generates an alert engine.enable_module('analysis_module_basic_test', ANALYSIS_MODE_CLOUDPHISH) engine.controlled_stop() engine.start() engine.wait() # get the message_id observable generated by the EmailAnalysis root = RootAnalysis(storage_dir=storage_dir_from_uuid(root.uuid)) root.load() file_observable = root.get_observable(file_observable.id) self.assertIsNotNone(file_observable) email_analysis = file_observable.get_analysis(EmailAnalysis) self.assertTrue(bool(email_analysis)) message_id = email_analysis.find_observable( lambda o: o.type == F_MESSAGE_ID) self.assertIsNotNone(message_id) # we should have a number of cloudphish alerts now c.execute("SELECT uuid FROM alerts WHERE tool != 'test_tool' LIMIT 1") row = c.fetchone() target_uuid = row[0] root = RootAnalysis(storage_dir=storage_dir_from_uuid(target_uuid)) root.load() # this cloudphish alert should have the message_id observable # and it should be tagged as tracked self.assertIsNotNone( root.find_observable(lambda o: o.type == F_MESSAGE_ID and o.value == message_id.value and o.has_tag('tracked')))
def test_submit(self): # disable cleaup for analysis mode analysis saq.CONFIG['analysis_mode_analysis']['cleanup'] = 'no' saq.CONFIG['analysis_mode_cloudphish']['cleanup'] = 'no' self.start_api_server() root = create_root_analysis(analysis_mode=ANALYSIS_MODE_ANALYSIS) root.initialize_storage() url = root.add_observable(F_URL, TEST_URL) url.add_directive(DIRECTIVE_CRAWL) root.save() root.schedule() engine = TestEngine(analysis_pools={ ANALYSIS_MODE_ANALYSIS: 1, ANALYSIS_MODE_CLOUDPHISH: 1 }, local_analysis_modes=[ ANALYSIS_MODE_ANALYSIS, ANALYSIS_MODE_CLOUDPHISH ]) engine.enable_module('analysis_module_cloudphish', ANALYSIS_MODE_ANALYSIS) engine.enable_module('analysis_module_cloudphish_request_analyzer', ANALYSIS_MODE_CLOUDPHISH) engine.enable_module('analysis_module_crawlphish', ANALYSIS_MODE_CLOUDPHISH) engine.start() # wait for delayed analysis to be added wait_for_log_count('added delayed analysis', 1, 5) # and then wait for analysis to complete wait_for_log_count('completed analysis RootAnalysis', 1, 5) # watch for the download request by cloudphish wait_for_log_count('requesting url', 1, 5) # watch for crawlphish to finish wait_for_log_count('analysis CrawlphishAnalysisV2 is completed', 1, 5) # the cloudphish request analyzer should see the downloaded file wait_for_log_count('found downloaded file', 1, 5) # and should update the database wait_for_log_count('executing cloudphish update', 1, 5) # should see cloudphish module complete wait_for_log_count('analysis CloudphishAnalysis is completed', 1, 10) # we should a work request for the original request and one for the cloudphish request wait_for_log_count('got work item RootAnalysis', 2, 5) # and we should see at least one request to handle a delayed analysis request wait_for_log_count('got work item DelayedAnalysisRequest', 1, 5) engine.controlled_stop() engine.wait() # check the results root = RootAnalysis(storage_dir=root.storage_dir) root.load() url = root.get_observable(url.id) self.assertIsNotNone(url) # this url should only have a single analysis object attached to it (the cloudphish analysis) self.assertEquals(len(url.analysis), 1) from saq.modules.cloudphish import CloudphishAnalysis cloudphish_analysis = url.get_analysis(CloudphishAnalysis) self.assertIsNotNone(cloudphish_analysis) for key in [ KEY_ANALYSIS_RESULT, KEY_DETAILS, KEY_FILE_NAME, KEY_HTTP_MESSAGE, KEY_HTTP_RESULT, KEY_LOCATION, KEY_RESULT, KEY_SHA256_CONTENT, KEY_SHA256_URL, KEY_STATUS, KEY_UUID ]: with self.subTest(key=key): self.assertTrue(key in cloudphish_analysis.query_result) q = cloudphish_analysis.query_result self.assertEquals(q[KEY_ANALYSIS_RESULT], SCAN_RESULT_CLEAR) for key in [ KEY_DETAILS_CONTEXT, KEY_DETAILS_SHA256_URL, KEY_DETAILS_URL ]: with self.subTest(key=key): self.assertTrue(key in q[KEY_DETAILS]) # this is what we should have for context self.assertEquals(q[KEY_DETAILS][KEY_DETAILS_CONTEXT]['c'], root.uuid) self.assertEquals( q[KEY_DETAILS][KEY_DETAILS_CONTEXT]['ignore_filters'], '0') self.assertEquals(q[KEY_DETAILS][KEY_DETAILS_CONTEXT]['reprocess'], '0') self.assertEquals( q[KEY_DETAILS][KEY_DETAILS_SHA256_URL].upper(), 'B009F8821B162674B819A2365B07A536645A42657E75BB3996C8B6127E993806') self.assertEquals(q[KEY_DETAILS][KEY_DETAILS_URL], TEST_URL) self.assertEquals(q[KEY_FILE_NAME], 'Payment_Advice.pdf') self.assertEquals(q[KEY_HTTP_MESSAGE], 'OK') self.assertEquals(q[KEY_HTTP_RESULT], 200) self.assertEquals(q[KEY_LOCATION], saq.SAQ_NODE) self.assertEquals(q[KEY_RESULT], RESULT_OK) self.assertEquals( q[KEY_SHA256_CONTENT].upper(), 'FA13C652534F9207BEEC811A50948860F5B3194AEAE686FCDECAC645FAE65D15') self.assertEquals( q[KEY_SHA256_URL].upper(), 'B009F8821B162674B819A2365B07A536645A42657E75BB3996C8B6127E993806') self.assertEquals(q[KEY_STATUS], STATUS_ANALYZED) self.assertIsNotNone(q[KEY_UUID]) # these properties should match the query_result keys-value pairs self.assertEquals(cloudphish_analysis.result, q[KEY_RESULT]) self.assertEquals(cloudphish_analysis.result_details, q[KEY_DETAILS]) self.assertEquals(cloudphish_analysis.status, q[KEY_STATUS]) self.assertEquals(cloudphish_analysis.analysis_result, q[KEY_ANALYSIS_RESULT]) self.assertEquals(cloudphish_analysis.http_result, q[KEY_HTTP_RESULT]) self.assertEquals(cloudphish_analysis.http_message, q[KEY_HTTP_MESSAGE]) self.assertEquals(cloudphish_analysis.sha256_content, q[KEY_SHA256_CONTENT]) self.assertEquals(cloudphish_analysis.location, q[KEY_LOCATION]) self.assertEquals(cloudphish_analysis.file_name, q[KEY_FILE_NAME])
def test_file_analysis_001_oletools_000(self): #from saq.modules.file_analysis import OLEVBA_Analysis_v1_2 KEY_STORAGE_DIR = 'storage_dir' KEY_TAGS = 'tags' KEY_MACRO_COUNT = 'macro_count' KEY_OID = 'oid' KEY_SANDBOX = 'sandbox' # expected results for the various files results = { 'Past Due Invoices.doc': { KEY_OID: None, KEY_STORAGE_DIR: None, KEY_TAGS: [ 'microsoft_office', 'ole' ], KEY_MACRO_COUNT: 4, KEY_SANDBOX: True, }, 'Outstanding Invoices.doc': { KEY_OID: None, KEY_STORAGE_DIR: None, KEY_TAGS: [ 'microsoft_office', 'ole' ], KEY_MACRO_COUNT: 3, KEY_SANDBOX: True, }, 'Paid Invoice.doc': { KEY_OID: None, KEY_STORAGE_DIR: None, KEY_TAGS: [ 'microsoft_office', 'ole' ], KEY_MACRO_COUNT: 3, KEY_SANDBOX: True, }, 'mortgage_payment-0873821-0565.docm': { KEY_OID: None, KEY_STORAGE_DIR: None, KEY_TAGS: [ 'microsoft_office' ], KEY_MACRO_COUNT: 1, KEY_SANDBOX: True, }, 'receipt_687790.doc': { KEY_OID: None, KEY_STORAGE_DIR: None, KEY_TAGS: [ 'microsoft_office' ], KEY_MACRO_COUNT: 5, KEY_SANDBOX: True, }, } for file_name in results.keys(): root = create_root_analysis(uuid=str(uuid.uuid4())) root.initialize_storage() target_path = os.path.join('test_data/ole_files', file_name) shutil.copy(target_path, root.storage_dir) file_observable = root.add_observable(F_FILE, file_name) root.save() root.schedule() results[file_name][KEY_OID] = file_observable.id results[file_name][KEY_STORAGE_DIR] = root.storage_dir engine = TestEngine() engine.enable_module('analysis_module_olevba_v1_2', 'test_groups') engine.enable_module('analysis_module_file_type', 'test_groups') engine.controlled_stop() engine.start() engine.wait() for file_name in results.keys(): with self.subTest(storage_dir=results[file_name][KEY_STORAGE_DIR], file_name=file_name): root = RootAnalysis(storage_dir=results[file_name][KEY_STORAGE_DIR]) root.load() file_observable = root.get_observable(results[file_name][KEY_OID]) self.assertIsNotNone(file_observable) if results[file_name][KEY_SANDBOX]: self.assertTrue(file_observable.has_directive(DIRECTIVE_SANDBOX)) for tag in results[file_name][KEY_TAGS]: with self.subTest(storage_dir=results[file_name][KEY_STORAGE_DIR], file_name=file_name, tag=tag): self.assertTrue(file_observable.has_tag(tag)) macro_count = len([f for f in root.all_observables if f.type == F_FILE and f.has_tag('macro')]) self.assertEquals(macro_count, results[file_name][KEY_MACRO_COUNT])