def test_legacy_submit(self): alert = ace_api.Alert(description='Test Alert') alert.add_observable(F_IPV4, '1.2.3.4', local_time(), directives=[DIRECTIVE_NO_SCAN]) alert.add_tag('test') temp_path = os.path.join(saq.TEMP_DIR, 'test.txt') with open(temp_path, 'w') as fp: fp.write('test') alert.add_attachment_link(temp_path, 'dest/test.txt') uuid = alert.submit( f'https://{saq.API_PREFIX}', ssl_verification=saq.CONFIG['SSL']['ca_chain_path']) self.assertTrue(validate_uuid(uuid)) root = RootAnalysis(storage_dir=storage_dir_from_uuid(uuid)) root.load() self.assertEquals(root.description, 'Test Alert') ipv4_observable = root.find_observable(lambda o: o.type == F_IPV4) self.assertIsNotNone(ipv4_observable) self.assertEquals(ipv4_observable.value, '1.2.3.4') self.assertTrue(ipv4_observable.has_directive(DIRECTIVE_NO_SCAN)) file_observable = root.find_observable(lambda o: o.type == F_FILE) self.assertIsNotNone(file_observable) self.assertEquals(file_observable.value, 'dest/test.txt') with open(os.path.join(root.storage_dir, file_observable.value), 'r') as fp: self.assertEquals(fp.read(), 'test')
def test_resubmit(self, db, c): # submit something so we have something to resubmit result = self._submit(analysis_mode=ANALYSIS_MODE_CORRELATION) self.assertIsNotNone(result) self.assertTrue('result' in result) result = result['result'] self.assertIsNotNone(result['uuid']) uuid = result['uuid'] # make sure this actually uploaded root = RootAnalysis(storage_dir=storage_dir_from_uuid(uuid)) root.load() self.assertEquals(root.analysis_mode, ANALYSIS_MODE_CORRELATION) self.assertEquals(root.tool, 'unittest_tool') self.assertEquals(root.tool_instance, 'unittest_tool_instance') self.assertEquals(root.alert_type, 'unittest_type') self.assertEquals(root.description, 'testing') self.assertEquals(root.details, {'hello': 'world'}) self.assertEquals(root.event_time, self._get_localized_submit_time()) self.assertEquals(root.tags[0].name, 'alert_tag_1') self.assertEquals(root.tags[1].name, 'alert_tag_2') # NOTE that this is 4 instead of 2 since adding a file adds a F_FILE observable type self.assertEquals(len(root.all_observables), 4) o = root.find_observable(lambda o: o.type == 'ipv4') self.assertIsNotNone(o) self.assertEquals(o.value, '1.2.3.4') self.assertEquals(len(o.tags), 2) self.assertTrue(o.has_directive('no_scan')) self.assertTrue('basic_test' in o.limited_analysis) o = root.find_observable( lambda o: o.type == 'file' and o.value == 'sample.dat') self.assertIsNotNone(o) with open(os.path.join(root.storage_dir, o.value), 'rb') as fp: self.assertEquals(fp.read(), b'Hello, world!') o = root.find_observable( lambda o: o.type == 'file' and o.value == 'submit_test.dat') self.assertIsNotNone(o) self.assertEquals( os.path.getsize(os.path.join(root.storage_dir, o.value)), 1024) # we should see a single workload entry c.execute( "SELECT id, uuid, node_id, analysis_mode FROM workload WHERE uuid = %s", (uuid, )) row = c.fetchone() self.assertIsNotNone(row) self.assertIsNotNone(row[0]) self.assertEquals(row[1], uuid) self.assertIsNotNone(row[2]) self.assertEquals(row[3], ANALYSIS_MODE_CORRELATION) # now resubmit the alert result = ace_api.resubmit_alert(uuid) self.assertFalse('error' in result)
def get_file(uuid, file_uuid_or_name): storage_dir = storage_dir_from_uuid(uuid) if saq.CONFIG['service_engine']['work_dir'] and not os.path.isdir(storage_dir): storage_dir = workload_storage_dir(uuid) root = RootAnalysis(storage_dir=storage_dir) root.load() # is this a UUID? try: validate_uuid(file_uuid_or_name) file_observable = root.get_observable(file_uuid_or_name) if file_observable is None: abort(Response("invalid file_uuid {}".format(file_uuid_or_name), 400)) except ValueError: file_observable = root.find_observable(lambda o: o.type == F_FILE and o.value == file_uuid_or_name) if file_observable is None: abort(Response("invalid file name {}".format(file_uuid_or_name), 400)) # NOTE we use an absolute path here because if we don't then # send_from_directory makes it relavive from the app root path # which is (/opt/ace/aceapi) target_path = os.path.join(saq.SAQ_HOME, root.storage_dir, file_observable.value) if not os.path.exists(target_path): abort(Response("file path {} does not exist".format(target_path), 400)) # XXX revisit how we save (name) files return send_from_directory(os.path.dirname(target_path), os.path.basename(target_path), as_attachment=True, attachment_filename=os.path.basename(target_path).encode().decode('latin-1', errors='ignore'))
def test_mailbox_submission(self): from flask import url_for from saq.analysis import _JSONEncoder from saq.modules.email import EmailAnalysis t = saq.LOCAL_TIMEZONE.localize(datetime.datetime.now()).astimezone(pytz.UTC).strftime(event_time_format_json_tz) with open(os.path.join('test_data', 'emails', 'splunk_logging.email.rfc822'), 'rb') as fp: result = self.client.post(url_for('analysis.submit'), data={ 'analysis': json.dumps({ 'analysis_mode': 'email', 'tool': 'unittest', 'tool_instance': 'unittest_instance', 'type': 'mailbox', 'description': 'testing', 'event_time': t, 'details': { }, 'observables': [ { 'type': F_FILE, 'value': 'rfc822.email', 'time': t, 'tags': [], 'directives': [ DIRECTIVE_ORIGINAL_EMAIL ], 'limited_analysis': [] }, ], 'tags': [ ], }, cls=_JSONEncoder), 'file': (fp, 'rfc822.email'), }, content_type='multipart/form-data') result = result.get_json() self.assertIsNotNone(result) self.assertTrue('result' in result) result = result['result'] self.assertIsNotNone(result['uuid']) uuid = result['uuid'] # make sure we don't clean up the anaysis so we can check it saq.CONFIG['analysis_mode_email']['cleanup'] = 'no' engine = TestEngine(local_analysis_modes=['email']) engine.enable_module('analysis_module_file_type', 'email') engine.enable_module('analysis_module_email_analyzer', 'email') engine.enable_module('analysis_module_mailbox_email_analyzer', 'email') engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=storage_dir_from_uuid(uuid)) root.load() observable = root.find_observable(lambda o: o.has_directive(DIRECTIVE_ORIGINAL_EMAIL)) self.assertIsNotNone(observable) analysis = observable.get_analysis(EmailAnalysis) self.assertIsNotNone(analysis) # these should be the same self.assertEquals(analysis.details, root.details)
def test_bro_smtp_stream_submission(self): from flask import url_for from saq.analysis import _JSONEncoder from saq.modules.email import EmailAnalysis, BroSMTPStreamAnalysis t = saq.LOCAL_TIMEZONE.localize(datetime.datetime.now()).astimezone(pytz.UTC).strftime(event_time_format_json_tz) with open(os.path.join('test_data', 'smtp_streams', 'CBmtfvapmTMqCEUw6'), 'rb') as fp: result = self.client.post(url_for('analysis.submit'), data={ 'analysis': json.dumps({ 'analysis_mode': ANALYSIS_MODE_EMAIL, 'tool': 'unittest', 'tool_instance': 'unittest_instance', 'type': ANALYSIS_TYPE_BRO_SMTP, 'description': 'BRO SMTP Scanner Detection - ', 'event_time': t, 'details': { }, 'observables': [ { 'type': F_FILE, 'value': 'CBmtfvapmTMqCEUw6', 'time': t, 'tags': [], 'directives': [ DIRECTIVE_ORIGINAL_SMTP ], 'limited_analysis': [] }, ], 'tags': [ ], }, cls=_JSONEncoder), 'file': (fp, 'CBmtfvapmTMqCEUw6'), }, content_type='multipart/form-data') result = result.get_json() self.assertIsNotNone(result) self.assertTrue('result' in result) result = result['result'] self.assertIsNotNone(result['uuid']) uuid = result['uuid'] # make sure we don't clean up the anaysis so we can check it saq.CONFIG['analysis_mode_email']['cleanup'] = 'no' engine = TestEngine(local_analysis_modes=[ANALYSIS_MODE_EMAIL]) engine.enable_module('analysis_module_file_type', 'email') engine.enable_module('analysis_module_email_analyzer', 'email') engine.enable_module('analysis_module_bro_smtp_analyzer', 'email') engine.controlled_stop() engine.start() engine.wait() root = RootAnalysis(storage_dir=storage_dir_from_uuid(uuid)) root.load() observable = root.find_observable(lambda o: o.has_directive(DIRECTIVE_ORIGINAL_SMTP)) self.assertIsNotNone(observable) analysis = observable.get_analysis(BroSMTPStreamAnalysis) self.assertIsNotNone(analysis)
def test_complete_processing(self): from saq.modules.email import BroSMTPStreamAnalysis # disable cleanup so we can check the results after saq.CONFIG['analysis_mode_email']['cleanup'] = 'no' self.process_pcap(os.path.join(saq.SAQ_HOME, 'test_data', 'pcaps', 'smtp.pcap')) self.start_api_server() engine = TestEngine() engine.enable_module('analysis_module_bro_smtp_analyzer', 'email') engine.start() collector = BroSMTPStreamCollector() collector.load_groups() collector.start() # look for all the expected log entries wait_for_log_count('found smtp stream', 1, 5) wait_for_log_count('copied file from', 1, 5) wait_for_log_count('scheduled BRO SMTP Scanner Detection -', 1, 5) wait_for_log_count('completed analysis RootAnalysis', 1, 20) engine.controlled_stop() engine.wait() collector.stop() collector.wait() # get the uuids returned by the api calls r = re.compile(r' uuid ([a-f0-9-]+)') for result in search_log('submit remote'): m = r.search(result.getMessage()) self.assertIsNotNone(m) uuid = m.group(1) with self.subTest(uuid=uuid): root = RootAnalysis(uuid=uuid, storage_dir=storage_dir_from_uuid(uuid)) root.load() # find the SMTP stream file_observable = root.find_observable(lambda x: x.type == F_FILE) self.assertTrue(bool(file_observable)) # ensure it has the required directives self.assertTrue(file_observable.has_directive(DIRECTIVE_ORIGINAL_SMTP)) self.assertTrue(file_observable.has_directive(DIRECTIVE_NO_SCAN)) # ensure the bro smtp analyzer ran on it smtp_analysis = file_observable.get_analysis(BroSMTPStreamAnalysis) self.assertIsNotNone(smtp_analysis) # ensure it extracted a file email_observable = smtp_analysis.find_observable(lambda x: x.type == F_FILE) self.assertTrue(bool(email_observable)) # and then ensure that it was treated as an email #import pdb; pdb.set_trace() self.assertTrue(email_observable.has_directive(DIRECTIVE_NO_SCAN)) self.assertTrue(email_observable.has_directive(DIRECTIVE_ORIGINAL_EMAIL)) self.assertTrue(email_observable.has_directive(DIRECTIVE_ARCHIVE))
def test_cloudphish_tracking(self, db, c): from saq.modules.email import EmailAnalysis saq.CONFIG['analysis_mode_email']['cleanup'] = 'no' self.start_api_server() root = create_root_analysis(alert_type='mailbox', analysis_mode=ANALYSIS_MODE_EMAIL) root.initialize_storage() shutil.copy( os.path.join('test_data', 'emails', 'splunk_logging.email.rfc822'), os.path.join(root.storage_dir, 'email.rfc822')) file_observable = root.add_observable(F_FILE, 'email.rfc822') file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL) test_observable = root.add_observable(F_TEST, 'test_detection') test_observable.add_directive(DIRECTIVE_TRACKED) root.save() root.schedule() analysis_modes = [ ANALYSIS_MODE_EMAIL, ANALYSIS_MODE_CLOUDPHISH, ANALYSIS_MODE_CORRELATION ] analysis_modules = [ 'analysis_module_file_type', 'analysis_module_email_analyzer', 'analysis_module_mailbox_email_analyzer', 'analysis_module_cloudphish', 'analysis_module_cloudphish_request_analyzer', 'analysis_module_crawlphish', 'analysis_module_url_extraction', 'analysis_module_detection' ] engine = TestEngine(local_analysis_modes=analysis_modes) for module in analysis_modules: engine.enable_module(module, analysis_modes) # we only enable the BasicTestAnalyzer for the cloudphish mode so that cloudphish generates an alert engine.enable_module('analysis_module_basic_test', ANALYSIS_MODE_CLOUDPHISH) engine.controlled_stop() engine.start() engine.wait() # get the message_id observable generated by the EmailAnalysis root = RootAnalysis(storage_dir=storage_dir_from_uuid(root.uuid)) root.load() file_observable = root.get_observable(file_observable.id) self.assertIsNotNone(file_observable) email_analysis = file_observable.get_analysis(EmailAnalysis) self.assertTrue(bool(email_analysis)) message_id = email_analysis.find_observable( lambda o: o.type == F_MESSAGE_ID) self.assertIsNotNone(message_id) # we should have a number of cloudphish alerts now c.execute("SELECT uuid FROM alerts WHERE tool != 'test_tool' LIMIT 1") row = c.fetchone() target_uuid = row[0] root = RootAnalysis(storage_dir=storage_dir_from_uuid(target_uuid)) root.load() # this cloudphish alert should have the message_id observable # and it should be tagged as tracked self.assertIsNotNone( root.find_observable(lambda o: o.type == F_MESSAGE_ID and o.value == message_id.value and o.has_tag('tracked')))