def test_file_analysis_004_yara_002_no_alert(self): self.initialize_yss() engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_yara_scanner_v3_4') self.start_engine(engine) root = create_root_analysis(uuid=str(uuid.uuid4())) root.initialize_storage() shutil.copy('test_data/scan_targets/no_alert', root.storage_dir) _file = root.add_observable(F_FILE, 'no_alert') root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() self.assertEquals(log_count('with yss (matches found: True)'), 1) root.load() _file = root.get_observable(_file.id) from saq.modules.file_analysis import YaraScanResults_v3_4 analysis = _file.get_analysis(YaraScanResults_v3_4) self.assertTrue(analysis) # the file should NOT be instructed to go to the sandbox self.assertFalse(_file.has_directive(DIRECTIVE_SANDBOX)) # the analysis should have a yara_rule observable yara_rule = analysis.get_observables_by_type(F_YARA_RULE) self.assertEquals(len(yara_rule), 1) yara_rule = yara_rule[0] # the yara rule should NOT have detections self.assertFalse(yara_rule.detections)
def test_carbon_black_asset_ident_000(self): from saq.modules.asset import CarbonBlackAssetIdentAnalysis # find an IP address in the past 24 hours to use q, result = splunk_query( """index=carbonblack | dedup local_ip | head limit=1 | fields local_ip""" ) self.assertTrue(result) self.assertTrue(isinstance(q.json(), list)) self.assertEquals(len(q.json()), 1) ipv4 = q.json()[0]['local_ip'] logging.info("using ipv4 {} for test".format(ipv4)) engine = AnalysisEngine() engine.enable_module('analysis_module_carbon_black_asset_ident') self.start_engine(engine) root = create_root_analysis(event_time=datetime.datetime.now()) root.initialize_storage() o_uuid = root.add_observable(F_IPV4, ipv4).id root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() ipv4 = root.get_observable(o_uuid) self.assertIsNotNone(ipv4) analysis = ipv4.get_analysis(CarbonBlackAssetIdentAnalysis) self.assertIsNotNone(analysis) self.assertIsNotNone(analysis.details) self.assertEquals(len(analysis.discovered_hostnames), 1)
def test_crawlphish_001_download_404(self): """We should not extract URLs from data downloaded from URLs that returned a 404.""" from saq.modules.url import CrawlphishAnalysisV2 engine = AnalysisEngine() engine.enable_module('analysis_module_crawlphish') self.start_engine(engine) root = create_root_analysis() root.initialize_storage() url = root.add_observable(F_URL, 'http://localhost:{}/test_data/crawlphish.001'.format(LOCAL_PORT)) url.add_directive(DIRECTIVE_CRAWL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() url = root.get_observable(url.id) analysis = url.get_analysis(CrawlphishAnalysisV2) self.assertEquals(analysis.proxy_results['GLOBAL'].status_code, 404) if 'tor' in analysis.proxy_results: self.assertIsNone(analysis.proxy_results['tor'].status_code) self.assertIsNone(analysis.file_name) # no file should have been downloaded self.assertFalse(analysis.downloaded) self.assertIsNotNone(analysis.error_reason) file_observables = analysis.get_observables_by_type(F_FILE) self.assertEquals(len(file_observables), 0)
def test_protected_url_003_sharepoint(self): engine = AnalysisEngine() engine.enable_module('analysis_module_protected_url_analyzer') self.start_engine(engine) root = create_root_analysis() root.initialize_storage() # taken from an actual sample url = root.add_observable(F_URL, 'https://lahia-my.sharepoint.com/:b:/g/personal/secure_onedrivemsw_bid/EVdjoBiqZTxMnjAcDW6yR4gBqJ59ALkT1C2I3L0yb_n0uQ?e=naeXYD') url.add_directive(DIRECTIVE_CRAWL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() url = root.get_observable(url.id) from saq.modules.url import ProtectedURLAnalysis, PROTECTION_TYPE_SHAREPOINT analysis = url.get_analysis(ProtectedURLAnalysis) self.assertIsNotNone(analysis) self.assertEquals(analysis.protection_type, PROTECTION_TYPE_SHAREPOINT) from urllib.parse import urlparse, parse_qs parsed_url = urlparse(analysis.extracted_url) self.assertEquals(parsed_url.path, '/personal/secure_onedrivemsw_bid/_layouts/15/download.aspx') parsed_qs = parse_qs(parsed_url.query) self.assertEquals(parsed_qs['e'][0], 'naeXYD') self.assertEquals(parsed_qs['share'][0], 'EVdjoBiqZTxMnjAcDW6yR4gBqJ59ALkT1C2I3L0yb_n0uQ') extracted_url = analysis.get_observables_by_type(F_URL) self.assertEquals(len(extracted_url), 1) extracted_url = extracted_url[0] self.assertTrue(extracted_url.has_directive(DIRECTIVE_CRAWL))
def test_vx_000_hash_lookup(self): engine = AnalysisEngine() engine.enable_module('analysis_module_vxstream_hash_analyzer') self.start_engine(engine) root = create_root_analysis(event_time=datetime.datetime.now()) root.initialize_storage() sha2 = root.add_observable(F_SHA256, SAMPLE_HASH) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() sha2 = root.get_observable(sha2.id) from saq.modules.vx import VxStreamHashAnalysis analysis = sha2.get_analysis(VxStreamHashAnalysis) self.assertIsNotNone(analysis) self.assertEquals(analysis.sha256, sha2.value) self.assertEquals(analysis.environment_id, saq.CONFIG['vxstream']['environmentid']) self.assertEquals(analysis.status, VXSTREAM_STATUS_SUCCESS) self.assertIsNotNone(analysis.submit_date) self.assertIsNotNone(analysis.complete_date) self.assertIsNone(analysis.fail_date) self.assertIsNotNone(analysis.vxstream_threat_level) self.assertIsNotNone(analysis.vxstream_threat_score)
def test_vx_003_file_with_hash_analysis(self): engine = AnalysisEngine() engine.enable_module('analysis_module_vxstream_hash_analyzer') engine.enable_module('analysis_module_vxstream_file_analyzer') engine.enable_module('analysis_module_file_hash_analyzer') engine.enable_module('analysis_module_file_type') self.start_engine(engine) root = create_root_analysis(event_time=datetime.datetime.now()) root.initialize_storage() with open('/dev/urandom', 'rb') as fp_in: # using an extension here that doesn't get hash anlaysis with open('test_data/invalid.pcap', 'wb') as fp_out: fp_out.write(fp_in.read(4096)) shutil.copy('test_data/invalid.pcap', root.storage_dir) _file = root.add_observable(F_FILE, 'invalid.pcap') _file.add_directive(DIRECTIVE_SANDBOX) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() _file = root.get_observable(_file.id) from saq.modules.vx import VxStreamFileAnalysis analysis = _file.get_analysis(VxStreamFileAnalysis) self.assertFalse(analysis)
def test_file_analysis_005_pcode_000_extract_pcode(self): engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_pcodedmp') engine.enable_module('analysis_module_file_type') self.start_engine(engine) root = create_root_analysis(uuid=str(uuid.uuid4())) root.initialize_storage() shutil.copy('test_data/ole_files/word2013_macro_stripped.doc', root.storage_dir) _file = root.add_observable(F_FILE, 'word2013_macro_stripped.doc') root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() _file = root.get_observable(_file.id) self.assertIsNotNone(_file) from saq.modules.file_analysis import PCodeAnalysis analysis = _file.get_analysis(PCodeAnalysis) self.assertTrue(analysis) # we should have extracted 11 lines of macro self.assertEquals(analysis.details, 11) # and we should have a file with the macros _file = analysis.get_observables_by_type(F_FILE) self.assertEquals(len(_file), 1) _file = _file[0] # and that should have a redirection self.assertIsNotNone(_file.redirection)
def test_protected_url_000_outlook_safelinks(self): engine = AnalysisEngine() engine.enable_module('analysis_module_protected_url_analyzer') self.start_engine(engine) root = create_root_analysis() root.initialize_storage() # taken from an actual sample url = root.add_observable(F_URL, 'https://na01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fwww.getbusinessready.com.au%2FInvoice-Number-49808%2F&data=02%7C01%7Ccyoung%40northernaviationservices.aero%7C8a388036cbf34f90ec5808d5724be7ed%7Cfc01978435d14339945c4161ac91c300%7C0%7C0%7C636540592704791165&sdata=%2FNQGqAp09WTNgnVnpoWIPcYNVAYsJ11ULuSS7cCsS3Q%3D&reserved=0') url.add_directive(DIRECTIVE_CRAWL) # not actually going to crawl, just testing that it gets copied over root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() url = root.get_observable(url.id) from saq.modules.url import ProtectedURLAnalysis, PROTECTION_TYPE_OUTLOOK_SAFELINKS analysis = url.get_analysis(ProtectedURLAnalysis) self.assertIsNotNone(analysis) self.assertEquals(analysis.protection_type, PROTECTION_TYPE_OUTLOOK_SAFELINKS) self.assertEquals(analysis.extracted_url, 'http://www.getbusinessready.com.au/Invoice-Number-49808/') extracted_url = analysis.get_observables_by_type(F_URL) self.assertEquals(len(extracted_url), 1) extracted_url = extracted_url[0] self.assertTrue(extracted_url.has_directive(DIRECTIVE_CRAWL))
def test_file_analysis_006_extracted_ole_000_js(self): engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_archive') engine.enable_module('analysis_module_extracted_ole_analyzer') engine.enable_module('analysis_module_officeparser_v1_0') engine.enable_module('analysis_module_file_type') self.start_engine(engine) root = create_root_analysis() root.initialize_storage() shutil.copy('test_data/docx/js_ole_obj.docx', root.storage_dir) _file = root.add_observable(F_FILE, 'js_ole_obj.docx') root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() _file = root.get_observable(_file.id) self.assertIsNotNone(_file) self.assertTrue( any([ d for d in root.all_detection_points if 'compiles as JavaScript' in d.description ]))
def test_live_browser_001_404(self): """We should not download screenshots for URLs that returned a 404 error message.""" from saq.modules.url import CrawlphishAnalysisV2 from saq.modules.url import LiveBrowserAnalysis engine = AnalysisEngine() engine.enable_module('analysis_module_crawlphish') engine.enable_module('analysis_module_live_browser_analyzer') self.start_engine(engine) root = create_root_analysis() root.initialize_storage() # this file does not exist url = root.add_observable(F_URL, 'http://localhost:{}/test_data/live_browser.dne.html'.format(LOCAL_PORT)) url.add_directive(DIRECTIVE_CRAWL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() url = root.get_observable(url.id) analysis = url.get_analysis(CrawlphishAnalysisV2) file_observables = analysis.get_observables_by_type(F_FILE) self.assertEquals(len(file_observables), 0)
def test_file_analysis_002_archive_002_ace(self): engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_archive') engine.enable_module('analysis_module_file_type') self.start_engine(engine) root = create_root_analysis(uuid=str(uuid.uuid4())) root.initialize_storage() shutil.copy('test_data/ace/dhl_report.ace', root.storage_dir) _file = root.add_observable(F_FILE, 'dhl_report.ace') root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() _file = root.get_observable(_file.id) from saq.modules.file_analysis import ArchiveAnalysis analysis = _file.get_analysis(ArchiveAnalysis) self.assertIsNotNone(analysis) self.assertEquals(analysis.file_count, 1) _file = analysis.get_observables_by_type(F_FILE) self.assertEquals(len(_file), 1)
def test_protected_url_002_google_drive(self): engine = AnalysisEngine() engine.enable_module('analysis_module_protected_url_analyzer') self.start_engine(engine) root = create_root_analysis() root.initialize_storage() # taken from an actual sample url = root.add_observable(F_URL, 'https://drive.google.com/file/d/1ls_eBCsmf3VG_e4dgQiSh_5VUM10b9s2/view') url.add_directive(DIRECTIVE_CRAWL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() url = root.get_observable(url.id) from saq.modules.url import ProtectedURLAnalysis, PROTECTION_TYPE_GOOGLE_DRIVE analysis = url.get_analysis(ProtectedURLAnalysis) self.assertIsNotNone(analysis) self.assertEquals(analysis.protection_type, PROTECTION_TYPE_GOOGLE_DRIVE) self.assertEquals(analysis.extracted_url, 'https://drive.google.com/uc?authuser=0&id=1ls_eBCsmf3VG_e4dgQiSh_5VUM10b9s2&export=download') extracted_url = analysis.get_observables_by_type(F_URL) self.assertEquals(len(extracted_url), 1) extracted_url = extracted_url[0] self.assertTrue(extracted_url.has_directive(DIRECTIVE_CRAWL))
def test_file_analysis_000_url_extraction_000_relative_html_urls(self): from saq.modules.file_analysis import URLExtractionAnalysis engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_url_extraction') engine.enable_module('analysis_module_file_type') self.start_engine(engine) root = create_root_analysis(event_time=datetime.datetime.now()) root.initialize_storage() shutil.copy('test_data/url_extraction_000', root.storage_dir) target_file = 'url_extraction_000' src_url = 'https://vaishaligarden.com/.opjl/' url_observable = root.add_observable(F_URL, src_url) file_observable = root.add_observable(F_FILE, target_file) file_observable.add_directive(DIRECTIVE_EXTRACT_URLS) file_observable.add_relationship(R_DOWNLOADED_FROM, url_observable) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() file_observable = root.get_observable(file_observable.id) self.assertIsNotNone(file_observable) analysis = file_observable.get_analysis(URLExtractionAnalysis) self.assertIsNotNone(analysis) self.assertEquals(len(analysis.get_observables_by_type(F_URL)), 10)
def test_live_browser_000_basic(self): """Basic test of LiveBrowserAnalysis.""" from saq.modules.url import CrawlphishAnalysisV2 from saq.modules.url import LiveBrowserAnalysis engine = AnalysisEngine() engine.enable_module('analysis_module_crawlphish') engine.enable_module('analysis_module_live_browser_analyzer') self.start_engine(engine) root = create_root_analysis() root.initialize_storage() url = root.add_observable(F_URL, 'http://localhost:{}/test_data/live_browser.000.html'.format(LOCAL_PORT)) url.add_directive(DIRECTIVE_CRAWL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() url = root.get_observable(url.id) analysis = url.get_analysis(CrawlphishAnalysisV2) file_observables = analysis.get_observables_by_type(F_FILE) self.assertEquals(len(file_observables), 1) file_observable = file_observables[0] analysis = file_observable.get_analysis(LiveBrowserAnalysis) file_observables = analysis.get_observables_by_type(F_FILE) self.assertEquals(len(file_observables), 1) file_observable = file_observables[0] self.assertEquals(file_observable.value, 'crawlphish/localhost_0/localhost_000.png')
def test_detections_000_ole(self): engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_archive') engine.enable_module('analysis_module_file_type') engine.enable_module('analysis_module_olevba_v1_1') engine.enable_module('analysis_module_officeparser_v1_0') engine.enable_module('analysis_module_yara_scanner_v3_4') self.start_engine(engine) submissions = {} # key = storage_dir, value = path to file for file_name in os.listdir(OFFICE_SAMPLES): source_path = os.path.join(OFFICE_SAMPLES, file_name) root = create_root_analysis(uuid=str(uuid.uuid4())) root.initialize_storage() shutil.copy(source_path, root.storage_dir) root.add_observable(F_FILE, file_name) root.save() submissions[root.storage_dir] = source_path engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) self.wait_engine(engine) for storage_dir in submissions: with self.subTest(storage_dir=storage_dir, source_path=submissions[storage_dir]): root = RootAnalysis() root.storage_dir = storage_dir root.load() detections = root.all_detection_points self.assertGreater(len(detections), 0)
def test_email_009_live_browser_no_render(self): # we usually render HTML attachments to emails # but not if it has a tag of "no_render" assigned by a yara rule engine = AnalysisEngine() engine.enable_module('analysis_module_file_type') engine.enable_module('analysis_module_email_analyzer') engine.enable_module('analysis_module_yara_scanner_v3_4') self.start_engine(engine) root = create_root_analysis(uuid=str(uuid.uuid4()), alert_type='mailbox') root.initialize_storage() shutil.copy( os.path.join('test_data', 'emails', 'phish_me.email.rfc822'), os.path.join(root.storage_dir, 'email.rfc822')) file_observable = root.add_observable(F_FILE, 'email.rfc822') file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() from saq.modules.email import EmailAnalysis file_observable = root.get_observable(file_observable.id) self.assertIsNotNone(file_observable) self.assertTrue(file_observable.has_tag('no_render')) from saq.modules.url import LiveBrowserAnalysis self.assertFalse(file_observable.get_analysis(LiveBrowserAnalysis))
def test_crawlphish_000_basic_download(self): from saq.modules.url import CrawlphishAnalysisV2 engine = AnalysisEngine() engine.enable_module('analysis_module_crawlphish') self.start_engine(engine) root = create_root_analysis() root.initialize_storage() url = root.add_observable(F_URL, 'http://localhost:{}/test_data/crawlphish.000'.format(LOCAL_PORT)) url.add_directive(DIRECTIVE_CRAWL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() url = root.get_observable(url.id) analysis = url.get_analysis(CrawlphishAnalysisV2) self.assertEquals(analysis.status_code, 200) self.assertEquals(analysis.file_name, 'crawlphish.000') self.assertTrue(analysis.downloaded) self.assertIsNone(analysis.error_reason) # there should be a single F_FILE observable file_observables = analysis.get_observables_by_type(F_FILE) self.assertEquals(len(file_observables), 1) file_observable = file_observables[0] self.assertTrue(file_observable.has_directive(DIRECTIVE_EXTRACT_URLS)) self.assertTrue(file_observable.has_relationship(R_DOWNLOADED_FROM))
def test_email_007_o365_journal_email_parsing(self): # parse an office365 journaled message engine = AnalysisEngine() engine.enable_module('analysis_module_file_type') engine.enable_module('analysis_module_email_analyzer') self.start_engine(engine) root = create_root_analysis(uuid=str(uuid.uuid4()), alert_type='mailbox') root.initialize_storage() shutil.copy( os.path.join('test_data', 'emails', 'o365_journaled.email.rfc822'), os.path.join(root.storage_dir, 'email.rfc822')) file_observable = root.add_observable(F_FILE, 'email.rfc822') file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() from saq.modules.email import EmailAnalysis file_observable = root.get_observable(file_observable.id) self.assertIsNotNone(file_observable) email_analysis = file_observable.get_analysis(EmailAnalysis) self.assertIsNotNone(email_analysis) self.assertIsNone(email_analysis.parsing_error) self.assertIsNotNone(email_analysis.email) self.assertIsNone(email_analysis.env_mail_from) self.assertTrue(isinstance(email_analysis.env_rcpt_to, list)) self.assertEquals(len(email_analysis.env_rcpt_to), 1) self.assertEquals(email_analysis.env_rcpt_to[0], '*****@*****.**') self.assertEquals(email_analysis.mail_from, 'Bobbie Fruitypie <*****@*****.**>') self.assertTrue(isinstance(email_analysis.mail_to, list)) self.assertEquals(len(email_analysis.mail_to), 1) self.assertEquals(email_analysis.mail_to[0], '<*****@*****.**>') self.assertIsNone(email_analysis.reply_to) self.assertEquals(email_analysis.subject, 'INVOICE PDL-06-38776') self.assertEquals(email_analysis.decoded_subject, email_analysis.subject) self.assertEquals( email_analysis.message_id, '<*****@*****.**>') self.assertIsNone(email_analysis.originating_ip, None) self.assertTrue(isinstance(email_analysis.received, list)) self.assertEquals(len(email_analysis.received), 7) self.assertTrue(isinstance(email_analysis.headers, list)) self.assertTrue(isinstance(email_analysis.log_entry, dict)) self.assertIsNone(email_analysis.x_mailer) self.assertIsNotNone(email_analysis.body) self.assertIsInstance(email_analysis.attachments, list) self.assertEquals(len(email_analysis.attachments), 0)
def test_email_006_basic_email_parsing(self): # parse a basic email message engine = AnalysisEngine() engine.enable_module('analysis_module_file_type') engine.enable_module('analysis_module_email_analyzer') self.start_engine(engine) root = create_root_analysis(uuid=str(uuid.uuid4()), alert_type='mailbox') root.initialize_storage() shutil.copy( os.path.join('test_data', 'emails', 'splunk_logging.email.rfc822'), os.path.join(root.storage_dir, 'email.rfc822')) file_observable = root.add_observable(F_FILE, 'email.rfc822') file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() from saq.modules.email import EmailAnalysis file_observable = root.get_observable(file_observable.id) self.assertIsNotNone(file_observable) email_analysis = file_observable.get_analysis(EmailAnalysis) self.assertIsNotNone(email_analysis) self.assertIsNone(email_analysis.parsing_error) self.assertIsNotNone(email_analysis.email) self.assertIsNone(email_analysis.env_mail_from) self.assertTrue(isinstance(email_analysis.env_rcpt_to, list)) self.assertEquals(len(email_analysis.env_rcpt_to), 1) self.assertEquals(email_analysis.env_rcpt_to[0], '*****@*****.**') self.assertEquals(email_analysis.mail_from, 'John Davison <*****@*****.**>') self.assertTrue(isinstance(email_analysis.mail_to, list)) self.assertEquals(len(email_analysis.mail_to), 1) self.assertEquals(email_analysis.mail_to[0], '*****@*****.**') self.assertIsNone(email_analysis.reply_to) self.assertEquals(email_analysis.subject, 'canary #3') self.assertEquals(email_analysis.decoded_subject, email_analysis.subject) self.assertEquals( email_analysis.message_id, '<CANTOGZsMiMb+7aB868zXSen_fO=NS-qFTUMo9h2eHtOexY8Qhw@mail.gmail.com>' ) self.assertIsNone(email_analysis.originating_ip, None) self.assertTrue(isinstance(email_analysis.received, list)) self.assertEquals(len(email_analysis.received), 6) self.assertTrue(isinstance(email_analysis.headers, list)) self.assertTrue(isinstance(email_analysis.log_entry, dict)) self.assertIsNone(email_analysis.x_mailer) self.assertIsNotNone(email_analysis.body) self.assertIsInstance(email_analysis.attachments, list) self.assertEquals(len(email_analysis.attachments), 0)
def test_email_000b_elk_logging(self): # clear elk logging directory elk_log_dir = os.path.join(saq.SAQ_HOME, saq.CONFIG['elk_logging']['elk_log_dir']) if os.path.isdir(elk_log_dir): shutil.rmtree(elk_log_dir) os.mkdir(elk_log_dir) engine = AnalysisEngine() engine.enable_module('analysis_module_file_type') engine.enable_module('analysis_module_email_analyzer') engine.enable_module('analysis_module_email_logger') engine.enable_module('analysis_module_url_extraction') self.start_engine(engine) root = create_root_analysis(alert_type='mailbox') root.initialize_storage() shutil.copy( os.path.join('test_data', 'emails', 'splunk_logging.email.rfc822'), os.path.join(root.storage_dir, 'email.rfc822')) file_observable = root.add_observable(F_FILE, 'email.rfc822') file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() wait_for_log_count('creating json logging directory ', 1, 5) entry = search_log('creating json logging directory ') target_dir = entry[0].getMessage( )[len('creating json logging directory '):] # we should expect three files in this directory now elk_files = [ os.path.join(target_dir, _) for _ in os.listdir(target_dir) ] self.assertEquals(len(elk_files), 1) with open(elk_files[0], 'r') as fp: log_entry = json.load(fp) for field in [ 'date', 'first_received', 'last_received', 'env_mail_from', 'env_rcpt_to', 'mail_from', 'mail_to', 'reply_to', 'cc', 'bcc', 'message_id', 'subject', 'path', 'size', 'user_agent', 'x_mailer', 'originating_ip', 'headers', 'attachment_count', 'attachment_sizes', 'attachment_types', 'attachment_names', 'attachment_hashes', 'thread_topic', 'thread_index', 'refereneces', 'x_sender' ]: self.assertTrue(field in log_entry)
def test_cloudphish_002_cached_entry(self): local_cache_dir = saq.CONFIG['analysis_module_cloudphish']['local_cache_dir'] shutil.rmtree(local_cache_dir) os.makedirs(local_cache_dir) self.start_gui_server() self.start_cloudphish_server() engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_cloudphish') self.start_engine(engine) root = create_root_analysis() url = root.add_observable(F_URL, 'http://www.valvoline.com/') self.assertIsNotNone(url) url.add_directive(DIRECTIVE_CRAWL) root.save() engine.queue_work_item(root.storage_dir) # wait for analysis to complete wait_for_log_count('executing post analysis on RootAnalysis({})'.format(root.uuid), 1) # we should NOT see cache results getting used here self.assertEquals(log_count('using local cache results for'), 0) # we should have a single cache entry p = Popen(['find', local_cache_dir, '-type', 'f'], stdout=PIPE, stderr=PIPE, universal_newlines=True) _stdout, _stderr = p.communicate() self.assertEquals(len(_stdout.strip().split('\n')), 1) from saq.cloudphish import hash_url sha2 = hash_url(url.value) target_path = os.path.join(local_cache_dir, sha2[:2], sha2) self.assertTrue(os.path.exists(target_path)) # now the second time we analyze we should see the cache get used root = create_root_analysis() url = root.add_observable(F_URL, 'http://www.valvoline.com/') self.assertIsNotNone(url) url.add_directive(DIRECTIVE_CRAWL) root.save() engine.queue_work_item(root.storage_dir) # wait for analysis to complete wait_for_log_count('executing post analysis on RootAnalysis({})'.format(root.uuid), 2) # we should see this now self.assertEquals(log_count('using local cache results for'), 1) engine.queue_work_item(TerminatingMarker()) engine.wait()
def test_cloudphish_001_invalid_fqdn(self): engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_cloudphish') self.start_engine(engine) root = create_root_analysis() url = root.add_observable(F_URL, 'http://invalid_domain/hello_world') self.assertIsNotNone(url) url.add_directive(DIRECTIVE_CRAWL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() self.assertEquals(log_count('ignoring invalid FQDN'), 1)
def test_cloudphish_000_invalid_scheme(self): engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_cloudphish') self.start_engine(engine) root = create_root_analysis() url = root.add_observable(F_URL, 'mailto:[email protected]') self.assertIsNotNone(url) url.add_directive(DIRECTIVE_CRAWL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() self.assertEquals(log_count('is not a supported scheme for cloudphish'), 1)
def test_protected_url_001_dropbox(self): engine = AnalysisEngine() engine.enable_module('analysis_module_protected_url_analyzer') self.start_engine(engine) root = create_root_analysis() root.initialize_storage() # taken from an actual sample url_with_dl0 = root.add_observable(F_URL, 'https://www.dropbox.com/s/ezdhsvdxf6wrxk6/RFQ-012018-000071984-13-Rev.1.zip?dl=0') url_with_dl1 = root.add_observable(F_URL, 'https://www.dropbox.com/s/ezdhsvdxf6wrxk6/RFQ-012018-000071984-13-Rev.1.zip?dl=1') url_without_dl = root.add_observable(F_URL, 'https://www.dropbox.com/s/ezdhsvdxf6wrxk6/RFQ-012018-000071984-13-Rev.1.zip') url_with_dl0.add_directive(DIRECTIVE_CRAWL) # not actually going to crawl, just testing that it gets copied over url_with_dl1.add_directive(DIRECTIVE_CRAWL) url_without_dl.add_directive(DIRECTIVE_CRAWL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() url_with_dl0 = root.get_observable(url_with_dl0.id) url_with_dl1 = root.get_observable(url_with_dl1.id) url_without_dl = root.get_observable(url_without_dl.id) from saq.modules.url import ProtectedURLAnalysis, PROTECTION_TYPE_DROPBOX analysis = url_with_dl0.get_analysis(ProtectedURLAnalysis) self.assertIsNotNone(analysis) self.assertEquals(analysis.protection_type, PROTECTION_TYPE_DROPBOX) self.assertEquals(analysis.extracted_url, 'https://www.dropbox.com/s/ezdhsvdxf6wrxk6/RFQ-012018-000071984-13-Rev.1.zip?dl=1') extracted_url = analysis.get_observables_by_type(F_URL) self.assertEquals(len(extracted_url), 1) extracted_url = extracted_url[0] self.assertTrue(extracted_url.has_directive(DIRECTIVE_CRAWL)) analysis = url_with_dl1.get_analysis(ProtectedURLAnalysis) self.assertFalse(analysis) analysis = url_without_dl.get_analysis(ProtectedURLAnalysis) self.assertIsNotNone(analysis) self.assertEquals(analysis.protection_type, PROTECTION_TYPE_DROPBOX) self.assertEquals(analysis.extracted_url, 'https://www.dropbox.com/s/ezdhsvdxf6wrxk6/RFQ-012018-000071984-13-Rev.1.zip?dl=1') extracted_url = analysis.get_observables_by_type(F_URL) self.assertEquals(len(extracted_url), 1) extracted_url = extracted_url[0] self.assertTrue(extracted_url.has_directive(DIRECTIVE_CRAWL))
def test_file_analysis_003_xml_000_rels(self): engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_archive') engine.enable_module('analysis_module_file_type') engine.enable_module('analysis_module_office_xml_rel') self.start_engine(engine) root = create_root_analysis(uuid=str(uuid.uuid4())) root.initialize_storage() shutil.copy('test_data/docx/xml_rel.docx', root.storage_dir) _file = root.add_observable(F_FILE, 'xml_rel.docx') root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() _file = root.get_observable(_file.id) from saq.modules.file_analysis import ArchiveAnalysis analysis = _file.get_analysis(ArchiveAnalysis) self.assertTrue(analysis) # there should be one file called document.xml.rels rel_file = None for sub_file in analysis.get_observables_by_type(F_FILE): if os.path.basename(sub_file.value) == 'document.xml.rels': rel_file = sub_file break self.assertIsNotNone(rel_file) from saq.modules.file_analysis import OfficeXMLRelationshipExternalURLAnalysis analysis = rel_file.get_analysis( OfficeXMLRelationshipExternalURLAnalysis) self.assertTrue(analysis) url = analysis.get_observables_by_type(F_URL) self.assertEquals(len(url), 1)
def test_file_analysis_004_yara_001_local_scan(self): # we do not initalize the local yss scanner so it should not be available for scanning engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_yara_scanner_v3_4') self.start_engine(engine) root = create_root_analysis(uuid=str(uuid.uuid4())) root.initialize_storage() shutil.copy('test_data/scan_targets/match', root.storage_dir) _file = root.add_observable(F_FILE, 'match') root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() self.assertEquals(log_count('with yss (matches found: True)'), 0) self.assertEquals(log_count('failed to connect to yara socket server'), 1) self.assertEquals(log_count('initializing local yara scanner'), 1) self.assertEquals(log_count('got yara results for'), 1) root.load() _file = root.get_observable(_file.id) from saq.modules.file_analysis import YaraScanResults_v3_4 analysis = _file.get_analysis(YaraScanResults_v3_4) self.assertTrue(analysis) # the file should be instructed to go to the sandbox self.assertTrue(_file.has_directive(DIRECTIVE_SANDBOX)) # and should have a single tag self.assertEquals(len(_file.tags), 1) # the analysis should have a yara_rule observable yara_rule = analysis.get_observables_by_type(F_YARA_RULE) self.assertEquals(len(yara_rule), 1) yara_rule = yara_rule[0] # the yara rule should have detections self.assertTrue(yara_rule.detections)
def test_email_005_message_id(self): # make sure we extract the correct message-id # this test email has an attachment that contains a message-id # we need to make sure we do not extract that one as the message-id observable engine = AnalysisEngine() engine.enable_module('analysis_module_file_type') engine.enable_module('analysis_module_email_analyzer') self.start_engine(engine) root = create_root_analysis(uuid=str(uuid.uuid4()), alert_type='mailbox') root.initialize_storage() shutil.copy( os.path.join('test_data', 'emails', 'extra_message_id.email.rfc822'), os.path.join(root.storage_dir, 'email.rfc822')) file_observable = root.add_observable(F_FILE, 'email.rfc822') file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() from saq.modules.email import EmailAnalysis file_observable = root.get_observable(file_observable.id) self.assertIsNotNone(file_observable) email_analysis = file_observable.get_analysis(EmailAnalysis) self.assertIsNotNone(email_analysis) message_id = email_analysis.get_observables_by_type(F_MESSAGE_ID) self.assertTrue(isinstance(message_id, list) and len(message_id) > 0) message_id = message_id[0] self.assertEquals( message_id.value, "<*****@*****.**>" )
def test_vx_001_file_lookup(self): engine = AnalysisEngine() engine.enable_module('analysis_module_vxstream_file_analyzer') engine.enable_module('analysis_module_vxstream_hash_analyzer') engine.enable_module('analysis_module_file_hash_analyzer') engine.enable_module('analysis_module_file_type') self.start_engine(engine) root = create_root_analysis(event_time=datetime.datetime.now()) root.initialize_storage() shutil.copy2('test_data/sample.jar', root.storage_dir) _file = root.add_observable(F_FILE, 'sample.jar') _file.add_directive(DIRECTIVE_SANDBOX) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() _file = root.get_observable(_file.id) from saq.modules.file_analysis import FileHashAnalysis from saq.modules.vx import VxStreamHashAnalysis hash_analysis = _file.get_analysis(FileHashAnalysis) self.assertIsNotNone(hash_analysis) sha2 = hash_analysis.get_observables_by_type(F_SHA256) self.assertIsInstance(sha2, list) self.assertEquals(len(sha2), 1) sha2 = sha2[0] analysis = sha2.get_analysis(VxStreamHashAnalysis) self.assertIsNotNone(analysis) self.assertEquals(analysis.sha256, sha2.value) self.assertEquals(analysis.environment_id, saq.CONFIG['vxstream']['environmentid']) self.assertEquals(analysis.status, VXSTREAM_STATUS_SUCCESS) self.assertIsNotNone(analysis.submit_date) self.assertIsNotNone(analysis.complete_date) self.assertIsNone(analysis.fail_date) self.assertIsNotNone(analysis.vxstream_threat_level) self.assertIsNotNone(analysis.vxstream_threat_score)
def test_email_008_whitelisting_001_mail_to(self): import saq whitelist_path = os.path.join('var', 'tmp', 'brotex.whitelist') saq.CONFIG['analysis_module_email_analyzer'][ 'whitelist_path'] = whitelist_path if os.path.exists(whitelist_path): os.remove(whitelist_path) with open(whitelist_path, 'w') as fp: fp.write('smtp_to:[email protected]') engine = AnalysisEngine() engine.enable_module('analysis_module_file_type') engine.enable_module('analysis_module_email_analyzer') self.start_engine(engine) root = create_root_analysis(uuid=str(uuid.uuid4()), alert_type='mailbox') root.initialize_storage() shutil.copy( os.path.join('test_data', 'emails', 'o365_journaled.email.rfc822'), os.path.join(root.storage_dir, 'email.rfc822')) file_observable = root.add_observable(F_FILE, 'email.rfc822') file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() from saq.modules.email import EmailAnalysis file_observable = root.get_observable(file_observable.id) self.assertIsNotNone(file_observable) email_analysis = file_observable.get_analysis(EmailAnalysis) self.assertFalse(email_analysis)
def test_file_analysis_000_url_extraction_001_pdfparser(self): engine = self.create_engine(AnalysisEngine) engine.enable_module('analysis_module_pdf_analyzer') engine.enable_module('analysis_module_url_extraction') engine.enable_module('analysis_module_file_type') self.start_engine(engine) root = create_root_analysis() root.initialize_storage() shutil.copy('test_data/pdf/Payment_Advice.pdf', root.storage_dir) target_file = 'Payment_Advice.pdf' file_observable = root.add_observable(F_FILE, target_file) root.save() engine.queue_work_item(root.storage_dir) engine.queue_work_item(TerminatingMarker()) engine.wait() root.load() file_observable = root.get_observable(file_observable.id) self.assertIsNotNone(file_observable) from saq.modules.file_analysis import URLExtractionAnalysis, PDFAnalysis pdf_analysis = file_observable.get_analysis(PDFAnalysis) self.assertIsNotNone(pdf_analysis) # should have a single file observable pdfparser_file = pdf_analysis.get_observables_by_type(F_FILE) self.assertEquals(len(pdfparser_file), 1) pdfparser_file = pdfparser_file[0] url_analysis = pdfparser_file.get_analysis(URLExtractionAnalysis) self.assertIsNotNone(url_analysis) # should have a bad url in it bad_url = 'http://www.williamtoms.com/wp-includes/354387473a/autodomain/autodomain/autodomain/autofil' self.assertTrue( bad_url in [url.value for url in url_analysis.get_observables_by_type(F_URL)])