def test_file_analysis_004_yara_002_no_alert(self):
self.initialize_yss()
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_yara_scanner_v3_4')
self.start_engine(engine)
root = create_root_analysis(uuid=str(uuid.uuid4()))
root.initialize_storage()
shutil.copy('test_data/scan_targets/no_alert', root.storage_dir)
_file = root.add_observable(F_FILE, 'no_alert')
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
self.assertEquals(log_count('with yss (matches found: True)'), 1)
root.load()
_file = root.get_observable(_file.id)
from saq.modules.file_analysis import YaraScanResults_v3_4
analysis = _file.get_analysis(YaraScanResults_v3_4)
self.assertTrue(analysis)
# the file should NOT be instructed to go to the sandbox
self.assertFalse(_file.has_directive(DIRECTIVE_SANDBOX))
# the analysis should have a yara_rule observable
yara_rule = analysis.get_observables_by_type(F_YARA_RULE)
self.assertEquals(len(yara_rule), 1)
yara_rule = yara_rule[0]
# the yara rule should NOT have detections
self.assertFalse(yara_rule.detections)
def test_carbon_black_asset_ident_000(self):
from saq.modules.asset import CarbonBlackAssetIdentAnalysis
# find an IP address in the past 24 hours to use
q, result = splunk_query(
"""index=carbonblack | dedup local_ip | head limit=1 | fields local_ip"""
)
self.assertTrue(result)
self.assertTrue(isinstance(q.json(), list))
self.assertEquals(len(q.json()), 1)
ipv4 = q.json()[0]['local_ip']
logging.info("using ipv4 {} for test".format(ipv4))
engine = AnalysisEngine()
engine.enable_module('analysis_module_carbon_black_asset_ident')
self.start_engine(engine)
root = create_root_analysis(event_time=datetime.datetime.now())
root.initialize_storage()
o_uuid = root.add_observable(F_IPV4, ipv4).id
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
ipv4 = root.get_observable(o_uuid)
self.assertIsNotNone(ipv4)
analysis = ipv4.get_analysis(CarbonBlackAssetIdentAnalysis)
self.assertIsNotNone(analysis)
self.assertIsNotNone(analysis.details)
self.assertEquals(len(analysis.discovered_hostnames), 1)
def test_crawlphish_001_download_404(self):
"""We should not extract URLs from data downloaded from URLs that returned a 404."""
from saq.modules.url import CrawlphishAnalysisV2
engine = AnalysisEngine()
engine.enable_module('analysis_module_crawlphish')
self.start_engine(engine)
root = create_root_analysis()
root.initialize_storage()
url = root.add_observable(F_URL, 'http://localhost:{}/test_data/crawlphish.001'.format(LOCAL_PORT))
url.add_directive(DIRECTIVE_CRAWL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
url = root.get_observable(url.id)
analysis = url.get_analysis(CrawlphishAnalysisV2)
self.assertEquals(analysis.proxy_results['GLOBAL'].status_code, 404)
if 'tor' in analysis.proxy_results:
self.assertIsNone(analysis.proxy_results['tor'].status_code)
self.assertIsNone(analysis.file_name) # no file should have been downloaded
self.assertFalse(analysis.downloaded)
self.assertIsNotNone(analysis.error_reason)
file_observables = analysis.get_observables_by_type(F_FILE)
self.assertEquals(len(file_observables), 0)
def test_protected_url_003_sharepoint(self):
engine = AnalysisEngine()
engine.enable_module('analysis_module_protected_url_analyzer')
self.start_engine(engine)
root = create_root_analysis()
root.initialize_storage()
# taken from an actual sample
url = root.add_observable(F_URL, 'https://lahia-my.sharepoint.com/:b:/g/personal/secure_onedrivemsw_bid/EVdjoBiqZTxMnjAcDW6yR4gBqJ59ALkT1C2I3L0yb_n0uQ?e=naeXYD')
url.add_directive(DIRECTIVE_CRAWL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
url = root.get_observable(url.id)
from saq.modules.url import ProtectedURLAnalysis, PROTECTION_TYPE_SHAREPOINT
analysis = url.get_analysis(ProtectedURLAnalysis)
self.assertIsNotNone(analysis)
self.assertEquals(analysis.protection_type, PROTECTION_TYPE_SHAREPOINT)
from urllib.parse import urlparse, parse_qs
parsed_url = urlparse(analysis.extracted_url)
self.assertEquals(parsed_url.path, '/personal/secure_onedrivemsw_bid/_layouts/15/download.aspx')
parsed_qs = parse_qs(parsed_url.query)
self.assertEquals(parsed_qs['e'][0], 'naeXYD')
self.assertEquals(parsed_qs['share'][0], 'EVdjoBiqZTxMnjAcDW6yR4gBqJ59ALkT1C2I3L0yb_n0uQ')
extracted_url = analysis.get_observables_by_type(F_URL)
self.assertEquals(len(extracted_url), 1)
extracted_url = extracted_url[0]
self.assertTrue(extracted_url.has_directive(DIRECTIVE_CRAWL))
def test_vx_000_hash_lookup(self):
engine = AnalysisEngine()
engine.enable_module('analysis_module_vxstream_hash_analyzer')
self.start_engine(engine)
root = create_root_analysis(event_time=datetime.datetime.now())
root.initialize_storage()
sha2 = root.add_observable(F_SHA256, SAMPLE_HASH)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
sha2 = root.get_observable(sha2.id)
from saq.modules.vx import VxStreamHashAnalysis
analysis = sha2.get_analysis(VxStreamHashAnalysis)
self.assertIsNotNone(analysis)
self.assertEquals(analysis.sha256, sha2.value)
self.assertEquals(analysis.environment_id,
saq.CONFIG['vxstream']['environmentid'])
self.assertEquals(analysis.status, VXSTREAM_STATUS_SUCCESS)
self.assertIsNotNone(analysis.submit_date)
self.assertIsNotNone(analysis.complete_date)
self.assertIsNone(analysis.fail_date)
self.assertIsNotNone(analysis.vxstream_threat_level)
self.assertIsNotNone(analysis.vxstream_threat_score)
def test_vx_003_file_with_hash_analysis(self):
engine = AnalysisEngine()
engine.enable_module('analysis_module_vxstream_hash_analyzer')
engine.enable_module('analysis_module_vxstream_file_analyzer')
engine.enable_module('analysis_module_file_hash_analyzer')
engine.enable_module('analysis_module_file_type')
self.start_engine(engine)
root = create_root_analysis(event_time=datetime.datetime.now())
root.initialize_storage()
with open('/dev/urandom', 'rb') as fp_in:
# using an extension here that doesn't get hash anlaysis
with open('test_data/invalid.pcap', 'wb') as fp_out:
fp_out.write(fp_in.read(4096))
shutil.copy('test_data/invalid.pcap', root.storage_dir)
_file = root.add_observable(F_FILE, 'invalid.pcap')
_file.add_directive(DIRECTIVE_SANDBOX)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
_file = root.get_observable(_file.id)
from saq.modules.vx import VxStreamFileAnalysis
analysis = _file.get_analysis(VxStreamFileAnalysis)
self.assertFalse(analysis)
def test_file_analysis_005_pcode_000_extract_pcode(self):
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_pcodedmp')
engine.enable_module('analysis_module_file_type')
self.start_engine(engine)
root = create_root_analysis(uuid=str(uuid.uuid4()))
root.initialize_storage()
shutil.copy('test_data/ole_files/word2013_macro_stripped.doc',
root.storage_dir)
_file = root.add_observable(F_FILE, 'word2013_macro_stripped.doc')
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
_file = root.get_observable(_file.id)
self.assertIsNotNone(_file)
from saq.modules.file_analysis import PCodeAnalysis
analysis = _file.get_analysis(PCodeAnalysis)
self.assertTrue(analysis)
# we should have extracted 11 lines of macro
self.assertEquals(analysis.details, 11)
# and we should have a file with the macros
_file = analysis.get_observables_by_type(F_FILE)
self.assertEquals(len(_file), 1)
_file = _file[0]
# and that should have a redirection
self.assertIsNotNone(_file.redirection)
def test_protected_url_000_outlook_safelinks(self):
engine = AnalysisEngine()
engine.enable_module('analysis_module_protected_url_analyzer')
self.start_engine(engine)
root = create_root_analysis()
root.initialize_storage()
# taken from an actual sample
url = root.add_observable(F_URL, 'https://na01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fwww.getbusinessready.com.au%2FInvoice-Number-49808%2F&data=02%7C01%7Ccyoung%40northernaviationservices.aero%7C8a388036cbf34f90ec5808d5724be7ed%7Cfc01978435d14339945c4161ac91c300%7C0%7C0%7C636540592704791165&sdata=%2FNQGqAp09WTNgnVnpoWIPcYNVAYsJ11ULuSS7cCsS3Q%3D&reserved=0')
url.add_directive(DIRECTIVE_CRAWL) # not actually going to crawl, just testing that it gets copied over
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
url = root.get_observable(url.id)
from saq.modules.url import ProtectedURLAnalysis, PROTECTION_TYPE_OUTLOOK_SAFELINKS
analysis = url.get_analysis(ProtectedURLAnalysis)
self.assertIsNotNone(analysis)
self.assertEquals(analysis.protection_type, PROTECTION_TYPE_OUTLOOK_SAFELINKS)
self.assertEquals(analysis.extracted_url, 'http://www.getbusinessready.com.au/Invoice-Number-49808/')
extracted_url = analysis.get_observables_by_type(F_URL)
self.assertEquals(len(extracted_url), 1)
extracted_url = extracted_url[0]
self.assertTrue(extracted_url.has_directive(DIRECTIVE_CRAWL))
def test_file_analysis_006_extracted_ole_000_js(self):
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_archive')
engine.enable_module('analysis_module_extracted_ole_analyzer')
engine.enable_module('analysis_module_officeparser_v1_0')
engine.enable_module('analysis_module_file_type')
self.start_engine(engine)
root = create_root_analysis()
root.initialize_storage()
shutil.copy('test_data/docx/js_ole_obj.docx', root.storage_dir)
_file = root.add_observable(F_FILE, 'js_ole_obj.docx')
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
_file = root.get_observable(_file.id)
self.assertIsNotNone(_file)
self.assertTrue(
any([
d for d in root.all_detection_points
if 'compiles as JavaScript' in d.description
]))
def test_live_browser_001_404(self):
"""We should not download screenshots for URLs that returned a 404 error message."""
from saq.modules.url import CrawlphishAnalysisV2
from saq.modules.url import LiveBrowserAnalysis
engine = AnalysisEngine()
engine.enable_module('analysis_module_crawlphish')
engine.enable_module('analysis_module_live_browser_analyzer')
self.start_engine(engine)
root = create_root_analysis()
root.initialize_storage()
# this file does not exist
url = root.add_observable(F_URL, 'http://localhost:{}/test_data/live_browser.dne.html'.format(LOCAL_PORT))
url.add_directive(DIRECTIVE_CRAWL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
url = root.get_observable(url.id)
analysis = url.get_analysis(CrawlphishAnalysisV2)
file_observables = analysis.get_observables_by_type(F_FILE)
self.assertEquals(len(file_observables), 0)
def test_file_analysis_002_archive_002_ace(self):
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_archive')
engine.enable_module('analysis_module_file_type')
self.start_engine(engine)
root = create_root_analysis(uuid=str(uuid.uuid4()))
root.initialize_storage()
shutil.copy('test_data/ace/dhl_report.ace', root.storage_dir)
_file = root.add_observable(F_FILE, 'dhl_report.ace')
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
_file = root.get_observable(_file.id)
from saq.modules.file_analysis import ArchiveAnalysis
analysis = _file.get_analysis(ArchiveAnalysis)
self.assertIsNotNone(analysis)
self.assertEquals(analysis.file_count, 1)
_file = analysis.get_observables_by_type(F_FILE)
self.assertEquals(len(_file), 1)
def test_protected_url_002_google_drive(self):
engine = AnalysisEngine()
engine.enable_module('analysis_module_protected_url_analyzer')
self.start_engine(engine)
root = create_root_analysis()
root.initialize_storage()
# taken from an actual sample
url = root.add_observable(F_URL, 'https://drive.google.com/file/d/1ls_eBCsmf3VG_e4dgQiSh_5VUM10b9s2/view')
url.add_directive(DIRECTIVE_CRAWL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
url = root.get_observable(url.id)
from saq.modules.url import ProtectedURLAnalysis, PROTECTION_TYPE_GOOGLE_DRIVE
analysis = url.get_analysis(ProtectedURLAnalysis)
self.assertIsNotNone(analysis)
self.assertEquals(analysis.protection_type, PROTECTION_TYPE_GOOGLE_DRIVE)
self.assertEquals(analysis.extracted_url, 'https://drive.google.com/uc?authuser=0&id=1ls_eBCsmf3VG_e4dgQiSh_5VUM10b9s2&export=download')
extracted_url = analysis.get_observables_by_type(F_URL)
self.assertEquals(len(extracted_url), 1)
extracted_url = extracted_url[0]
self.assertTrue(extracted_url.has_directive(DIRECTIVE_CRAWL))
def test_file_analysis_000_url_extraction_000_relative_html_urls(self):
from saq.modules.file_analysis import URLExtractionAnalysis
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_url_extraction')
engine.enable_module('analysis_module_file_type')
self.start_engine(engine)
root = create_root_analysis(event_time=datetime.datetime.now())
root.initialize_storage()
shutil.copy('test_data/url_extraction_000', root.storage_dir)
target_file = 'url_extraction_000'
src_url = 'https://vaishaligarden.com/.opjl/'
url_observable = root.add_observable(F_URL, src_url)
file_observable = root.add_observable(F_FILE, target_file)
file_observable.add_directive(DIRECTIVE_EXTRACT_URLS)
file_observable.add_relationship(R_DOWNLOADED_FROM, url_observable)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
file_observable = root.get_observable(file_observable.id)
self.assertIsNotNone(file_observable)
analysis = file_observable.get_analysis(URLExtractionAnalysis)
self.assertIsNotNone(analysis)
self.assertEquals(len(analysis.get_observables_by_type(F_URL)), 10)
def test_live_browser_000_basic(self):
"""Basic test of LiveBrowserAnalysis."""
from saq.modules.url import CrawlphishAnalysisV2
from saq.modules.url import LiveBrowserAnalysis
engine = AnalysisEngine()
engine.enable_module('analysis_module_crawlphish')
engine.enable_module('analysis_module_live_browser_analyzer')
self.start_engine(engine)
root = create_root_analysis()
root.initialize_storage()
url = root.add_observable(F_URL, 'http://localhost:{}/test_data/live_browser.000.html'.format(LOCAL_PORT))
url.add_directive(DIRECTIVE_CRAWL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
url = root.get_observable(url.id)
analysis = url.get_analysis(CrawlphishAnalysisV2)
file_observables = analysis.get_observables_by_type(F_FILE)
self.assertEquals(len(file_observables), 1)
file_observable = file_observables[0]
analysis = file_observable.get_analysis(LiveBrowserAnalysis)
file_observables = analysis.get_observables_by_type(F_FILE)
self.assertEquals(len(file_observables), 1)
file_observable = file_observables[0]
self.assertEquals(file_observable.value, 'crawlphish/localhost_0/localhost_000.png')
def test_detections_000_ole(self):
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_archive')
engine.enable_module('analysis_module_file_type')
engine.enable_module('analysis_module_olevba_v1_1')
engine.enable_module('analysis_module_officeparser_v1_0')
engine.enable_module('analysis_module_yara_scanner_v3_4')
self.start_engine(engine)
submissions = {} # key = storage_dir, value = path to file
for file_name in os.listdir(OFFICE_SAMPLES):
source_path = os.path.join(OFFICE_SAMPLES, file_name)
root = create_root_analysis(uuid=str(uuid.uuid4()))
root.initialize_storage()
shutil.copy(source_path, root.storage_dir)
root.add_observable(F_FILE, file_name)
root.save()
submissions[root.storage_dir] = source_path
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
self.wait_engine(engine)
for storage_dir in submissions:
with self.subTest(storage_dir=storage_dir,
source_path=submissions[storage_dir]):
root = RootAnalysis()
root.storage_dir = storage_dir
root.load()
detections = root.all_detection_points
self.assertGreater(len(detections), 0)
def test_email_009_live_browser_no_render(self):
# we usually render HTML attachments to emails
# but not if it has a tag of "no_render" assigned by a yara rule
engine = AnalysisEngine()
engine.enable_module('analysis_module_file_type')
engine.enable_module('analysis_module_email_analyzer')
engine.enable_module('analysis_module_yara_scanner_v3_4')
self.start_engine(engine)
root = create_root_analysis(uuid=str(uuid.uuid4()),
alert_type='mailbox')
root.initialize_storage()
shutil.copy(
os.path.join('test_data', 'emails', 'phish_me.email.rfc822'),
os.path.join(root.storage_dir, 'email.rfc822'))
file_observable = root.add_observable(F_FILE, 'email.rfc822')
file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
from saq.modules.email import EmailAnalysis
file_observable = root.get_observable(file_observable.id)
self.assertIsNotNone(file_observable)
self.assertTrue(file_observable.has_tag('no_render'))
from saq.modules.url import LiveBrowserAnalysis
self.assertFalse(file_observable.get_analysis(LiveBrowserAnalysis))
def test_crawlphish_000_basic_download(self):
from saq.modules.url import CrawlphishAnalysisV2
engine = AnalysisEngine()
engine.enable_module('analysis_module_crawlphish')
self.start_engine(engine)
root = create_root_analysis()
root.initialize_storage()
url = root.add_observable(F_URL, 'http://localhost:{}/test_data/crawlphish.000'.format(LOCAL_PORT))
url.add_directive(DIRECTIVE_CRAWL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
url = root.get_observable(url.id)
analysis = url.get_analysis(CrawlphishAnalysisV2)
self.assertEquals(analysis.status_code, 200)
self.assertEquals(analysis.file_name, 'crawlphish.000')
self.assertTrue(analysis.downloaded)
self.assertIsNone(analysis.error_reason)
# there should be a single F_FILE observable
file_observables = analysis.get_observables_by_type(F_FILE)
self.assertEquals(len(file_observables), 1)
file_observable = file_observables[0]
self.assertTrue(file_observable.has_directive(DIRECTIVE_EXTRACT_URLS))
self.assertTrue(file_observable.has_relationship(R_DOWNLOADED_FROM))
def test_email_007_o365_journal_email_parsing(self):
# parse an office365 journaled message
engine = AnalysisEngine()
engine.enable_module('analysis_module_file_type')
engine.enable_module('analysis_module_email_analyzer')
self.start_engine(engine)
root = create_root_analysis(uuid=str(uuid.uuid4()),
alert_type='mailbox')
root.initialize_storage()
shutil.copy(
os.path.join('test_data', 'emails', 'o365_journaled.email.rfc822'),
os.path.join(root.storage_dir, 'email.rfc822'))
file_observable = root.add_observable(F_FILE, 'email.rfc822')
file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
from saq.modules.email import EmailAnalysis
file_observable = root.get_observable(file_observable.id)
self.assertIsNotNone(file_observable)
email_analysis = file_observable.get_analysis(EmailAnalysis)
self.assertIsNotNone(email_analysis)
self.assertIsNone(email_analysis.parsing_error)
self.assertIsNotNone(email_analysis.email)
self.assertIsNone(email_analysis.env_mail_from)
self.assertTrue(isinstance(email_analysis.env_rcpt_to, list))
self.assertEquals(len(email_analysis.env_rcpt_to), 1)
self.assertEquals(email_analysis.env_rcpt_to[0],
'*****@*****.**')
self.assertEquals(email_analysis.mail_from,
'Bobbie Fruitypie <*****@*****.**>')
self.assertTrue(isinstance(email_analysis.mail_to, list))
self.assertEquals(len(email_analysis.mail_to), 1)
self.assertEquals(email_analysis.mail_to[0],
'<*****@*****.**>')
self.assertIsNone(email_analysis.reply_to)
self.assertEquals(email_analysis.subject, 'INVOICE PDL-06-38776')
self.assertEquals(email_analysis.decoded_subject,
email_analysis.subject)
self.assertEquals(
email_analysis.message_id,
'<*****@*****.**>')
self.assertIsNone(email_analysis.originating_ip, None)
self.assertTrue(isinstance(email_analysis.received, list))
self.assertEquals(len(email_analysis.received), 7)
self.assertTrue(isinstance(email_analysis.headers, list))
self.assertTrue(isinstance(email_analysis.log_entry, dict))
self.assertIsNone(email_analysis.x_mailer)
self.assertIsNotNone(email_analysis.body)
self.assertIsInstance(email_analysis.attachments, list)
self.assertEquals(len(email_analysis.attachments), 0)
def test_email_006_basic_email_parsing(self):
# parse a basic email message
engine = AnalysisEngine()
engine.enable_module('analysis_module_file_type')
engine.enable_module('analysis_module_email_analyzer')
self.start_engine(engine)
root = create_root_analysis(uuid=str(uuid.uuid4()),
alert_type='mailbox')
root.initialize_storage()
shutil.copy(
os.path.join('test_data', 'emails', 'splunk_logging.email.rfc822'),
os.path.join(root.storage_dir, 'email.rfc822'))
file_observable = root.add_observable(F_FILE, 'email.rfc822')
file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
from saq.modules.email import EmailAnalysis
file_observable = root.get_observable(file_observable.id)
self.assertIsNotNone(file_observable)
email_analysis = file_observable.get_analysis(EmailAnalysis)
self.assertIsNotNone(email_analysis)
self.assertIsNone(email_analysis.parsing_error)
self.assertIsNotNone(email_analysis.email)
self.assertIsNone(email_analysis.env_mail_from)
self.assertTrue(isinstance(email_analysis.env_rcpt_to, list))
self.assertEquals(len(email_analysis.env_rcpt_to), 1)
self.assertEquals(email_analysis.env_rcpt_to[0],
'*****@*****.**')
self.assertEquals(email_analysis.mail_from,
'John Davison <*****@*****.**>')
self.assertTrue(isinstance(email_analysis.mail_to, list))
self.assertEquals(len(email_analysis.mail_to), 1)
self.assertEquals(email_analysis.mail_to[0], '*****@*****.**')
self.assertIsNone(email_analysis.reply_to)
self.assertEquals(email_analysis.subject, 'canary #3')
self.assertEquals(email_analysis.decoded_subject,
email_analysis.subject)
self.assertEquals(
email_analysis.message_id,
'<CANTOGZsMiMb+7aB868zXSen_fO=NS-qFTUMo9h2eHtOexY8Qhw@mail.gmail.com>'
)
self.assertIsNone(email_analysis.originating_ip, None)
self.assertTrue(isinstance(email_analysis.received, list))
self.assertEquals(len(email_analysis.received), 6)
self.assertTrue(isinstance(email_analysis.headers, list))
self.assertTrue(isinstance(email_analysis.log_entry, dict))
self.assertIsNone(email_analysis.x_mailer)
self.assertIsNotNone(email_analysis.body)
self.assertIsInstance(email_analysis.attachments, list)
self.assertEquals(len(email_analysis.attachments), 0)
def test_email_000b_elk_logging(self):
# clear elk logging directory
elk_log_dir = os.path.join(saq.SAQ_HOME,
saq.CONFIG['elk_logging']['elk_log_dir'])
if os.path.isdir(elk_log_dir):
shutil.rmtree(elk_log_dir)
os.mkdir(elk_log_dir)
engine = AnalysisEngine()
engine.enable_module('analysis_module_file_type')
engine.enable_module('analysis_module_email_analyzer')
engine.enable_module('analysis_module_email_logger')
engine.enable_module('analysis_module_url_extraction')
self.start_engine(engine)
root = create_root_analysis(alert_type='mailbox')
root.initialize_storage()
shutil.copy(
os.path.join('test_data', 'emails', 'splunk_logging.email.rfc822'),
os.path.join(root.storage_dir, 'email.rfc822'))
file_observable = root.add_observable(F_FILE, 'email.rfc822')
file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
wait_for_log_count('creating json logging directory ', 1, 5)
entry = search_log('creating json logging directory ')
target_dir = entry[0].getMessage(
)[len('creating json logging directory '):]
# we should expect three files in this directory now
elk_files = [
os.path.join(target_dir, _) for _ in os.listdir(target_dir)
]
self.assertEquals(len(elk_files), 1)
with open(elk_files[0], 'r') as fp:
log_entry = json.load(fp)
for field in [
'date', 'first_received', 'last_received', 'env_mail_from',
'env_rcpt_to', 'mail_from', 'mail_to', 'reply_to', 'cc', 'bcc',
'message_id', 'subject', 'path', 'size', 'user_agent',
'x_mailer', 'originating_ip', 'headers', 'attachment_count',
'attachment_sizes', 'attachment_types', 'attachment_names',
'attachment_hashes', 'thread_topic', 'thread_index',
'refereneces', 'x_sender'
]:
self.assertTrue(field in log_entry)
def test_cloudphish_002_cached_entry(self):
local_cache_dir = saq.CONFIG['analysis_module_cloudphish']['local_cache_dir']
shutil.rmtree(local_cache_dir)
os.makedirs(local_cache_dir)
self.start_gui_server()
self.start_cloudphish_server()
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_cloudphish')
self.start_engine(engine)
root = create_root_analysis()
url = root.add_observable(F_URL, 'http://www.valvoline.com/')
self.assertIsNotNone(url)
url.add_directive(DIRECTIVE_CRAWL)
root.save()
engine.queue_work_item(root.storage_dir)
# wait for analysis to complete
wait_for_log_count('executing post analysis on RootAnalysis({})'.format(root.uuid), 1)
# we should NOT see cache results getting used here
self.assertEquals(log_count('using local cache results for'), 0)
# we should have a single cache entry
p = Popen(['find', local_cache_dir, '-type', 'f'], stdout=PIPE, stderr=PIPE, universal_newlines=True)
_stdout, _stderr = p.communicate()
self.assertEquals(len(_stdout.strip().split('\n')), 1)
from saq.cloudphish import hash_url
sha2 = hash_url(url.value)
target_path = os.path.join(local_cache_dir, sha2[:2], sha2)
self.assertTrue(os.path.exists(target_path))
# now the second time we analyze we should see the cache get used
root = create_root_analysis()
url = root.add_observable(F_URL, 'http://www.valvoline.com/')
self.assertIsNotNone(url)
url.add_directive(DIRECTIVE_CRAWL)
root.save()
engine.queue_work_item(root.storage_dir)
# wait for analysis to complete
wait_for_log_count('executing post analysis on RootAnalysis({})'.format(root.uuid), 2)
# we should see this now
self.assertEquals(log_count('using local cache results for'), 1)
engine.queue_work_item(TerminatingMarker())
engine.wait()
def test_cloudphish_001_invalid_fqdn(self):
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_cloudphish')
self.start_engine(engine)
root = create_root_analysis()
url = root.add_observable(F_URL, 'http://invalid_domain/hello_world')
self.assertIsNotNone(url)
url.add_directive(DIRECTIVE_CRAWL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
self.assertEquals(log_count('ignoring invalid FQDN'), 1)
def test_cloudphish_000_invalid_scheme(self):
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_cloudphish')
self.start_engine(engine)
root = create_root_analysis()
url = root.add_observable(F_URL, 'mailto:[email protected]')
self.assertIsNotNone(url)
url.add_directive(DIRECTIVE_CRAWL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
self.assertEquals(log_count('is not a supported scheme for cloudphish'), 1)
def test_protected_url_001_dropbox(self):
engine = AnalysisEngine()
engine.enable_module('analysis_module_protected_url_analyzer')
self.start_engine(engine)
root = create_root_analysis()
root.initialize_storage()
# taken from an actual sample
url_with_dl0 = root.add_observable(F_URL, 'https://www.dropbox.com/s/ezdhsvdxf6wrxk6/RFQ-012018-000071984-13-Rev.1.zip?dl=0')
url_with_dl1 = root.add_observable(F_URL, 'https://www.dropbox.com/s/ezdhsvdxf6wrxk6/RFQ-012018-000071984-13-Rev.1.zip?dl=1')
url_without_dl = root.add_observable(F_URL, 'https://www.dropbox.com/s/ezdhsvdxf6wrxk6/RFQ-012018-000071984-13-Rev.1.zip')
url_with_dl0.add_directive(DIRECTIVE_CRAWL) # not actually going to crawl, just testing that it gets copied over
url_with_dl1.add_directive(DIRECTIVE_CRAWL)
url_without_dl.add_directive(DIRECTIVE_CRAWL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
url_with_dl0 = root.get_observable(url_with_dl0.id)
url_with_dl1 = root.get_observable(url_with_dl1.id)
url_without_dl = root.get_observable(url_without_dl.id)
from saq.modules.url import ProtectedURLAnalysis, PROTECTION_TYPE_DROPBOX
analysis = url_with_dl0.get_analysis(ProtectedURLAnalysis)
self.assertIsNotNone(analysis)
self.assertEquals(analysis.protection_type, PROTECTION_TYPE_DROPBOX)
self.assertEquals(analysis.extracted_url, 'https://www.dropbox.com/s/ezdhsvdxf6wrxk6/RFQ-012018-000071984-13-Rev.1.zip?dl=1')
extracted_url = analysis.get_observables_by_type(F_URL)
self.assertEquals(len(extracted_url), 1)
extracted_url = extracted_url[0]
self.assertTrue(extracted_url.has_directive(DIRECTIVE_CRAWL))
analysis = url_with_dl1.get_analysis(ProtectedURLAnalysis)
self.assertFalse(analysis)
analysis = url_without_dl.get_analysis(ProtectedURLAnalysis)
self.assertIsNotNone(analysis)
self.assertEquals(analysis.protection_type, PROTECTION_TYPE_DROPBOX)
self.assertEquals(analysis.extracted_url, 'https://www.dropbox.com/s/ezdhsvdxf6wrxk6/RFQ-012018-000071984-13-Rev.1.zip?dl=1')
extracted_url = analysis.get_observables_by_type(F_URL)
self.assertEquals(len(extracted_url), 1)
extracted_url = extracted_url[0]
self.assertTrue(extracted_url.has_directive(DIRECTIVE_CRAWL))
def test_file_analysis_003_xml_000_rels(self):
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_archive')
engine.enable_module('analysis_module_file_type')
engine.enable_module('analysis_module_office_xml_rel')
self.start_engine(engine)
root = create_root_analysis(uuid=str(uuid.uuid4()))
root.initialize_storage()
shutil.copy('test_data/docx/xml_rel.docx', root.storage_dir)
_file = root.add_observable(F_FILE, 'xml_rel.docx')
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
_file = root.get_observable(_file.id)
from saq.modules.file_analysis import ArchiveAnalysis
analysis = _file.get_analysis(ArchiveAnalysis)
self.assertTrue(analysis)
# there should be one file called document.xml.rels
rel_file = None
for sub_file in analysis.get_observables_by_type(F_FILE):
if os.path.basename(sub_file.value) == 'document.xml.rels':
rel_file = sub_file
break
self.assertIsNotNone(rel_file)
from saq.modules.file_analysis import OfficeXMLRelationshipExternalURLAnalysis
analysis = rel_file.get_analysis(
OfficeXMLRelationshipExternalURLAnalysis)
self.assertTrue(analysis)
url = analysis.get_observables_by_type(F_URL)
self.assertEquals(len(url), 1)
def test_file_analysis_004_yara_001_local_scan(self):
# we do not initalize the local yss scanner so it should not be available for scanning
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_yara_scanner_v3_4')
self.start_engine(engine)
root = create_root_analysis(uuid=str(uuid.uuid4()))
root.initialize_storage()
shutil.copy('test_data/scan_targets/match', root.storage_dir)
_file = root.add_observable(F_FILE, 'match')
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
self.assertEquals(log_count('with yss (matches found: True)'), 0)
self.assertEquals(log_count('failed to connect to yara socket server'),
1)
self.assertEquals(log_count('initializing local yara scanner'), 1)
self.assertEquals(log_count('got yara results for'), 1)
root.load()
_file = root.get_observable(_file.id)
from saq.modules.file_analysis import YaraScanResults_v3_4
analysis = _file.get_analysis(YaraScanResults_v3_4)
self.assertTrue(analysis)
# the file should be instructed to go to the sandbox
self.assertTrue(_file.has_directive(DIRECTIVE_SANDBOX))
# and should have a single tag
self.assertEquals(len(_file.tags), 1)
# the analysis should have a yara_rule observable
yara_rule = analysis.get_observables_by_type(F_YARA_RULE)
self.assertEquals(len(yara_rule), 1)
yara_rule = yara_rule[0]
# the yara rule should have detections
self.assertTrue(yara_rule.detections)
def test_email_005_message_id(self):
# make sure we extract the correct message-id
# this test email has an attachment that contains a message-id
# we need to make sure we do not extract that one as the message-id observable
engine = AnalysisEngine()
engine.enable_module('analysis_module_file_type')
engine.enable_module('analysis_module_email_analyzer')
self.start_engine(engine)
root = create_root_analysis(uuid=str(uuid.uuid4()),
alert_type='mailbox')
root.initialize_storage()
shutil.copy(
os.path.join('test_data', 'emails',
'extra_message_id.email.rfc822'),
os.path.join(root.storage_dir, 'email.rfc822'))
file_observable = root.add_observable(F_FILE, 'email.rfc822')
file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
from saq.modules.email import EmailAnalysis
file_observable = root.get_observable(file_observable.id)
self.assertIsNotNone(file_observable)
email_analysis = file_observable.get_analysis(EmailAnalysis)
self.assertIsNotNone(email_analysis)
message_id = email_analysis.get_observables_by_type(F_MESSAGE_ID)
self.assertTrue(isinstance(message_id, list) and len(message_id) > 0)
message_id = message_id[0]
self.assertEquals(
message_id.value,
"<*****@*****.**>"
)
def test_vx_001_file_lookup(self):
engine = AnalysisEngine()
engine.enable_module('analysis_module_vxstream_file_analyzer')
engine.enable_module('analysis_module_vxstream_hash_analyzer')
engine.enable_module('analysis_module_file_hash_analyzer')
engine.enable_module('analysis_module_file_type')
self.start_engine(engine)
root = create_root_analysis(event_time=datetime.datetime.now())
root.initialize_storage()
shutil.copy2('test_data/sample.jar', root.storage_dir)
_file = root.add_observable(F_FILE, 'sample.jar')
_file.add_directive(DIRECTIVE_SANDBOX)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
_file = root.get_observable(_file.id)
from saq.modules.file_analysis import FileHashAnalysis
from saq.modules.vx import VxStreamHashAnalysis
hash_analysis = _file.get_analysis(FileHashAnalysis)
self.assertIsNotNone(hash_analysis)
sha2 = hash_analysis.get_observables_by_type(F_SHA256)
self.assertIsInstance(sha2, list)
self.assertEquals(len(sha2), 1)
sha2 = sha2[0]
analysis = sha2.get_analysis(VxStreamHashAnalysis)
self.assertIsNotNone(analysis)
self.assertEquals(analysis.sha256, sha2.value)
self.assertEquals(analysis.environment_id,
saq.CONFIG['vxstream']['environmentid'])
self.assertEquals(analysis.status, VXSTREAM_STATUS_SUCCESS)
self.assertIsNotNone(analysis.submit_date)
self.assertIsNotNone(analysis.complete_date)
self.assertIsNone(analysis.fail_date)
self.assertIsNotNone(analysis.vxstream_threat_level)
self.assertIsNotNone(analysis.vxstream_threat_score)
def test_email_008_whitelisting_001_mail_to(self):
import saq
whitelist_path = os.path.join('var', 'tmp', 'brotex.whitelist')
saq.CONFIG['analysis_module_email_analyzer'][
'whitelist_path'] = whitelist_path
if os.path.exists(whitelist_path):
os.remove(whitelist_path)
with open(whitelist_path, 'w') as fp:
fp.write('smtp_to:[email protected]')
engine = AnalysisEngine()
engine.enable_module('analysis_module_file_type')
engine.enable_module('analysis_module_email_analyzer')
self.start_engine(engine)
root = create_root_analysis(uuid=str(uuid.uuid4()),
alert_type='mailbox')
root.initialize_storage()
shutil.copy(
os.path.join('test_data', 'emails', 'o365_journaled.email.rfc822'),
os.path.join(root.storage_dir, 'email.rfc822'))
file_observable = root.add_observable(F_FILE, 'email.rfc822')
file_observable.add_directive(DIRECTIVE_ORIGINAL_EMAIL)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
from saq.modules.email import EmailAnalysis
file_observable = root.get_observable(file_observable.id)
self.assertIsNotNone(file_observable)
email_analysis = file_observable.get_analysis(EmailAnalysis)
self.assertFalse(email_analysis)
def test_file_analysis_000_url_extraction_001_pdfparser(self):
engine = self.create_engine(AnalysisEngine)
engine.enable_module('analysis_module_pdf_analyzer')
engine.enable_module('analysis_module_url_extraction')
engine.enable_module('analysis_module_file_type')
self.start_engine(engine)
root = create_root_analysis()
root.initialize_storage()
shutil.copy('test_data/pdf/Payment_Advice.pdf', root.storage_dir)
target_file = 'Payment_Advice.pdf'
file_observable = root.add_observable(F_FILE, target_file)
root.save()
engine.queue_work_item(root.storage_dir)
engine.queue_work_item(TerminatingMarker())
engine.wait()
root.load()
file_observable = root.get_observable(file_observable.id)
self.assertIsNotNone(file_observable)
from saq.modules.file_analysis import URLExtractionAnalysis, PDFAnalysis
pdf_analysis = file_observable.get_analysis(PDFAnalysis)
self.assertIsNotNone(pdf_analysis)
# should have a single file observable
pdfparser_file = pdf_analysis.get_observables_by_type(F_FILE)
self.assertEquals(len(pdfparser_file), 1)
pdfparser_file = pdfparser_file[0]
url_analysis = pdfparser_file.get_analysis(URLExtractionAnalysis)
self.assertIsNotNone(url_analysis)
# should have a bad url in it
bad_url = 'http://www.williamtoms.com/wp-includes/354387473a/autodomain/autodomain/autodomain/autofil'
self.assertTrue(
bad_url in
[url.value for url in url_analysis.get_observables_by_type(F_URL)])
