async def test_scan_notnil(self) -> None: s = Stoq(plugin_dir_list=[self.plugin_dir]) plugin = s.load_plugin(self.plugin_name) payload = Payload(self.entropy_not_nil) response = await plugin.scan(payload, RequestMeta()) self.assertIsInstance(response, WorkerResponse) self.assertEqual(1.584962500721156, response.results['entropy'])
def test_archiver_in_results(self): s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['simple_archiver']) response = s.scan( self.generic_content, request_meta=RequestMeta(archive_payloads=True) ) self.assertIn('simple_archiver', response.results[0].archivers) self.assertIn('file_save_id', response.results[0].archivers['simple_archiver'])
def test_dont_dest_archive_yara(self): s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['dummy_archiver']) response = s.scan( self.generic_content, request_meta=RequestMeta(archive_payloads=True) ) # The yara rule 'similar_simple_rule' should set save = False self.assertNotIn('dummy_archiver', response.results[0].archivers)
def test_scan(self) -> None: s = Stoq(plugin_dir_list=[self.plugin_dir]) plugin = s.load_plugin(self.plugin_name) payload = Payload(self.generic_data) response = plugin.scan(payload, RequestMeta()) self.assertIsInstance(response, WorkerResponse) self.assertEqual('3:hMCE7pr3Kn:huJ6', response.results['ssdeep'])
def test_dispatcher(self) -> None: s = Stoq( plugin_dir_list=[self.plugin_dir], plugin_opts={ self.plugin_name: { 'dispatch_rules': f'{self.data_dir}/dispatch_rules.yar' } }, ) plugin = s.load_plugin(self.plugin_name) payload = Payload(self.generic_data) response = plugin.get_dispatches(payload, RequestMeta()) self.assertIsInstance(response, DispatcherResponse) self.assertIn('test_dispatch_plugin', response.plugin_names) self.assertEqual( 'test_dispatch_rule', response.meta['test_dispatch_plugin']['rule'] ) self.assertIn( 'test_dispatch_plugin', response.meta['test_dispatch_plugin']['meta']['plugin'], ) self.assertIn('True', response.meta['test_dispatch_plugin']['meta']['save']) self.assertEqual( ['tag1', 'tag2'], response.meta['test_dispatch_plugin']['tags'] )
def test_scan(self) -> None: s = Stoq(plugin_dir_list=[self.plugin_dir]) plugin = s.load_plugin(self.plugin_name) payload = Payload(self.generic_data) response = plugin.scan(payload, RequestMeta()) self.assertIsInstance(response, WorkerResponse) self.assertEqual('text/plain', response.results['mimetype'])
async def test_scan(self) -> None: s = Stoq(plugin_dir_list=[str(self.plugin_dir)]) plugin = s.load_plugin(self.plugin_name) payload = Payload(base64.b64encode(self.generic_data)) response = await plugin.scan(payload, RequestMeta()) self.assertIsInstance(response, WorkerResponse) self.assertEqual(1, len(response.extracted)) self.assertEqual(self.generic_data, response.extracted[0].content)
def test_scan(self) -> None: s = Stoq(plugin_dir_list=[self.plugin_dir]) plugin = s.load_plugin(self.plugin_name) with open(f'{self.data_dir}/sample.pdf', 'rb') as f: payload = Payload(f.read()) response = plugin.scan(payload, RequestMeta()) self.assertIsInstance(response, WorkerResponse) self.assertIn('FileType', response.results) self.assertEqual('PDF', response.results['FileType']) self.assertEqual(6, response.results['PageCount'])
def test_dest_archive(self): s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['dummy_archiver']) dummy_archiver = s.load_plugin('dummy_archiver') dummy_archiver.archive = create_autospec( dummy_archiver.archive, return_value=None ) response = s.scan( self.generic_content, request_meta=RequestMeta(archive_payloads=True) ) dummy_archiver.archive.assert_called_once() self.assertIn('dummy_archiver', response.results[0].plugins_run['archivers'])
def test_scan(self) -> None: s = Stoq(plugin_dir_list=[self.plugin_dir]) plugin = s.load_plugin(self.plugin_name) with open(f'{self.data_dir}/TestJavaClass.class', 'rb') as f: payload = Payload(f.read()) response = plugin.scan(payload, RequestMeta()) self.assertIsInstance(response, WorkerResponse) self.assertIn('TestJavaClass', response.results['provided']) self.assertGreaterEqual(len(response.results['provided']), 4) self.assertGreaterEqual(len(response.results['required']), 2) self.assertGreaterEqual(len(response.results['constants']), 10)
def test_scan(self) -> None: s = Stoq(plugin_dir_list=[self.plugin_dir]) plugin = s.load_plugin(self.plugin_name) xord = bytes(x ^ 92 for x in self.generic_data) payload = Payload(xord) response = plugin.scan(payload, RequestMeta()) self.assertIsInstance(response, WorkerResponse) self.assertIn('0x5C', response.results) self.assertEqual( 'AdjustTokenPrivileges CurrentVersion', response.results['0x5C'][0]['match'] ) self.assertEqual('CurrentVersion', response.results['0x5C'][1]['match'])
async def test_scan_xor_single_value(self) -> None: s = Stoq(plugin_dir_list=[self.plugin_dir]) plugin = s.load_plugin(self.plugin_name) payload = Payload(self._xor_encode(self.generic_data, self.xorkeys)) dispatch_meta = {'test': {'test': {'meta': {'xorkey': self.xorkeys}}}} payload.dispatch_meta = dispatch_meta response = await plugin.scan(payload, RequestMeta()) self.assertIsInstance(response, WorkerResponse) self.assertEqual(self.generic_data, response.extracted[0].content) self.assertEqual( self.xorkeys, response.extracted[0].payload_meta.extra_data['xorkey'])
def test_dont_dest_archive_request(self): s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['dummy_archiver']) dummy_archiver = s.load_plugin('dummy_archiver') dummy_archiver.archive = Mock(return_value=None) response = s.scan( self.generic_content, add_start_dispatch=['extract_random'], request_meta=RequestMeta(archive_payloads=False), ) dummy_archiver.archive.assert_not_called() self.assertNotIn('dummy_archiver', response.results[0].plugins_run['archivers']) self.assertNotIn('dummy_archiver', response.results[1].plugins_run['archivers'])
def test_scan(self) -> None: s = Stoq( plugin_dir_list=[self.plugin_dir], plugin_opts={ self.plugin_name: {'worker_rules': f'{self.data_dir}/scan_rules.yar'} }, ) plugin = s.load_plugin(self.plugin_name) payload = Payload(self.generic_data) response = plugin.scan(payload, RequestMeta()) self.assertIsInstance(response, WorkerResponse) self.assertEqual('test_scan_rule', response.results['matches'][0]['rule'])
def test_dispatcher_save_false(self) -> None: s = Stoq( plugin_dir_list=[self.plugin_dir], plugin_opts={ self.plugin_name: { 'dispatch_rules': f'{self.data_dir}/dispatch_rules.yar' } }, ) plugin = s.load_plugin(self.plugin_name) payload = Payload(b'save_false') response = plugin.get_dispatches(payload, RequestMeta()) self.assertIsInstance(response, DispatcherResponse) self.assertIn('False', response.meta['save_false']['meta']['save'])
def test_scan(self) -> None: s = Stoq(plugin_dir_list=[self.plugin_dir]) plugin = s.load_plugin(self.plugin_name) payload = Payload(self.generic_data) response = plugin.scan(payload, RequestMeta()) self.assertIsInstance(response, WorkerResponse) self.assertEqual('cfe671457bc475ef2f51cf12b1457475', response.results['md5']) self.assertEqual('f610f70b1464d97f7897fefd6420ffc904df5e4f', response.results['sha1']) self.assertEqual( '2fa284e62b11fea1226b35cdd726a7a56090853ed135240665ceb3939f631af7', response.results['sha256'], )
def test_dont_dest_archive_payload(self): s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['dummy_archiver']) dummy_archiver = s.load_plugin('dummy_archiver') dummy_archiver.archive = create_autospec( dummy_archiver.archive, return_value=None ) response = s.scan( self.generic_content, payload_meta=PayloadMeta(should_archive=False), add_start_dispatch=['extract_random'], request_meta=RequestMeta(archive_payloads=True), ) dummy_archiver.archive.assert_called_once() self.assertNotIn('dummy_archiver', response.results[0].plugins_run['archivers']) self.assertIn('dummy_archiver', response.results[1].plugins_run['archivers'])
def test_scan_meta_bytes(self) -> None: s = Stoq( plugin_dir_list=[self.plugin_dir], plugin_opts={ self.plugin_name: {'worker_rules': f'{self.data_dir}/scan_rules.yar'} }, ) plugin = s.load_plugin(self.plugin_name) payload = Payload(b'meta_bytes') response = plugin.scan(payload, RequestMeta()) self.assertIsInstance(response, WorkerResponse) self.assertEqual( 'test_scan_metadata_bytes', response.results['matches'][0]['rule'] ) self.assertEqual('ANeato', response.results['matches'][0]['meta']['bytes']) self.assertEqual( 'Peter Rabbit', response.results['matches'][0]['meta']['author'] ) self.assertEqual('save_false', response.results['matches'][0]['meta']['plugin'])
def test_reconstruct_all_subresponses(self): # Construct a fake stoq_response as if it were generated from a file # A.zip that contains two files, B.txt and C.zip, where C.zip contains D.txt initial_response = StoqResponse( results=[ PayloadResults( payload_id="A.zip", size=0, payload_meta=PayloadMeta(), workers=[{"fake": "result1"}], plugins_run={"workers": [["fake"]]}, ), PayloadResults( payload_id="B.txt", size=0, payload_meta=PayloadMeta(), workers=[{"fake": "result2"}], plugins_run={"workers": [["fake"]]}, extracted_from="A.zip", extracted_by="fake", ), PayloadResults( payload_id="C.zip", size=0, payload_meta=PayloadMeta(), workers=[{"fake": "result3"}], plugins_run={"workers": [["fake"]]}, extracted_from="A.zip", extracted_by="fake", ), PayloadResults( payload_id="D.txt", size=0, payload_meta=PayloadMeta(), workers=[{"fake": "result4"}], plugins_run={"workers": [["fake"]]}, extracted_from="C.zip", extracted_by="fake", ), ], request_meta=RequestMeta(extra_data={"check": "me"}), errors={}, ) s = Stoq(base_dir=utils.get_data_dir(), decorators=["simple_decorator"]) all_subresponses = list(s.reconstruct_all_subresponses(initial_response)) # We expect there to be four "artificial" responses generated, one for # each payload as the root. self.assertEqual(len(all_subresponses), 4) # We expect the first response to have all 4 payloads, the second response # to have just the second payload, the third response to have the third # and fourth payload, and the fourth response to have just the fourth payload self.assertEqual( [len(stoq_response.results) for stoq_response in all_subresponses], [4, 1, 2, 1] ) self.assertEqual( [ stoq_response.results[0].workers[0]["fake"] for stoq_response in all_subresponses ], ["result1", "result2", "result3", "result4"], ) self.assertTrue( all( "simple_decorator" in stoq_response.decorators for stoq_response in all_subresponses ) ) # Assert that they all have the same scan ID self.assertEqual( len({stoq_response.scan_id for stoq_response in all_subresponses}), 1 )
def test_requestmeta_to_str(self): response = RequestMeta() response_str = str(response) response_dict = json.loads(response_str) self.assertIsInstance(response_str, str) self.assertIsInstance(response_dict, dict)
def test_stoqresponse_to_str(self): response = StoqResponse({}, RequestMeta(), []) response_str = str(response) response_dict = json.loads(response_str) self.assertIsInstance(response_str, str) self.assertIsInstance(response_dict, dict)
def test_archiver_not_in_results(self): s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['dummy_archiver']) response = s.scan( self.generic_content, request_meta=RequestMeta(archive_payloads=True) ) self.assertNotIn('dummy_archiver', response.results[0].archivers)
def test_scan_invalid_payload(self) -> None: s = Stoq(plugin_dir_list=[self.plugin_dir]) plugin = s.load_plugin(self.plugin_name) payload = Payload(b'definitely not a javaclass payload') with self.assertRaises(StoqPluginException): response = plugin.scan(payload, RequestMeta())
def main() -> None: about = f'stoQ :: v{__version__} :: an automated analysis framework' # If $STOQ_HOME exists, set our base directory to that, otherwise # use $HOME/.stoq try: stoq_home = str( Path(os.getenv('STOQ_HOME', f'{str(Path.home())}/.stoq')).resolve( strict=True ) ) except FileNotFoundError as err: print(f"$STOQ_HOME is invalid, exiting: {err}", file=sys.stderr) sys.exit(1) parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=about, epilog=''' Examples: - Scan a file with installed plugins and dispatch rules: $ %(prog)s scan mybadfile.exe - Scan a file and force it to go through the yara plugin: $ %(prog)s scan mybadfile.exe -s yara - Ingest from PubSub, force all payloads through yara, trid, and exif, then save results to file: $ %(prog)s run -a yara trid exif -P pubsub -C file - Monitor a directory (specified in dirmon.stoq) for newly created files send them to workers, and archive all payloads into MongoDB: $ %(prog)s run -P dirmon -A mongodb - Install a plugin from a directory $ %(prog)s install path/to/plugin_directory ''', ) subparsers = parser.add_subparsers(title='commands', dest='command') subparsers.required = True scan = subparsers.add_parser('scan', help='Scan a given payload') scan.add_argument( 'file', nargs='?', type=argparse.FileType('rb'), default=sys.stdin.buffer, help='File to scan, can also be provided from stdin', ) run = subparsers.add_parser( 'run', help='Continually ingest and scan payloads from Provider plugins' ) run.add_argument( '-P', '--providers', nargs='+', help='Provider plugins to ingest payloads from' ) # Add shared arguments so they still show up in the help dialog for subparser in [scan, run]: subparser.add_argument( '-A', '--dest-archivers', nargs='+', help='Archiver plugins to send payloads to', ) subparser.add_argument( '-S', '--source-archivers', nargs='+', help='Archiver plugins to read payload from', ) subparser.add_argument( '-D', '--decorators', nargs='+', help='Decorator plugins to send results to before saving', ) subparser.add_argument( '-C', '--connectors', nargs='+', help='Connector plugins to send results to' ) subparser.add_argument( '-R', '--dispatchers', nargs='+', help='Dispatcher plugins to use send payloads to', ) subparser.add_argument( '-a', '--always-dispatch', nargs='+', help='Worker plugins to always dispatch plugins to', ) subparser.add_argument( '-s', '--start-dispatch', nargs='+', help='Worker plugins to add to the original payload dispatch', ) subparser.add_argument( '--max-recursion', type=int, default=None, help='Maximum level of recursion into a payload and extracted payloads', ) subparser.add_argument('--plugin-opts', nargs='+', help='Plugin options') subparser.add_argument( '--request-source', default=None, help='Source name to add to initial scan request', ) subparser.add_argument( '--request-extra', nargs='+', help='Key/value pair to add to initial scan request metadata', ) subparser.add_argument( '--plugin-dir', nargs='+', help='Directory(ies) containing stoQ plugins' ) subparser.add_argument( '--config-file', default=f'{stoq_home}/stoq.cfg', help='Path to stoQ configuration file', ) subparser.add_argument( '--log-level', default=None, choices=['debug', 'info', 'warning', 'error' 'crtical'], help='Log level for stoQ events', ) plugin_list = subparsers.add_parser('list', help='List available plugins') plugin_list.add_argument( '--plugin-dir', nargs='+', help='Directory(ies) containing stoQ plugins' ) install = subparsers.add_parser('install', help='Install a given plugin') install.add_argument( 'plugin_path', help='Directory or Github repo of the plugin to install' ) install.add_argument( '--install_dir', default=os.path.join(stoq_home, 'plugins'), help='Override the default plugin installation directory', ) install.add_argument( '--upgrade', action='store_true', help='Force the plugin to be upgraded if it already exists', ) install.add_argument( '--github', action='store_true', help='Install plugin from Github repository' ) subparsers.add_parser('test', help='Run stoQ tests') args = parser.parse_args() plugin_opts: Union[Dict, None] = None try: if args.plugin_opts: plugin_opts = {} for arg in args.plugin_opts: plugin_name, plugin_option = arg.split(':', 1) opt, value = plugin_option.split('=', 1) if value.lower() == 'true': value = True elif value.lower() == 'false': value = False if plugin_name in plugin_opts: plugin_opts[plugin_name].update({opt: value}) else: plugin_opts[plugin_name] = {opt: value} except AttributeError: pass except ValueError as err: print(f'Failed parsing plugin option: {err}') request_meta = RequestMeta() try: if args.request_source: request_meta.source = args.request_source if args.request_extra: for arg in args.request_extra: extra_key, extra_value = arg.split('=', 1) if extra_value.lower() == 'true': extra_value = True elif extra_value.lower() == 'false': extra_value = False request_meta.extra_data[extra_key] = extra_value except AttributeError: pass except ValueError as err: print(f'Failed parsing request metadata option: {err}') try: if not os.path.isfile(args.config_file): print(f'Warning: {args.config_file} does not exist, using stoQ defaults!') except AttributeError: pass if args.command == 'scan': with args.file as f: # Verify that the file or stdin has some sort of data if not select.select([f], [], [], 0.0)[0]: print('Error: No content to scan was provided') sys.exit(2) content = f.read() if not content: print('Error: The provided content to scan was empty') sys.exit(2) if args.file.name == '<stdin>': filename = None else: path = args.file.name try: filename = os.path.basename(path.encode('utf-8')) except AttributeError: filename = os.path.basename(path) stoq = Stoq( base_dir=stoq_home, config_file=args.config_file, log_level=args.log_level, plugin_opts=plugin_opts, source_archivers=args.source_archivers, dest_archivers=args.dest_archivers, connectors=args.connectors, dispatchers=args.dispatchers, decorators=args.decorators, always_dispatch=args.always_dispatch, max_recursion=args.max_recursion, plugin_dir_list=args.plugin_dir, ) response = asyncio.get_event_loop().run_until_complete( stoq.scan( content, PayloadMeta(extra_data={'filename': filename}), request_meta=request_meta, add_start_dispatch=args.start_dispatch, ) ) if not args.connectors: print(response) elif args.command == 'run': stoq = Stoq( base_dir=stoq_home, config_file=args.config_file, log_level=args.log_level, plugin_opts=plugin_opts, providers=args.providers, source_archivers=args.source_archivers, dest_archivers=args.dest_archivers, connectors=args.connectors, dispatchers=args.dispatchers, decorators=args.decorators, always_dispatch=args.always_dispatch, max_recursion=args.max_recursion, plugin_dir_list=args.plugin_dir, ) asyncio.get_event_loop().run_until_complete( stoq.run(request_meta=request_meta, add_start_dispatch=args.start_dispatch) ) elif args.command == 'list': stoq = Stoq(base_dir=stoq_home, plugin_dir_list=args.plugin_dir) print(about) print('-' * len(about)) for name, info in stoq.list_plugins().items(): print(f'{name:<20s} v{info["version"]:<10s}{info["description"]}') print(f'\t\t\t\t- {", ".join(info["classes"]):<20s}') elif args.command == 'install': StoqPluginInstaller.install( args.plugin_path, args.install_dir, args.upgrade, args.github ) print(f'Successfully installed {args.plugin_path} into {args.install_dir}') elif args.command == 'test': test_path = os.path.dirname(tests.__file__) test_suite = unittest.TestLoader().discover(test_path) unittest.TextTestRunner(verbosity=1).run(test_suite)