async def test_scan_notnil(self) -> None:
s = Stoq(plugin_dir_list=[self.plugin_dir])
plugin = s.load_plugin(self.plugin_name)
payload = Payload(self.entropy_not_nil)
response = await plugin.scan(payload, RequestMeta())
self.assertIsInstance(response, WorkerResponse)
self.assertEqual(1.584962500721156, response.results['entropy'])
def test_archiver_in_results(self):
s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['simple_archiver'])
response = s.scan(
self.generic_content, request_meta=RequestMeta(archive_payloads=True)
)
self.assertIn('simple_archiver', response.results[0].archivers)
self.assertIn('file_save_id', response.results[0].archivers['simple_archiver'])
def test_dont_dest_archive_yara(self):
s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['dummy_archiver'])
response = s.scan(
self.generic_content, request_meta=RequestMeta(archive_payloads=True)
)
# The yara rule 'similar_simple_rule' should set save = False
self.assertNotIn('dummy_archiver', response.results[0].archivers)
def test_scan(self) -> None:
s = Stoq(plugin_dir_list=[self.plugin_dir])
plugin = s.load_plugin(self.plugin_name)
payload = Payload(self.generic_data)
response = plugin.scan(payload, RequestMeta())
self.assertIsInstance(response, WorkerResponse)
self.assertEqual('3:hMCE7pr3Kn:huJ6', response.results['ssdeep'])
def test_dispatcher(self) -> None:
s = Stoq(
plugin_dir_list=[self.plugin_dir],
plugin_opts={
self.plugin_name: {
'dispatch_rules': f'{self.data_dir}/dispatch_rules.yar'
}
},
)
plugin = s.load_plugin(self.plugin_name)
payload = Payload(self.generic_data)
response = plugin.get_dispatches(payload, RequestMeta())
self.assertIsInstance(response, DispatcherResponse)
self.assertIn('test_dispatch_plugin', response.plugin_names)
self.assertEqual(
'test_dispatch_rule', response.meta['test_dispatch_plugin']['rule']
)
self.assertIn(
'test_dispatch_plugin',
response.meta['test_dispatch_plugin']['meta']['plugin'],
)
self.assertIn('True', response.meta['test_dispatch_plugin']['meta']['save'])
self.assertEqual(
['tag1', 'tag2'], response.meta['test_dispatch_plugin']['tags']
)
def test_scan(self) -> None:
s = Stoq(plugin_dir_list=[self.plugin_dir])
plugin = s.load_plugin(self.plugin_name)
payload = Payload(self.generic_data)
response = plugin.scan(payload, RequestMeta())
self.assertIsInstance(response, WorkerResponse)
self.assertEqual('text/plain', response.results['mimetype'])
async def test_scan(self) -> None:
s = Stoq(plugin_dir_list=[str(self.plugin_dir)])
plugin = s.load_plugin(self.plugin_name)
payload = Payload(base64.b64encode(self.generic_data))
response = await plugin.scan(payload, RequestMeta())
self.assertIsInstance(response, WorkerResponse)
self.assertEqual(1, len(response.extracted))
self.assertEqual(self.generic_data, response.extracted[0].content)
def test_scan(self) -> None:
s = Stoq(plugin_dir_list=[self.plugin_dir])
plugin = s.load_plugin(self.plugin_name)
with open(f'{self.data_dir}/sample.pdf', 'rb') as f:
payload = Payload(f.read())
response = plugin.scan(payload, RequestMeta())
self.assertIsInstance(response, WorkerResponse)
self.assertIn('FileType', response.results)
self.assertEqual('PDF', response.results['FileType'])
self.assertEqual(6, response.results['PageCount'])
def test_dest_archive(self):
s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['dummy_archiver'])
dummy_archiver = s.load_plugin('dummy_archiver')
dummy_archiver.archive = create_autospec(
dummy_archiver.archive, return_value=None
)
response = s.scan(
self.generic_content, request_meta=RequestMeta(archive_payloads=True)
)
dummy_archiver.archive.assert_called_once()
self.assertIn('dummy_archiver', response.results[0].plugins_run['archivers'])
def test_scan(self) -> None:
s = Stoq(plugin_dir_list=[self.plugin_dir])
plugin = s.load_plugin(self.plugin_name)
with open(f'{self.data_dir}/TestJavaClass.class', 'rb') as f:
payload = Payload(f.read())
response = plugin.scan(payload, RequestMeta())
self.assertIsInstance(response, WorkerResponse)
self.assertIn('TestJavaClass', response.results['provided'])
self.assertGreaterEqual(len(response.results['provided']), 4)
self.assertGreaterEqual(len(response.results['required']), 2)
self.assertGreaterEqual(len(response.results['constants']), 10)
def test_scan(self) -> None:
s = Stoq(plugin_dir_list=[self.plugin_dir])
plugin = s.load_plugin(self.plugin_name)
xord = bytes(x ^ 92 for x in self.generic_data)
payload = Payload(xord)
response = plugin.scan(payload, RequestMeta())
self.assertIsInstance(response, WorkerResponse)
self.assertIn('0x5C', response.results)
self.assertEqual(
'AdjustTokenPrivileges CurrentVersion', response.results['0x5C'][0]['match']
)
self.assertEqual('CurrentVersion', response.results['0x5C'][1]['match'])
async def test_scan_xor_single_value(self) -> None:
s = Stoq(plugin_dir_list=[self.plugin_dir])
plugin = s.load_plugin(self.plugin_name)
payload = Payload(self._xor_encode(self.generic_data, self.xorkeys))
dispatch_meta = {'test': {'test': {'meta': {'xorkey': self.xorkeys}}}}
payload.dispatch_meta = dispatch_meta
response = await plugin.scan(payload, RequestMeta())
self.assertIsInstance(response, WorkerResponse)
self.assertEqual(self.generic_data, response.extracted[0].content)
self.assertEqual(
self.xorkeys,
response.extracted[0].payload_meta.extra_data['xorkey'])
def test_dont_dest_archive_request(self):
s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['dummy_archiver'])
dummy_archiver = s.load_plugin('dummy_archiver')
dummy_archiver.archive = Mock(return_value=None)
response = s.scan(
self.generic_content,
add_start_dispatch=['extract_random'],
request_meta=RequestMeta(archive_payloads=False),
)
dummy_archiver.archive.assert_not_called()
self.assertNotIn('dummy_archiver', response.results[0].plugins_run['archivers'])
self.assertNotIn('dummy_archiver', response.results[1].plugins_run['archivers'])
def test_scan(self) -> None:
s = Stoq(
plugin_dir_list=[self.plugin_dir],
plugin_opts={
self.plugin_name: {'worker_rules': f'{self.data_dir}/scan_rules.yar'}
},
)
plugin = s.load_plugin(self.plugin_name)
payload = Payload(self.generic_data)
response = plugin.scan(payload, RequestMeta())
self.assertIsInstance(response, WorkerResponse)
self.assertEqual('test_scan_rule', response.results['matches'][0]['rule'])
def test_dispatcher_save_false(self) -> None:
s = Stoq(
plugin_dir_list=[self.plugin_dir],
plugin_opts={
self.plugin_name: {
'dispatch_rules': f'{self.data_dir}/dispatch_rules.yar'
}
},
)
plugin = s.load_plugin(self.plugin_name)
payload = Payload(b'save_false')
response = plugin.get_dispatches(payload, RequestMeta())
self.assertIsInstance(response, DispatcherResponse)
self.assertIn('False', response.meta['save_false']['meta']['save'])
def test_scan(self) -> None:
s = Stoq(plugin_dir_list=[self.plugin_dir])
plugin = s.load_plugin(self.plugin_name)
payload = Payload(self.generic_data)
response = plugin.scan(payload, RequestMeta())
self.assertIsInstance(response, WorkerResponse)
self.assertEqual('cfe671457bc475ef2f51cf12b1457475',
response.results['md5'])
self.assertEqual('f610f70b1464d97f7897fefd6420ffc904df5e4f',
response.results['sha1'])
self.assertEqual(
'2fa284e62b11fea1226b35cdd726a7a56090853ed135240665ceb3939f631af7',
response.results['sha256'],
)
def test_dont_dest_archive_payload(self):
s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['dummy_archiver'])
dummy_archiver = s.load_plugin('dummy_archiver')
dummy_archiver.archive = create_autospec(
dummy_archiver.archive, return_value=None
)
response = s.scan(
self.generic_content,
payload_meta=PayloadMeta(should_archive=False),
add_start_dispatch=['extract_random'],
request_meta=RequestMeta(archive_payloads=True),
)
dummy_archiver.archive.assert_called_once()
self.assertNotIn('dummy_archiver', response.results[0].plugins_run['archivers'])
self.assertIn('dummy_archiver', response.results[1].plugins_run['archivers'])
def test_scan_meta_bytes(self) -> None:
s = Stoq(
plugin_dir_list=[self.plugin_dir],
plugin_opts={
self.plugin_name: {'worker_rules': f'{self.data_dir}/scan_rules.yar'}
},
)
plugin = s.load_plugin(self.plugin_name)
payload = Payload(b'meta_bytes')
response = plugin.scan(payload, RequestMeta())
self.assertIsInstance(response, WorkerResponse)
self.assertEqual(
'test_scan_metadata_bytes', response.results['matches'][0]['rule']
)
self.assertEqual('ANeato', response.results['matches'][0]['meta']['bytes'])
self.assertEqual(
'Peter Rabbit', response.results['matches'][0]['meta']['author']
)
self.assertEqual('save_false', response.results['matches'][0]['meta']['plugin'])
def test_reconstruct_all_subresponses(self):
# Construct a fake stoq_response as if it were generated from a file
# A.zip that contains two files, B.txt and C.zip, where C.zip contains D.txt
initial_response = StoqResponse(
results=[
PayloadResults(
payload_id="A.zip",
size=0,
payload_meta=PayloadMeta(),
workers=[{"fake": "result1"}],
plugins_run={"workers": [["fake"]]},
),
PayloadResults(
payload_id="B.txt",
size=0,
payload_meta=PayloadMeta(),
workers=[{"fake": "result2"}],
plugins_run={"workers": [["fake"]]},
extracted_from="A.zip",
extracted_by="fake",
),
PayloadResults(
payload_id="C.zip",
size=0,
payload_meta=PayloadMeta(),
workers=[{"fake": "result3"}],
plugins_run={"workers": [["fake"]]},
extracted_from="A.zip",
extracted_by="fake",
),
PayloadResults(
payload_id="D.txt",
size=0,
payload_meta=PayloadMeta(),
workers=[{"fake": "result4"}],
plugins_run={"workers": [["fake"]]},
extracted_from="C.zip",
extracted_by="fake",
),
],
request_meta=RequestMeta(extra_data={"check": "me"}),
errors={},
)
s = Stoq(base_dir=utils.get_data_dir(), decorators=["simple_decorator"])
all_subresponses = list(s.reconstruct_all_subresponses(initial_response))
# We expect there to be four "artificial" responses generated, one for
# each payload as the root.
self.assertEqual(len(all_subresponses), 4)
# We expect the first response to have all 4 payloads, the second response
# to have just the second payload, the third response to have the third
# and fourth payload, and the fourth response to have just the fourth payload
self.assertEqual(
[len(stoq_response.results) for stoq_response in all_subresponses], [4, 1, 2, 1]
)
self.assertEqual(
[
stoq_response.results[0].workers[0]["fake"]
for stoq_response in all_subresponses
],
["result1", "result2", "result3", "result4"],
)
self.assertTrue(
all(
"simple_decorator" in stoq_response.decorators
for stoq_response in all_subresponses
)
)
# Assert that they all have the same scan ID
self.assertEqual(
len({stoq_response.scan_id for stoq_response in all_subresponses}), 1
)
def test_requestmeta_to_str(self):
response = RequestMeta()
response_str = str(response)
response_dict = json.loads(response_str)
self.assertIsInstance(response_str, str)
self.assertIsInstance(response_dict, dict)
def test_stoqresponse_to_str(self):
response = StoqResponse({}, RequestMeta(), [])
response_str = str(response)
response_dict = json.loads(response_str)
self.assertIsInstance(response_str, str)
self.assertIsInstance(response_dict, dict)
def test_archiver_not_in_results(self):
s = Stoq(base_dir=utils.get_data_dir(), dest_archivers=['dummy_archiver'])
response = s.scan(
self.generic_content, request_meta=RequestMeta(archive_payloads=True)
)
self.assertNotIn('dummy_archiver', response.results[0].archivers)
def test_scan_invalid_payload(self) -> None:
s = Stoq(plugin_dir_list=[self.plugin_dir])
plugin = s.load_plugin(self.plugin_name)
payload = Payload(b'definitely not a javaclass payload')
with self.assertRaises(StoqPluginException):
response = plugin.scan(payload, RequestMeta())
def main() -> None:
about = f'stoQ :: v{__version__} :: an automated analysis framework'
# If $STOQ_HOME exists, set our base directory to that, otherwise
# use $HOME/.stoq
try:
stoq_home = str(
Path(os.getenv('STOQ_HOME', f'{str(Path.home())}/.stoq')).resolve(
strict=True
)
)
except FileNotFoundError as err:
print(f"$STOQ_HOME is invalid, exiting: {err}", file=sys.stderr)
sys.exit(1)
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=about,
epilog='''
Examples:
- Scan a file with installed plugins and dispatch rules:
$ %(prog)s scan mybadfile.exe
- Scan a file and force it to go through the yara plugin:
$ %(prog)s scan mybadfile.exe -s yara
- Ingest from PubSub, force all payloads through yara, trid, and exif,
then save results to file:
$ %(prog)s run -a yara trid exif -P pubsub -C file
- Monitor a directory (specified in dirmon.stoq) for newly created files
send them to workers, and archive all payloads into MongoDB:
$ %(prog)s run -P dirmon -A mongodb
- Install a plugin from a directory
$ %(prog)s install path/to/plugin_directory
''',
)
subparsers = parser.add_subparsers(title='commands', dest='command')
subparsers.required = True
scan = subparsers.add_parser('scan', help='Scan a given payload')
scan.add_argument(
'file',
nargs='?',
type=argparse.FileType('rb'),
default=sys.stdin.buffer,
help='File to scan, can also be provided from stdin',
)
run = subparsers.add_parser(
'run', help='Continually ingest and scan payloads from Provider plugins'
)
run.add_argument(
'-P', '--providers', nargs='+', help='Provider plugins to ingest payloads from'
)
# Add shared arguments so they still show up in the help dialog
for subparser in [scan, run]:
subparser.add_argument(
'-A',
'--dest-archivers',
nargs='+',
help='Archiver plugins to send payloads to',
)
subparser.add_argument(
'-S',
'--source-archivers',
nargs='+',
help='Archiver plugins to read payload from',
)
subparser.add_argument(
'-D',
'--decorators',
nargs='+',
help='Decorator plugins to send results to before saving',
)
subparser.add_argument(
'-C', '--connectors', nargs='+', help='Connector plugins to send results to'
)
subparser.add_argument(
'-R',
'--dispatchers',
nargs='+',
help='Dispatcher plugins to use send payloads to',
)
subparser.add_argument(
'-a',
'--always-dispatch',
nargs='+',
help='Worker plugins to always dispatch plugins to',
)
subparser.add_argument(
'-s',
'--start-dispatch',
nargs='+',
help='Worker plugins to add to the original payload dispatch',
)
subparser.add_argument(
'--max-recursion',
type=int,
default=None,
help='Maximum level of recursion into a payload and extracted payloads',
)
subparser.add_argument('--plugin-opts', nargs='+', help='Plugin options')
subparser.add_argument(
'--request-source',
default=None,
help='Source name to add to initial scan request',
)
subparser.add_argument(
'--request-extra',
nargs='+',
help='Key/value pair to add to initial scan request metadata',
)
subparser.add_argument(
'--plugin-dir', nargs='+', help='Directory(ies) containing stoQ plugins'
)
subparser.add_argument(
'--config-file',
default=f'{stoq_home}/stoq.cfg',
help='Path to stoQ configuration file',
)
subparser.add_argument(
'--log-level',
default=None,
choices=['debug', 'info', 'warning', 'error' 'crtical'],
help='Log level for stoQ events',
)
plugin_list = subparsers.add_parser('list', help='List available plugins')
plugin_list.add_argument(
'--plugin-dir', nargs='+', help='Directory(ies) containing stoQ plugins'
)
install = subparsers.add_parser('install', help='Install a given plugin')
install.add_argument(
'plugin_path', help='Directory or Github repo of the plugin to install'
)
install.add_argument(
'--install_dir',
default=os.path.join(stoq_home, 'plugins'),
help='Override the default plugin installation directory',
)
install.add_argument(
'--upgrade',
action='store_true',
help='Force the plugin to be upgraded if it already exists',
)
install.add_argument(
'--github', action='store_true', help='Install plugin from Github repository'
)
subparsers.add_parser('test', help='Run stoQ tests')
args = parser.parse_args()
plugin_opts: Union[Dict, None] = None
try:
if args.plugin_opts:
plugin_opts = {}
for arg in args.plugin_opts:
plugin_name, plugin_option = arg.split(':', 1)
opt, value = plugin_option.split('=', 1)
if value.lower() == 'true':
value = True
elif value.lower() == 'false':
value = False
if plugin_name in plugin_opts:
plugin_opts[plugin_name].update({opt: value})
else:
plugin_opts[plugin_name] = {opt: value}
except AttributeError:
pass
except ValueError as err:
print(f'Failed parsing plugin option: {err}')
request_meta = RequestMeta()
try:
if args.request_source:
request_meta.source = args.request_source
if args.request_extra:
for arg in args.request_extra:
extra_key, extra_value = arg.split('=', 1)
if extra_value.lower() == 'true':
extra_value = True
elif extra_value.lower() == 'false':
extra_value = False
request_meta.extra_data[extra_key] = extra_value
except AttributeError:
pass
except ValueError as err:
print(f'Failed parsing request metadata option: {err}')
try:
if not os.path.isfile(args.config_file):
print(f'Warning: {args.config_file} does not exist, using stoQ defaults!')
except AttributeError:
pass
if args.command == 'scan':
with args.file as f:
# Verify that the file or stdin has some sort of data
if not select.select([f], [], [], 0.0)[0]:
print('Error: No content to scan was provided')
sys.exit(2)
content = f.read()
if not content:
print('Error: The provided content to scan was empty')
sys.exit(2)
if args.file.name == '<stdin>':
filename = None
else:
path = args.file.name
try:
filename = os.path.basename(path.encode('utf-8'))
except AttributeError:
filename = os.path.basename(path)
stoq = Stoq(
base_dir=stoq_home,
config_file=args.config_file,
log_level=args.log_level,
plugin_opts=plugin_opts,
source_archivers=args.source_archivers,
dest_archivers=args.dest_archivers,
connectors=args.connectors,
dispatchers=args.dispatchers,
decorators=args.decorators,
always_dispatch=args.always_dispatch,
max_recursion=args.max_recursion,
plugin_dir_list=args.plugin_dir,
)
response = asyncio.get_event_loop().run_until_complete(
stoq.scan(
content,
PayloadMeta(extra_data={'filename': filename}),
request_meta=request_meta,
add_start_dispatch=args.start_dispatch,
)
)
if not args.connectors:
print(response)
elif args.command == 'run':
stoq = Stoq(
base_dir=stoq_home,
config_file=args.config_file,
log_level=args.log_level,
plugin_opts=plugin_opts,
providers=args.providers,
source_archivers=args.source_archivers,
dest_archivers=args.dest_archivers,
connectors=args.connectors,
dispatchers=args.dispatchers,
decorators=args.decorators,
always_dispatch=args.always_dispatch,
max_recursion=args.max_recursion,
plugin_dir_list=args.plugin_dir,
)
asyncio.get_event_loop().run_until_complete(
stoq.run(request_meta=request_meta, add_start_dispatch=args.start_dispatch)
)
elif args.command == 'list':
stoq = Stoq(base_dir=stoq_home, plugin_dir_list=args.plugin_dir)
print(about)
print('-' * len(about))
for name, info in stoq.list_plugins().items():
print(f'{name:<20s} v{info["version"]:<10s}{info["description"]}')
print(f'\t\t\t\t- {", ".join(info["classes"]):<20s}')
elif args.command == 'install':
StoqPluginInstaller.install(
args.plugin_path, args.install_dir, args.upgrade, args.github
)
print(f'Successfully installed {args.plugin_path} into {args.install_dir}')
elif args.command == 'test':
test_path = os.path.dirname(tests.__file__)
test_suite = unittest.TestLoader().discover(test_path)
unittest.TextTestRunner(verbosity=1).run(test_suite)