def _parse_dispatch_results(self, dispatch_tuple, **kwargs):
results = []
for content in dispatch_tuple:
try:
meta = content[0]
payload = content[1]
meta['puuid'] = kwargs['uuid']
meta['uuid'] = self.stoq.get_uuid
meta['scan'] = self.scan(payload)
except:
continue
if 'save' in meta:
if meta['save'].lower() == "true" and self.archive_connector:
hashes = self.save_payload(payload, self.archive_connector)
try:
meta.update(hashes)
except:
meta.update(get_hashes(payload))
results.append(meta)
return results
def save_payload(self, payload, connector):
"""
Save a payload using the designated connector
:param bytes payload: Payload to pass to the connector for saving
:param str connector: Connector plugin to save the payload with
"""
arc = get_hashes(payload)
# Let's make sure we add some additional metadata so we don't need
# to create this later.
arc['ssdeep'] = get_ssdeep(payload)
arc['content-type'] = get_magic(payload)
# Make sure our connector is loaded
self.load_connector(connector)
# Save our payload to the appropriate plugin
res = self.connectors[connector].save(payload, archive=True, **arc)
arc['conn_id'] = res
return arc
def save_payload(self, payload, connector):
"""
Save a payload using the designated connector
:param bytes payload: Payload to pass to the connector for saving
:param str connector: Connector plugin to save the payload with
"""
arc = get_hashes(payload)
# Let's make sure we add some additional metadata so we don't need
# to create this later.
arc['ssdeep'] = get_ssdeep(payload)
arc['content-type'] = get_magic(payload)
# Make sure our connector is loaded
self.load_connector(connector)
# Save our payload to the appropriate plugin
res = self.connectors[connector].save(payload, archive=True, **arc)
arc['conn_id'] = res
return arc
def do_results(self, input):
"""
results
Display results of previous plugin run
"""
try:
# This is a mess. Plugins can produce either str(), bytes(),
# or a list(). If it is a list(), there may be a tuple() in it.
# Let's go over them and make sure we produce that right content
# to display
if self.results:
if type(self.results) is dict:
print(self.stoq.dumps(self.results, compactly=False))
elif type(self.results) is list:
for idx, r in enumerate(self.results):
if type(r) is dict:
print(self.stoq.dumps(r, compactly=False))
if type(r) is tuple:
if type(r[0]) is dict:
print(
"[*] Extracted content: id {}".format(idx))
for sub_key, sub_value in r[0].items():
print(" - {}: {}".format(
sub_key, sub_value))
hashes = get_hashes(r[1])
mime = get_magic(r[1])
for key, value in hashes.items():
print(" - {}: {}".format(key, value))
print(" - magic: {}".format(mime))
else:
print(r)
else:
print(r)
else:
print(self.results)
else:
print(
"[!] No results. Did you run a plugin? Try 'run <category> <plugin>'"
)
except Exception as err:
print("[!] Error: {}".format(str(err)))
def do_results(self, input):
"""
results
Display results of previous plugin run
"""
try:
# This is a mess. Plugins can produce either str(), bytes(),
# or a list(). If it is a list(), there may be a tuple() in it.
# Let's go over them and make sure we produce that right content
# to display
if self.results:
if type(self.results) is dict:
print(self.stoq.dumps(self.results, compactly=False))
elif type(self.results) is list:
for idx, r in enumerate(self.results):
if type(r) is dict:
print(self.stoq.dumps(r, compactly=False))
if type(r) is tuple:
if type(r[0]) is dict:
print("[*] Extracted content: id {}".format(idx))
for sub_key, sub_value in r[0].items():
print(" - {}: {}".format(sub_key, sub_value))
hashes = get_hashes(r[1])
mime = get_magic(r[1])
for key, value in hashes.items():
print(" - {}: {}".format(key, value))
print(" - magic: {}".format(mime))
else:
print(r)
else:
print(r)
else:
print(self.results)
else:
print("[!] No results. Did you run a plugin? Try 'run <category> <plugin>'")
except Exception as err:
print("[!] Error: {}".format(str(err)))
def do_read(self, input):
"""
read <path to file>
Open a file at specified path
"""
try:
self.filename = os.path.basename(input)
self.payload = self.stoq.get_file(input)
if not self.payload:
print("[!] No payload found.")
else:
hashes = get_hashes(self.payload)
mime = get_magic(self.payload)
print("[*] Filename: {}".format(input))
print("[*] Size: {}".format(len(self.payload)))
# Iterate over all of the hashes that were generated
for key, value in hashes.items():
print("[*] {}: {}".format(key, value))
print("[*] magic: {}".format(mime))
except Exception as err:
print("[!] Error: {}".format(str(err)))
def do_read(self, input):
"""
read <path to file>
Open a file at specified path
"""
try:
self.filename = os.path.basename(input)
self.payload = self.stoq.get_file(input)
if not self.payload:
print("[!] No payload found.")
else:
hashes = get_hashes(self.payload)
mime = get_magic(self.payload)
print("[*] Filename: {}".format(input))
print("[*] Size: {}".format(len(self.payload)))
# Iterate over all of the hashes that were generated
for key, value in hashes.items():
print("[*] {}: {}".format(key, value))
print("[*] magic: {}".format(mime))
except Exception as err:
print("[!] Error: {}".format(str(err)))
def start(self, payload=None, **kwargs):
"""
Process the payload with the worker plugin
:param bytes payload: (optional) Payload to be processed
:param \*\*kwargs: addtional arguments that may be needed by the
worker plugin (i.e., username and password via HTTP)
:type kwargs: dict or None
:returns: Tuple of JSON results and template rendered results
:rtype: dict and str
"""
archive_type = False
template_results = None
payload_hashes = None
results = {}
results['results'] = []
worker_result = {}
results['date'] = self.stoq.get_time
# If we don't have a uuid, let's generate one
if 'uuid' not in kwargs:
kwargs['uuid'] = self.stoq.get_uuid
# If we have no payload, let's try to find one to process
if not payload and 'archive' in kwargs:
# We are going to use the 'archive' field in kwargs to define where
# we are going to get the file from. Once we know that, we will
# load the appropriate plugin if required. Then, we will call
# get_file() to grab the payload.
archive_type = kwargs['archive']
self.load_connector(archive_type)
if hasattr(self.connectors[archive_type], 'get_file'):
payload = self.connectors[archive_type].get_file(**kwargs)
else:
self.stoq.log.warn("Connector unable to get file..skipping")
return False
if payload:
# Make sure we define this before possibly modifying the full file
# path when/if we archive.
if 'filename' not in kwargs:
if 'path' in kwargs:
kwargs['filename'] = os.path.basename(kwargs['path'])
else:
kwargs['filename'] = "Unknown"
# If this worker wants us to save this payload to the archive, let's
# handle that now before anything else. Otherwise any subsequent
# plugins may not be able to retrieve the files. We are however going
# to skip saving the payload if our source is the same as the
# connector.
if self.archive_connector and self.archive_connector != archive_type:
payload_hashes = self.save_payload(payload,
self.archive_connector)
# Some workers don't need a hash to be generated, so let's only
# generate hashes if needed. This is defined in the .stoq
# configuration file for the worker plugin. We are also only going
# to generate a hash if our save_payload function hasn't been
# called. Otherwise, we will just use those results.
if self.hashpayload:
if payload_hashes:
worker_result.update(payload_hashes)
else:
worker_result.update(get_hashes(payload))
# Send our payload to the worker, and store the results
worker_result['scan'] = self.scan(payload, **kwargs)
worker_result['plugin'] = self.name
worker_result['uuid'] = kwargs['uuid']
if payload:
worker_result['size'] = len(payload)
worker_result['payload_id'] = 0
# Keep track of our total count of payloads, in case yara dispatch
# finds something
payload_id = 1
results['results'].append(worker_result)
# If we want to use the dispatcher, let's do that now
if self.dispatch:
# Our carver, extractor, and decoder plugins will return a list of
# set()s. Let's make sure we handle the initial payload the same
# way, so we can simplify the below routine.
dispatch_payloads = [(None, payload)]
current_depth = 0
# Track hashes of payloads so we don't handle duplicates.
processed_sha1s = {}
while dispatch_payloads and self.stoq.max_recursion >= current_depth:
for dispatch_payload in dispatch_payloads:
# Skip over this payload if we've already processed it
current_hash = get_sha1(dispatch_payload[1])
if current_hash in processed_sha1s:
continue
processed_sha1s[current_hash] = True
dispatch_payloads = self.yara_dispatcher(
dispatch_payload[1])
# Something was carved, let's gather the metadata
if dispatch_payloads:
dispatch_results = self._parse_dispatch_results(
dispatch_payloads, **kwargs)
if dispatch_results:
# Iterate over the results, grab the sha1, and add
# it to the list of processed hashes. Then, add the
# dispatch results to the primary results
for index, res in enumerate(dispatch_results):
if res['sha1'] in processed_sha1s:
continue
processed_sha1s[res['sha1']] = True
res['payload_id'] = payload_id
payload_id += 1
results['results'].append(res)
current_depth += 1
results['payloads'] = payload_id
# Parse output with a template
if self.template:
try:
template_path = "{}/templates".format(self.plugin_path)
tpl_env = Environment(loader=FileSystemLoader(template_path),
trim_blocks=True,
lstrip_blocks=True)
template_results = tpl_env.get_template(
self.template).render(results=results)
except TemplateNotFound:
self.stoq.log.error("Unable to load template. Does {}/{} "
"exist?".format(template_path,
self.template))
except Exception as err:
self.stoq.log.error(str(err))
# If we are saving the results from the worker, let's take care of
# it. This is defined in the .stoq configuration file for the
# worker plugin. An output_connector must also be defined.
if self.saveresults and self.output_connector:
# Just to ensure we have loaded a connector for output
self.load_connector(self.output_connector)
# If there is a template that is named after the output connector
# pass the templated results to the connector, otherwise pass the
# raw results
if template_results:
if self.template.split(".")[0] == self.output_connector:
self.connectors[self.output_connector].save(
template_results)
else:
# Attempt to save the results, and pass along the primary results
# as **kwargs, otherwise just pass along the results.
try:
kwargs = {'sha1': results['results'][0]['sha1']}
self.connectors[self.output_connector].save(
results, **kwargs)
except (KeyError, IndexError):
self.connectors[self.output_connector].save(results)
return results, template_results
def start(self, payload=None, **kwargs):
"""
Process the payload with the worker plugin
:param bytes payload: (optional) Payload to be processed
:param \*\*kwargs: addtional arguments that may be needed by the
worker plugin (i.e., username and password via HTTP)
:type kwargs: dict or None
:returns: Tuple of JSON results and template rendered results
:rtype: dict and str or lists
"""
archive_type = False
payload_hashes = None
template_results = None
results = {}
results['results'] = []
results['plugins'] = {}
worker_result = {}
results['date'] = self.stoq.get_time
# If we don't have a uuid, let's generate one
kwargs['uuid'] = kwargs.get('uuid', self.stoq.get_uuid)
# If we have no payload, let's try to find one to process
if not payload and 'archive' in kwargs:
# We are going to use the 'archive' field in kwargs to define where
# we are going to get the file from. Once we know that, we will
# load the appropriate plugin if required. Then, we will call
# get_file() to grab the payload.
archive_type = kwargs['archive']
worker_result['archive'] = kwargs['archive']
self.load_connector(archive_type)
if hasattr(self.connectors[archive_type], 'get_file'):
payload = self.connectors[archive_type].get_file(**kwargs)
else:
self.stoq.log.warn("Connector unable to get file..skipping")
return False
if payload:
# Make sure we define this before possibly modifying the full file
# path when/if we archive.
if 'filename' not in kwargs:
if 'path' in kwargs:
kwargs['filename'] = os.path.basename(kwargs['path'])
worker_result['path'] = kwargs['path']
else:
kwargs['filename'] = "Unknown"
# Make sure we save the filename in the worker results as well
worker_result['filename'] = kwargs['filename']
# If this worker wants us to save this payload to the archive,
# let's handle that now before anything else. Otherwise any
# subsequent plugins may not be able to retrieve the files. We are
# however going to skip saving the payload if our source is the
# same as the connector.
if self.archive_connector and self.archive_connector != archive_type:
payload_hashes = self.save_payload(payload, self.archive_connector)
# Some workers don't need a hash to be generated, so let's only
# generate hashes if needed. This is defined in the .stoq
# configuration file for the worker plugin. We are also only going
# to generate a hash if our save_payload function hasn't been
# called. Otherwise, we will just use those results.
if self.hashpayload:
if payload_hashes:
worker_result.update(payload_hashes)
else:
worker_result.update(get_hashes(payload))
# Send our payload to the worker, and store the results
worker_result['scan'] = self.scan(payload, **kwargs)
worker_result['plugin'] = self.name
worker_result['uuid'] = kwargs['uuid']
if payload:
worker_result['size'] = len(payload)
# Preserve the original metadata that was submitted with this payload
worker_result['source_meta'] = kwargs.copy()
# Check to see if the keys are in the primary result dict, if so,
# we will remove them from the source_meta key, otherwise, we will
# leave it be. Meant to reduce duplication of data when chaining
# plugins.
for k, v in kwargs.items():
if k in worker_result:
if v == worker_result[k]:
worker_result['source_meta'].pop(k, None)
# Sometimes when chaining plugins source_meta will be appended
# but the keys should be at the root of the results. Let's make
# sure we move them to the root rather than storing them in the
# source_meta
elif k in ('filename', 'puuid', 'magic', 'ssdeep', 'path', 'size'):
worker_result[k] = v
worker_result['source_meta'].pop(k, None)
worker_result['payload_id'] = 0
results['plugins'].update({"0": self.name})
# Keep track of our total count of payloads, in case yara dispatch
# finds something
payload_id = 1
results['results'].append(worker_result)
# If we want to use the dispatcher, let's do that now
if self.dispatch:
# Our carver, extractor, and decoder plugins will return a list of
# set()s. Let's make sure we handle the initial payload the same
# way, so we can simplify the below routine.
dispatch_payloads = [({}, payload)]
current_depth = 0
# Track hashes of payloads so we don't handle duplicates.
processed_hashes = {}
while dispatch_payloads and int(self.stoq.max_recursion) >= current_depth:
for index, dispatch_payload in enumerate(dispatch_payloads):
dispatch_payloads.pop(index)
current_hash = dispatch_payload[0].get('sha1', get_sha1(dispatch_payload[1]))
# Skip over this payload if we've already processed it
if current_hash in processed_hashes:
self.stoq.log.info("Skipping duplicate hash: {}".format(current_hash))
continue
processed_hashes.setdefault(current_hash, True)
# We are copy()ing processed hashes so we don't dispatch
# payloads twice, but we still want to be able to send
# dispatched payloads for additional processing
temp_processed_hashes = processed_hashes.copy()
# Send the payload to the yara dispatcher
for yara_result in self.yara_dispatcher(dispatch_payload[1]):
dispatch_result = self._parse_dispatch_results(yara_result, **kwargs)
if dispatch_result['sha1'] in temp_processed_hashes:
self.stoq.log.info("Skipping duplicate hash: {}".format(dispatch_result['sha1']))
continue
temp_processed_hashes.setdefault(dispatch_result['sha1'], True)
dispatch_payloads.append(yara_result)
dispatch_result['payload_id'] = payload_id
if dispatch_result.get('save').lower() == 'true' and self.archive_connector:
self.save_payload(yara_result[1], self.archive_connector)
results['results'].append(dispatch_result)
results['plugins'].update({str(payload_id): dispatch_result['dispatcher']})
payload_id += 1
current_depth += 1
results['payloads'] = payload_id
# If we want the results for all plugins to be returned in one
# big json blob, combined_results must be true.
if self.combined_results:
results, template_results = self._save_results(results)
else:
# Looks like we want to save each result individually, this
# gets complex.
split_results = []
split_template_results = []
# Make sure we save the top level key/values so we can append
# them to the new individual result dict
result_date = results['date']
result_payloads = results['payloads']
result_plugins = results['plugins']
for result in results['results']:
# Create the new individual results dict
plugin_result = {}
plugin_result['date'] = result_date
plugin_result['payloads'] = result_payloads
plugin_result['plugins'] = result_plugins
plugin_result['results'] = [result]
# Because this function returns the results, we are going
# to save the individual results as it is returned from
# the _save_results function
r, t = self._save_results(plugin_result)
# Append the results to the main results list. In many cases
# templates won't be utilized, so no sense in saving them if
# nothing is there.
split_results.append(r)
if t:
split_template_results.append(t)
# Replace the original results with our newly created list of
# results.
results = split_results
if split_template_results:
template_results = split_template_results
return results, template_results
def yara_dispatcher(self, payload, **kwargs):
"""
Determine if a payload needs additional processing to extract
or carve content from a payload
:param bytes payload: Payload to be processed
:param \*\*kwargs: addtional arguments that may be needed
:type kwargs: dict or None
:returns: Set of metadata and content from plugin
:rtype: Generator
"""
self.yara_dispatcher_hits = []
self.yara_dispatcher_rules.match(data=payload, timeout=60,
callback=self._dispatcher_callback)
for hit in self.yara_dispatcher_hits:
if 'meta' in hit:
plugin_kwargs = hit['meta']
if 'plugin' in hit['meta']:
plugin_type, plugin_name = hit['meta']['plugin'].lower().split(":")
else:
continue
# Make sure this is a valid plugin category
if plugin_type not in self.stoq.plugin_categories:
self.stoq.log.error("{} is not a valid plugin type".format(plugin_type))
continue
if plugin_type == 'carver':
self.load_carver(plugin_name)
try:
content = self.carvers[plugin_name].carve(payload, **plugin_kwargs)
except:
content = None
elif plugin_type == 'extractor':
self.load_extractor(plugin_name)
try:
content = self.extractors[plugin_name].extract(payload, **plugin_kwargs)
except:
content = None
elif plugin_type == 'decoder':
self.load_decoder(plugin_name)
try:
content = self.decoders[plugin_name].decode(payload, **plugin_kwargs)
except:
content = None
else:
content = None
if content:
# Iterate over the results from the plugin and append the
# yara rule metadata to it
for meta in content:
dispatch_result = hit['meta'].copy()
# Make sure we hash the extracted content
dispatch_result.update(get_hashes(meta[1]))
dispatch_result['source_meta'] = {}
# Keep any metadata returned by the plugin as source_meta,
# but move some keys to the top lvel of the result.
for k, v in meta[0].items():
if k in ('filename', 'puuid', 'magic', 'ssdeep', 'path', 'size'):
dispatch_result[k] = v
else:
dispatch_result['source_meta'][k] = v
yield (dispatch_result, meta[1])
# Cleanup
self.yara_dispatcher_hits = None
def start(self, payload=None, **kwargs):
"""
Process the payload with the worker plugin
:param bytes payload: (optional) Payload to be processed
:param \*\*kwargs: addtional arguments that may be needed by the
worker plugin (i.e., username and password via HTTP)
:type kwargs: dict or None
:returns: Tuple of JSON results and template rendered results
:rtype: dict and str or lists
"""
archive_type = False
payload_hashes = None
template_results = None
results = {}
results['results'] = []
results['plugins'] = {}
worker_result = {}
results['date'] = self.stoq.get_time
# If we don't have a uuid, let's generate one
kwargs['uuid'] = kwargs.get('uuid', self.stoq.get_uuid)
# Set the Originating uuid to that of the first payload submitted
kwargs['ouuid'] = kwargs.get('ouuid', kwargs['uuid'])
# If we have no payload, let's try to find one to process
if not payload and 'archive' in kwargs:
# We are going to use the 'archive' field in kwargs to define where
# we are going to get the file from. Once we know that, we will
# load the appropriate plugin if required. Then, we will call
# get_file() to grab the payload.
archive_type = kwargs['archive']
worker_result['archive'] = kwargs['archive']
self.load_connector(archive_type)
if hasattr(self.connectors[archive_type], 'get_file'):
payload = self.connectors[archive_type].get_file(**kwargs)
else:
self.stoq.log.warn("Connector unable to get file..skipping")
return False
if payload:
# Make sure we define this before possibly modifying the full file
# path when/if we archive.
if 'filename' not in kwargs:
if 'path' in kwargs:
kwargs['filename'] = os.path.basename(kwargs['path'])
worker_result['path'] = kwargs['path']
else:
kwargs['filename'] = "Unknown"
# Make sure we save the filename in the worker results as well
worker_result['filename'] = kwargs['filename']
# If this worker wants us to save this payload to the archive,
# let's handle that now before anything else. Otherwise any
# subsequent plugins may not be able to retrieve the files. We are
# however going to skip saving the payload if our source is the
# same as the connector.
if self.archive_connector and self.archive_connector != archive_type:
payload_hashes = self.save_payload(payload,
self.archive_connector)
# Some workers don't need a hash to be generated, so let's only
# generate hashes if needed. This is defined in the .stoq
# configuration file for the worker plugin. We are also only going
# to generate a hash if our save_payload function hasn't been
# called. Otherwise, we will just use those results.
if self.hashpayload:
if payload_hashes:
worker_result.update(payload_hashes)
else:
worker_result.update(get_hashes(payload))
# Send our payload to the worker, and store the results
worker_result['scan'] = self.scan(payload, **kwargs)
worker_result['plugin'] = self.name
worker_result['uuid'] = kwargs['uuid']
if payload:
worker_result['size'] = len(payload)
# Preserve the original metadata that was submitted with this payload
worker_result['source_meta'] = kwargs.copy()
# Check to see if the keys are in the primary result dict, if so,
# we will remove them from the source_meta key, otherwise, we will
# leave it be. Meant to reduce duplication of data when chaining
# plugins.
for k, v in kwargs.items():
if k in worker_result:
if v == worker_result[k]:
worker_result['source_meta'].pop(k, None)
# Sometimes when chaining plugins source_meta will be appended
# but the keys should be at the root of the results. Let's make
# sure we move them to the root rather than storing them in the
# source_meta
elif k in ('filename', 'puuid', 'magic', 'ssdeep', 'path',
'ouuid'):
worker_result[k] = v
worker_result['source_meta'].pop(k, None)
worker_result['payload_id'] = 0
results['plugins'].update({"0": self.name})
# Keep track of our total count of payloads, in case yara dispatch
# finds something
payload_id = 1
results['results'].append(worker_result)
# If we want to use the dispatcher, let's do that now
if self.dispatch:
# Our carver, extractor, and decoder plugins will return a list of
# set()s. Let's make sure we handle the initial payload the same
# way, so we can simplify the below routine.
dispatch_payloads = [({}, payload)]
dispatch_queue = []
current_depth = 0
# Track hashes of payloads so we don't handle duplicates.
processed_hashes = {}
while dispatch_payloads and int(
self.stoq.max_recursion) >= current_depth:
for index, dispatch_payload in enumerate(dispatch_payloads):
dispatch_payloads.pop(index)
current_hash = dispatch_payload[0].get(
'sha1', get_sha1(dispatch_payload[1]))
# Skip over this payload if we've already processed it
if current_hash in processed_hashes:
self.stoq.log.info(
"Skipping duplicate hash: {}".format(current_hash))
continue
processed_hashes.setdefault(current_hash, True)
# We are copy()ing processed hashes so we don't dispatch
# payloads twice, but we still want to be able to send
# dispatched payloads for additional processing
temp_processed_hashes = processed_hashes.copy()
# Send the payload to the yara dispatcher
for yara_result in self.yara_dispatcher(
dispatch_payload[1]):
dispatch_result = self._parse_dispatch_results(
yara_result, **kwargs)
if dispatch_result['sha1'] in temp_processed_hashes:
self.stoq.log.info(
"Skipping duplicate hash: {}".format(
dispatch_result['sha1']))
continue
temp_processed_hashes.setdefault(
dispatch_result['sha1'], True)
dispatch_queue.append(yara_result)
dispatch_result['payload_id'] = payload_id
if dispatch_result.get('save').lower(
) == 'true' and self.archive_connector:
self.save_payload(yara_result[1],
self.archive_connector)
results['results'].append(dispatch_result)
results['plugins'].update(
{str(payload_id): dispatch_result['plugin']})
payload_id += 1
dispatch_payloads = dispatch_queue.copy()
dispatch_queue = []
current_depth += 1
results['payloads'] = payload_id
# If we want the results for all plugins to be returned in one
# big json blob, combined_results must be true.
if self.combined_results:
results, template_results = self._save_results(results)
else:
# Looks like we want to save each result individually, this
# gets complex.
split_results = []
split_template_results = []
# Make sure we save the top level key/values so we can append
# them to the new individual result dict
result_date = results['date']
result_payloads = results['payloads']
result_plugins = results['plugins']
for result in results['results']:
# Create the new individual results dict
plugin_result = {}
plugin_result['date'] = result_date
plugin_result['payloads'] = result_payloads
plugin_result['plugins'] = result_plugins
plugin_result['results'] = [result]
# Because this function returns the results, we are going
# to save the individual results as it is returned from
# the _save_results function
r, t = self._save_results(plugin_result)
# Append the results to the main results list. In many cases
# templates won't be utilized, so no sense in saving them if
# nothing is there.
split_results.append(r)
if t:
split_template_results.append(t)
# Replace the original results with our newly created list of
# results.
results = split_results
if split_template_results:
template_results = split_template_results
return results, template_results
def yara_dispatcher(self, payload, **kwargs):
"""
Determine if a payload needs additional processing to extract
or carve content from a payload
:param bytes payload: Payload to be processed
:param \*\*kwargs: addtional arguments that may be needed
:type kwargs: dict or None
:returns: Set of metadata and content from plugin
:rtype: Generator
"""
self.yara_dispatcher_hits = []
self.yara_dispatcher_rules.match(data=payload,
timeout=60,
callback=self._dispatcher_callback)
for hit in self.yara_dispatcher_hits:
if 'meta' in hit:
plugin_kwargs = hit['meta']
if 'plugin' in hit['meta']:
plugin_type, plugin_name = hit['meta']['plugin'].lower(
).split(":")
else:
continue
# Make sure this is a valid plugin category
if plugin_type not in self.stoq.plugin_categories:
self.stoq.log.error(
"{} is not a valid plugin type".format(plugin_type))
continue
if plugin_type == 'carver':
self.load_carver(plugin_name)
try:
content = self.carvers[plugin_name].carve(
payload, **plugin_kwargs)
except:
content = None
elif plugin_type == 'extractor':
self.load_extractor(plugin_name)
try:
content = self.extractors[plugin_name].extract(
payload, **plugin_kwargs)
except:
content = None
elif plugin_type == 'decoder':
self.load_decoder(plugin_name)
try:
content = self.decoders[plugin_name].decode(
payload, **plugin_kwargs)
except:
content = None
else:
content = None
if content:
# Iterate over the results from the plugin and append the
# yara rule metadata to it
for meta in content:
dispatch_result = hit['meta'].copy()
# Make sure we hash the extracted content
dispatch_result.update(get_hashes(meta[1]))
# Keep any metadata returned by the plugin as source_meta
dispatch_result['source_meta'] = meta[0]
yield (dispatch_result, meta[1])
# Cleanup
self.yara_dispatcher_hits = None
