def _parse_dispatch_results(self, dispatch_tuple, **kwargs): results = [] for content in dispatch_tuple: try: meta = content[0] payload = content[1] meta['puuid'] = kwargs['uuid'] meta['uuid'] = self.stoq.get_uuid meta['scan'] = self.scan(payload) except: continue if 'save' in meta: if meta['save'].lower() == "true" and self.archive_connector: hashes = self.save_payload(payload, self.archive_connector) try: meta.update(hashes) except: meta.update(get_hashes(payload)) results.append(meta) return results
def save_payload(self, payload, connector): """ Save a payload using the designated connector :param bytes payload: Payload to pass to the connector for saving :param str connector: Connector plugin to save the payload with """ arc = get_hashes(payload) # Let's make sure we add some additional metadata so we don't need # to create this later. arc['ssdeep'] = get_ssdeep(payload) arc['content-type'] = get_magic(payload) # Make sure our connector is loaded self.load_connector(connector) # Save our payload to the appropriate plugin res = self.connectors[connector].save(payload, archive=True, **arc) arc['conn_id'] = res return arc
def do_results(self, input): """ results Display results of previous plugin run """ try: # This is a mess. Plugins can produce either str(), bytes(), # or a list(). If it is a list(), there may be a tuple() in it. # Let's go over them and make sure we produce that right content # to display if self.results: if type(self.results) is dict: print(self.stoq.dumps(self.results, compactly=False)) elif type(self.results) is list: for idx, r in enumerate(self.results): if type(r) is dict: print(self.stoq.dumps(r, compactly=False)) if type(r) is tuple: if type(r[0]) is dict: print( "[*] Extracted content: id {}".format(idx)) for sub_key, sub_value in r[0].items(): print(" - {}: {}".format( sub_key, sub_value)) hashes = get_hashes(r[1]) mime = get_magic(r[1]) for key, value in hashes.items(): print(" - {}: {}".format(key, value)) print(" - magic: {}".format(mime)) else: print(r) else: print(r) else: print(self.results) else: print( "[!] No results. Did you run a plugin? Try 'run <category> <plugin>'" ) except Exception as err: print("[!] Error: {}".format(str(err)))
def do_results(self, input): """ results Display results of previous plugin run """ try: # This is a mess. Plugins can produce either str(), bytes(), # or a list(). If it is a list(), there may be a tuple() in it. # Let's go over them and make sure we produce that right content # to display if self.results: if type(self.results) is dict: print(self.stoq.dumps(self.results, compactly=False)) elif type(self.results) is list: for idx, r in enumerate(self.results): if type(r) is dict: print(self.stoq.dumps(r, compactly=False)) if type(r) is tuple: if type(r[0]) is dict: print("[*] Extracted content: id {}".format(idx)) for sub_key, sub_value in r[0].items(): print(" - {}: {}".format(sub_key, sub_value)) hashes = get_hashes(r[1]) mime = get_magic(r[1]) for key, value in hashes.items(): print(" - {}: {}".format(key, value)) print(" - magic: {}".format(mime)) else: print(r) else: print(r) else: print(self.results) else: print("[!] No results. Did you run a plugin? Try 'run <category> <plugin>'") except Exception as err: print("[!] Error: {}".format(str(err)))
def do_read(self, input): """ read <path to file> Open a file at specified path """ try: self.filename = os.path.basename(input) self.payload = self.stoq.get_file(input) if not self.payload: print("[!] No payload found.") else: hashes = get_hashes(self.payload) mime = get_magic(self.payload) print("[*] Filename: {}".format(input)) print("[*] Size: {}".format(len(self.payload))) # Iterate over all of the hashes that were generated for key, value in hashes.items(): print("[*] {}: {}".format(key, value)) print("[*] magic: {}".format(mime)) except Exception as err: print("[!] Error: {}".format(str(err)))
def start(self, payload=None, **kwargs): """ Process the payload with the worker plugin :param bytes payload: (optional) Payload to be processed :param \*\*kwargs: addtional arguments that may be needed by the worker plugin (i.e., username and password via HTTP) :type kwargs: dict or None :returns: Tuple of JSON results and template rendered results :rtype: dict and str """ archive_type = False template_results = None payload_hashes = None results = {} results['results'] = [] worker_result = {} results['date'] = self.stoq.get_time # If we don't have a uuid, let's generate one if 'uuid' not in kwargs: kwargs['uuid'] = self.stoq.get_uuid # If we have no payload, let's try to find one to process if not payload and 'archive' in kwargs: # We are going to use the 'archive' field in kwargs to define where # we are going to get the file from. Once we know that, we will # load the appropriate plugin if required. Then, we will call # get_file() to grab the payload. archive_type = kwargs['archive'] self.load_connector(archive_type) if hasattr(self.connectors[archive_type], 'get_file'): payload = self.connectors[archive_type].get_file(**kwargs) else: self.stoq.log.warn("Connector unable to get file..skipping") return False if payload: # Make sure we define this before possibly modifying the full file # path when/if we archive. if 'filename' not in kwargs: if 'path' in kwargs: kwargs['filename'] = os.path.basename(kwargs['path']) else: kwargs['filename'] = "Unknown" # If this worker wants us to save this payload to the archive, let's # handle that now before anything else. Otherwise any subsequent # plugins may not be able to retrieve the files. We are however going # to skip saving the payload if our source is the same as the # connector. if self.archive_connector and self.archive_connector != archive_type: payload_hashes = self.save_payload(payload, self.archive_connector) # Some workers don't need a hash to be generated, so let's only # generate hashes if needed. This is defined in the .stoq # configuration file for the worker plugin. We are also only going # to generate a hash if our save_payload function hasn't been # called. Otherwise, we will just use those results. if self.hashpayload: if payload_hashes: worker_result.update(payload_hashes) else: worker_result.update(get_hashes(payload)) # Send our payload to the worker, and store the results worker_result['scan'] = self.scan(payload, **kwargs) worker_result['plugin'] = self.name worker_result['uuid'] = kwargs['uuid'] if payload: worker_result['size'] = len(payload) worker_result['payload_id'] = 0 # Keep track of our total count of payloads, in case yara dispatch # finds something payload_id = 1 results['results'].append(worker_result) # If we want to use the dispatcher, let's do that now if self.dispatch: # Our carver, extractor, and decoder plugins will return a list of # set()s. Let's make sure we handle the initial payload the same # way, so we can simplify the below routine. dispatch_payloads = [(None, payload)] current_depth = 0 # Track hashes of payloads so we don't handle duplicates. processed_sha1s = {} while dispatch_payloads and self.stoq.max_recursion >= current_depth: for dispatch_payload in dispatch_payloads: # Skip over this payload if we've already processed it current_hash = get_sha1(dispatch_payload[1]) if current_hash in processed_sha1s: continue processed_sha1s[current_hash] = True dispatch_payloads = self.yara_dispatcher( dispatch_payload[1]) # Something was carved, let's gather the metadata if dispatch_payloads: dispatch_results = self._parse_dispatch_results( dispatch_payloads, **kwargs) if dispatch_results: # Iterate over the results, grab the sha1, and add # it to the list of processed hashes. Then, add the # dispatch results to the primary results for index, res in enumerate(dispatch_results): if res['sha1'] in processed_sha1s: continue processed_sha1s[res['sha1']] = True res['payload_id'] = payload_id payload_id += 1 results['results'].append(res) current_depth += 1 results['payloads'] = payload_id # Parse output with a template if self.template: try: template_path = "{}/templates".format(self.plugin_path) tpl_env = Environment(loader=FileSystemLoader(template_path), trim_blocks=True, lstrip_blocks=True) template_results = tpl_env.get_template( self.template).render(results=results) except TemplateNotFound: self.stoq.log.error("Unable to load template. Does {}/{} " "exist?".format(template_path, self.template)) except Exception as err: self.stoq.log.error(str(err)) # If we are saving the results from the worker, let's take care of # it. This is defined in the .stoq configuration file for the # worker plugin. An output_connector must also be defined. if self.saveresults and self.output_connector: # Just to ensure we have loaded a connector for output self.load_connector(self.output_connector) # If there is a template that is named after the output connector # pass the templated results to the connector, otherwise pass the # raw results if template_results: if self.template.split(".")[0] == self.output_connector: self.connectors[self.output_connector].save( template_results) else: # Attempt to save the results, and pass along the primary results # as **kwargs, otherwise just pass along the results. try: kwargs = {'sha1': results['results'][0]['sha1']} self.connectors[self.output_connector].save( results, **kwargs) except (KeyError, IndexError): self.connectors[self.output_connector].save(results) return results, template_results
def start(self, payload=None, **kwargs): """ Process the payload with the worker plugin :param bytes payload: (optional) Payload to be processed :param \*\*kwargs: addtional arguments that may be needed by the worker plugin (i.e., username and password via HTTP) :type kwargs: dict or None :returns: Tuple of JSON results and template rendered results :rtype: dict and str or lists """ archive_type = False payload_hashes = None template_results = None results = {} results['results'] = [] results['plugins'] = {} worker_result = {} results['date'] = self.stoq.get_time # If we don't have a uuid, let's generate one kwargs['uuid'] = kwargs.get('uuid', self.stoq.get_uuid) # If we have no payload, let's try to find one to process if not payload and 'archive' in kwargs: # We are going to use the 'archive' field in kwargs to define where # we are going to get the file from. Once we know that, we will # load the appropriate plugin if required. Then, we will call # get_file() to grab the payload. archive_type = kwargs['archive'] worker_result['archive'] = kwargs['archive'] self.load_connector(archive_type) if hasattr(self.connectors[archive_type], 'get_file'): payload = self.connectors[archive_type].get_file(**kwargs) else: self.stoq.log.warn("Connector unable to get file..skipping") return False if payload: # Make sure we define this before possibly modifying the full file # path when/if we archive. if 'filename' not in kwargs: if 'path' in kwargs: kwargs['filename'] = os.path.basename(kwargs['path']) worker_result['path'] = kwargs['path'] else: kwargs['filename'] = "Unknown" # Make sure we save the filename in the worker results as well worker_result['filename'] = kwargs['filename'] # If this worker wants us to save this payload to the archive, # let's handle that now before anything else. Otherwise any # subsequent plugins may not be able to retrieve the files. We are # however going to skip saving the payload if our source is the # same as the connector. if self.archive_connector and self.archive_connector != archive_type: payload_hashes = self.save_payload(payload, self.archive_connector) # Some workers don't need a hash to be generated, so let's only # generate hashes if needed. This is defined in the .stoq # configuration file for the worker plugin. We are also only going # to generate a hash if our save_payload function hasn't been # called. Otherwise, we will just use those results. if self.hashpayload: if payload_hashes: worker_result.update(payload_hashes) else: worker_result.update(get_hashes(payload)) # Send our payload to the worker, and store the results worker_result['scan'] = self.scan(payload, **kwargs) worker_result['plugin'] = self.name worker_result['uuid'] = kwargs['uuid'] if payload: worker_result['size'] = len(payload) # Preserve the original metadata that was submitted with this payload worker_result['source_meta'] = kwargs.copy() # Check to see if the keys are in the primary result dict, if so, # we will remove them from the source_meta key, otherwise, we will # leave it be. Meant to reduce duplication of data when chaining # plugins. for k, v in kwargs.items(): if k in worker_result: if v == worker_result[k]: worker_result['source_meta'].pop(k, None) # Sometimes when chaining plugins source_meta will be appended # but the keys should be at the root of the results. Let's make # sure we move them to the root rather than storing them in the # source_meta elif k in ('filename', 'puuid', 'magic', 'ssdeep', 'path', 'size'): worker_result[k] = v worker_result['source_meta'].pop(k, None) worker_result['payload_id'] = 0 results['plugins'].update({"0": self.name}) # Keep track of our total count of payloads, in case yara dispatch # finds something payload_id = 1 results['results'].append(worker_result) # If we want to use the dispatcher, let's do that now if self.dispatch: # Our carver, extractor, and decoder plugins will return a list of # set()s. Let's make sure we handle the initial payload the same # way, so we can simplify the below routine. dispatch_payloads = [({}, payload)] current_depth = 0 # Track hashes of payloads so we don't handle duplicates. processed_hashes = {} while dispatch_payloads and int(self.stoq.max_recursion) >= current_depth: for index, dispatch_payload in enumerate(dispatch_payloads): dispatch_payloads.pop(index) current_hash = dispatch_payload[0].get('sha1', get_sha1(dispatch_payload[1])) # Skip over this payload if we've already processed it if current_hash in processed_hashes: self.stoq.log.info("Skipping duplicate hash: {}".format(current_hash)) continue processed_hashes.setdefault(current_hash, True) # We are copy()ing processed hashes so we don't dispatch # payloads twice, but we still want to be able to send # dispatched payloads for additional processing temp_processed_hashes = processed_hashes.copy() # Send the payload to the yara dispatcher for yara_result in self.yara_dispatcher(dispatch_payload[1]): dispatch_result = self._parse_dispatch_results(yara_result, **kwargs) if dispatch_result['sha1'] in temp_processed_hashes: self.stoq.log.info("Skipping duplicate hash: {}".format(dispatch_result['sha1'])) continue temp_processed_hashes.setdefault(dispatch_result['sha1'], True) dispatch_payloads.append(yara_result) dispatch_result['payload_id'] = payload_id if dispatch_result.get('save').lower() == 'true' and self.archive_connector: self.save_payload(yara_result[1], self.archive_connector) results['results'].append(dispatch_result) results['plugins'].update({str(payload_id): dispatch_result['dispatcher']}) payload_id += 1 current_depth += 1 results['payloads'] = payload_id # If we want the results for all plugins to be returned in one # big json blob, combined_results must be true. if self.combined_results: results, template_results = self._save_results(results) else: # Looks like we want to save each result individually, this # gets complex. split_results = [] split_template_results = [] # Make sure we save the top level key/values so we can append # them to the new individual result dict result_date = results['date'] result_payloads = results['payloads'] result_plugins = results['plugins'] for result in results['results']: # Create the new individual results dict plugin_result = {} plugin_result['date'] = result_date plugin_result['payloads'] = result_payloads plugin_result['plugins'] = result_plugins plugin_result['results'] = [result] # Because this function returns the results, we are going # to save the individual results as it is returned from # the _save_results function r, t = self._save_results(plugin_result) # Append the results to the main results list. In many cases # templates won't be utilized, so no sense in saving them if # nothing is there. split_results.append(r) if t: split_template_results.append(t) # Replace the original results with our newly created list of # results. results = split_results if split_template_results: template_results = split_template_results return results, template_results
def yara_dispatcher(self, payload, **kwargs): """ Determine if a payload needs additional processing to extract or carve content from a payload :param bytes payload: Payload to be processed :param \*\*kwargs: addtional arguments that may be needed :type kwargs: dict or None :returns: Set of metadata and content from plugin :rtype: Generator """ self.yara_dispatcher_hits = [] self.yara_dispatcher_rules.match(data=payload, timeout=60, callback=self._dispatcher_callback) for hit in self.yara_dispatcher_hits: if 'meta' in hit: plugin_kwargs = hit['meta'] if 'plugin' in hit['meta']: plugin_type, plugin_name = hit['meta']['plugin'].lower().split(":") else: continue # Make sure this is a valid plugin category if plugin_type not in self.stoq.plugin_categories: self.stoq.log.error("{} is not a valid plugin type".format(plugin_type)) continue if plugin_type == 'carver': self.load_carver(plugin_name) try: content = self.carvers[plugin_name].carve(payload, **plugin_kwargs) except: content = None elif plugin_type == 'extractor': self.load_extractor(plugin_name) try: content = self.extractors[plugin_name].extract(payload, **plugin_kwargs) except: content = None elif plugin_type == 'decoder': self.load_decoder(plugin_name) try: content = self.decoders[plugin_name].decode(payload, **plugin_kwargs) except: content = None else: content = None if content: # Iterate over the results from the plugin and append the # yara rule metadata to it for meta in content: dispatch_result = hit['meta'].copy() # Make sure we hash the extracted content dispatch_result.update(get_hashes(meta[1])) dispatch_result['source_meta'] = {} # Keep any metadata returned by the plugin as source_meta, # but move some keys to the top lvel of the result. for k, v in meta[0].items(): if k in ('filename', 'puuid', 'magic', 'ssdeep', 'path', 'size'): dispatch_result[k] = v else: dispatch_result['source_meta'][k] = v yield (dispatch_result, meta[1]) # Cleanup self.yara_dispatcher_hits = None
def start(self, payload=None, **kwargs): """ Process the payload with the worker plugin :param bytes payload: (optional) Payload to be processed :param \*\*kwargs: addtional arguments that may be needed by the worker plugin (i.e., username and password via HTTP) :type kwargs: dict or None :returns: Tuple of JSON results and template rendered results :rtype: dict and str or lists """ archive_type = False payload_hashes = None template_results = None results = {} results['results'] = [] results['plugins'] = {} worker_result = {} results['date'] = self.stoq.get_time # If we don't have a uuid, let's generate one kwargs['uuid'] = kwargs.get('uuid', self.stoq.get_uuid) # Set the Originating uuid to that of the first payload submitted kwargs['ouuid'] = kwargs.get('ouuid', kwargs['uuid']) # If we have no payload, let's try to find one to process if not payload and 'archive' in kwargs: # We are going to use the 'archive' field in kwargs to define where # we are going to get the file from. Once we know that, we will # load the appropriate plugin if required. Then, we will call # get_file() to grab the payload. archive_type = kwargs['archive'] worker_result['archive'] = kwargs['archive'] self.load_connector(archive_type) if hasattr(self.connectors[archive_type], 'get_file'): payload = self.connectors[archive_type].get_file(**kwargs) else: self.stoq.log.warn("Connector unable to get file..skipping") return False if payload: # Make sure we define this before possibly modifying the full file # path when/if we archive. if 'filename' not in kwargs: if 'path' in kwargs: kwargs['filename'] = os.path.basename(kwargs['path']) worker_result['path'] = kwargs['path'] else: kwargs['filename'] = "Unknown" # Make sure we save the filename in the worker results as well worker_result['filename'] = kwargs['filename'] # If this worker wants us to save this payload to the archive, # let's handle that now before anything else. Otherwise any # subsequent plugins may not be able to retrieve the files. We are # however going to skip saving the payload if our source is the # same as the connector. if self.archive_connector and self.archive_connector != archive_type: payload_hashes = self.save_payload(payload, self.archive_connector) # Some workers don't need a hash to be generated, so let's only # generate hashes if needed. This is defined in the .stoq # configuration file for the worker plugin. We are also only going # to generate a hash if our save_payload function hasn't been # called. Otherwise, we will just use those results. if self.hashpayload: if payload_hashes: worker_result.update(payload_hashes) else: worker_result.update(get_hashes(payload)) # Send our payload to the worker, and store the results worker_result['scan'] = self.scan(payload, **kwargs) worker_result['plugin'] = self.name worker_result['uuid'] = kwargs['uuid'] if payload: worker_result['size'] = len(payload) # Preserve the original metadata that was submitted with this payload worker_result['source_meta'] = kwargs.copy() # Check to see if the keys are in the primary result dict, if so, # we will remove them from the source_meta key, otherwise, we will # leave it be. Meant to reduce duplication of data when chaining # plugins. for k, v in kwargs.items(): if k in worker_result: if v == worker_result[k]: worker_result['source_meta'].pop(k, None) # Sometimes when chaining plugins source_meta will be appended # but the keys should be at the root of the results. Let's make # sure we move them to the root rather than storing them in the # source_meta elif k in ('filename', 'puuid', 'magic', 'ssdeep', 'path', 'ouuid'): worker_result[k] = v worker_result['source_meta'].pop(k, None) worker_result['payload_id'] = 0 results['plugins'].update({"0": self.name}) # Keep track of our total count of payloads, in case yara dispatch # finds something payload_id = 1 results['results'].append(worker_result) # If we want to use the dispatcher, let's do that now if self.dispatch: # Our carver, extractor, and decoder plugins will return a list of # set()s. Let's make sure we handle the initial payload the same # way, so we can simplify the below routine. dispatch_payloads = [({}, payload)] dispatch_queue = [] current_depth = 0 # Track hashes of payloads so we don't handle duplicates. processed_hashes = {} while dispatch_payloads and int( self.stoq.max_recursion) >= current_depth: for index, dispatch_payload in enumerate(dispatch_payloads): dispatch_payloads.pop(index) current_hash = dispatch_payload[0].get( 'sha1', get_sha1(dispatch_payload[1])) # Skip over this payload if we've already processed it if current_hash in processed_hashes: self.stoq.log.info( "Skipping duplicate hash: {}".format(current_hash)) continue processed_hashes.setdefault(current_hash, True) # We are copy()ing processed hashes so we don't dispatch # payloads twice, but we still want to be able to send # dispatched payloads for additional processing temp_processed_hashes = processed_hashes.copy() # Send the payload to the yara dispatcher for yara_result in self.yara_dispatcher( dispatch_payload[1]): dispatch_result = self._parse_dispatch_results( yara_result, **kwargs) if dispatch_result['sha1'] in temp_processed_hashes: self.stoq.log.info( "Skipping duplicate hash: {}".format( dispatch_result['sha1'])) continue temp_processed_hashes.setdefault( dispatch_result['sha1'], True) dispatch_queue.append(yara_result) dispatch_result['payload_id'] = payload_id if dispatch_result.get('save').lower( ) == 'true' and self.archive_connector: self.save_payload(yara_result[1], self.archive_connector) results['results'].append(dispatch_result) results['plugins'].update( {str(payload_id): dispatch_result['plugin']}) payload_id += 1 dispatch_payloads = dispatch_queue.copy() dispatch_queue = [] current_depth += 1 results['payloads'] = payload_id # If we want the results for all plugins to be returned in one # big json blob, combined_results must be true. if self.combined_results: results, template_results = self._save_results(results) else: # Looks like we want to save each result individually, this # gets complex. split_results = [] split_template_results = [] # Make sure we save the top level key/values so we can append # them to the new individual result dict result_date = results['date'] result_payloads = results['payloads'] result_plugins = results['plugins'] for result in results['results']: # Create the new individual results dict plugin_result = {} plugin_result['date'] = result_date plugin_result['payloads'] = result_payloads plugin_result['plugins'] = result_plugins plugin_result['results'] = [result] # Because this function returns the results, we are going # to save the individual results as it is returned from # the _save_results function r, t = self._save_results(plugin_result) # Append the results to the main results list. In many cases # templates won't be utilized, so no sense in saving them if # nothing is there. split_results.append(r) if t: split_template_results.append(t) # Replace the original results with our newly created list of # results. results = split_results if split_template_results: template_results = split_template_results return results, template_results
def yara_dispatcher(self, payload, **kwargs): """ Determine if a payload needs additional processing to extract or carve content from a payload :param bytes payload: Payload to be processed :param \*\*kwargs: addtional arguments that may be needed :type kwargs: dict or None :returns: Set of metadata and content from plugin :rtype: Generator """ self.yara_dispatcher_hits = [] self.yara_dispatcher_rules.match(data=payload, timeout=60, callback=self._dispatcher_callback) for hit in self.yara_dispatcher_hits: if 'meta' in hit: plugin_kwargs = hit['meta'] if 'plugin' in hit['meta']: plugin_type, plugin_name = hit['meta']['plugin'].lower( ).split(":") else: continue # Make sure this is a valid plugin category if plugin_type not in self.stoq.plugin_categories: self.stoq.log.error( "{} is not a valid plugin type".format(plugin_type)) continue if plugin_type == 'carver': self.load_carver(plugin_name) try: content = self.carvers[plugin_name].carve( payload, **plugin_kwargs) except: content = None elif plugin_type == 'extractor': self.load_extractor(plugin_name) try: content = self.extractors[plugin_name].extract( payload, **plugin_kwargs) except: content = None elif plugin_type == 'decoder': self.load_decoder(plugin_name) try: content = self.decoders[plugin_name].decode( payload, **plugin_kwargs) except: content = None else: content = None if content: # Iterate over the results from the plugin and append the # yara rule metadata to it for meta in content: dispatch_result = hit['meta'].copy() # Make sure we hash the extracted content dispatch_result.update(get_hashes(meta[1])) # Keep any metadata returned by the plugin as source_meta dispatch_result['source_meta'] = meta[0] yield (dispatch_result, meta[1]) # Cleanup self.yara_dispatcher_hits = None