class EfetchHelper(object): """This class provides helper methods to be used in Efetch and its plugins""" def __init__(self, curr_directory, output_directory, max_file_size, plugins_file, es_url): """Initializes the Efetch Helper""" # Setup directory references self.curr_dir = curr_directory self.resource_dir = self.curr_dir + os.path.sep + u'resources' + os.path.sep self.icon_dir = self.resource_dir + u'icons' + os.path.sep self.output_dir = output_directory self.max_file_size = max_file_size if not os.path.isdir(self.icon_dir): logging.error(u'Could not find icon directory ' + self.icon_dir) self.pathspec_helper = PathspecHelper(output_directory, max_file_size) # Create plugin manager and begin polling for changes to plugins self.plugin_manager = EfetchPluginManager(plugins_file, self.curr_dir) self.poll = Poll(self.plugin_manager) self.poll.start() self.standard_office_2007_extensions = ['xlsx', 'docx', 'pptx', 'dotx', 'docm', 'doct', 'xlsm', 'xltx', 'xltm', 'pptx', 'pptm', 'potx', 'ppam', 'ppsx', 'ppsm', 'sldx', 'sldm'] # Elastic Search DB setup if es_url: self.db_util = DBUtil() else: self.db_util = DBUtil(es_url) def get_request_value(self, request, variable_name, default=None): """Gets the value of a variable in either a GET or POST request""" if variable_name in request.query: return request.query[variable_name] elif request.forms.get(variable_name): return request.forms.get(variable_name) return default def get_query_string(self, request, default_query=''): """Returns the query string of the given request""" if request.query_string: return "?" + request.query_string else: return default_query def get_query(self, request): """Gets the Kibana Query from the request""" return self.db_util.get_query(self.get_request_value(request, '_a', '()')) def get_theme(self, request): """Gets the Kibana Theme from the request""" return self.db_util.get_theme(self.get_request_value(request, '_a', '()')) def get_filters(self, request, must=[], must_not=[]): """Gets the Kibana Filter from the request""" return self.db_util.get_filters(self.get_request_value(request, '_a', '()'), self.get_request_value(request, '_g', '()'), self.get_request_value(request, 'timefield', 'datetime'), must, must_not) def is_expandable_evidence(self, evidence): """Returns True if evidence should be expandable and false if not""" # Separated for cleanliness, ordering matters here # Partitions and volume shadows are expandable if 'type_indicator' in evidence and evidence['type_indicator'] in ['TSK_PARTITION', 'VSHADOW']: return True # Volumes are expandable if 'volume_type' in evidence: return True # Most Storage types are expandable if 'storage_type' in evidence: # Not E02-E0N files if evidence['extension'].lower().startswith('e0') and evidence['extension'].lower() != 'e01': return False # All other storage types are expandable return True # Compression types are expandable if 'compression_type' in evidence: return True # Most archive types are expandable if 'archive_type' in evidence: # Not files with office 2007 mimetypes if evidence['mimetype'].startswith('application/vnd'): return False # Not files with office 2007 extensions if evidence['extension'].lower() in self.standard_office_2007_extensions: return False # All other archives are expandable return True # Everything else is not expandable return False def get_icon(self, evidence, resource=True): """Returns either an icon or thumbnail of the provided file""" if resource: curr_icon_dir = '/resources/icons/' else: curr_icon_dir = self.icon_dir if 'volume_type' in evidence or 'storage_type' in evidence or 'compression_type' in evidence \ or 'archive_type' in evidence: if not evidence['mimetype_known']: evidence['mimetype'] = self.pathspec_helper.get_mimetype(evidence['pathspec']) if self.is_expandable_evidence(evidence): return curr_icon_dir + '_evidence.png' if not 'meta_type' in evidence: return curr_icon_dir + '_blank.png' # If it is folder just return the folder icon if evidence['meta_type'] == 'Directory' or unicode(evidence['file_name']).strip() == "." or unicode( evidence['file_name']).strip() == "..": return curr_icon_dir + '_folder.png' if evidence['meta_type'] != 'File' and evidence['meta_type'] != 'Device': return curr_icon_dir + '_blank.png' # If the file is an image create a thumbnail if evidence['mimetype'].startswith('image') and resource: return '/plugins/thumbnail?' + evidence['url_query'] elif evidence['mimetype'].startswith('image'): self.pathspec_helper.create_thumbnail(evidence) if os.path.isfile(evidence['thumbnail_cache_path']): return evidence['thumbnail_cache_path'] else: return curr_icon_dir + '_missing.png' # TODO if mimetype is known, perform a mimetype to extension lookup instead of using extension # If file is not an image return the icon associated with the files extension else: if not os.path.isfile(self.icon_dir + str(evidence['extension']).lower() + '.png'): return curr_icon_dir + '_blank.png' else: return curr_icon_dir + evidence['extension'].lower() + '.png' def action_get(self, evidence, request, display_name, function, term, update_term = False): """Runs a function that takes an evidence item, updates the term in elastic, and returns the results""" index = self.get_request_value(request, 'index', False) value = '' # Only needed when using elasticsearch just else just return the OCR if index: # If using elasticsearch get the first entry query = {'_source': [term, evidence['pathspec']], 'query': {'term': {'pathspec.raw': evidence['pathspec']}}} first_elastic_entry = self.db_util.query_sources(query, index, 1) # If this plugin has not been run on this entry run it on all entries if term not in first_elastic_entry or update_term: # Not masking the exception, should be handled by plugin value = function(evidence, self) try: update = {term: value} events = self.db_util.scan(query, index) for item in events: logging.debug('Updating elasticsearch item: ' + str(item)) self.db_util.update(item['_id'], index, update, doc_type=item['_type']) except: logging.warn('Failed to update value in elasticsearch') else: value = first_elastic_entry[term] else: value = function(evidence, self) return value
class EfetchHelper(object): """This class provides helper methods to be used in Efetch and its plugins""" def __init__(self, curr_directory, output_directory, max_file_size, plugins_file, es_url): """Initializes the Efetch Helper""" # Setup directory references self.curr_dir = curr_directory self.resource_dir = self.curr_dir + os.path.sep + u'static' + os.path.sep self.icon_dir = self.resource_dir + u'icons' + os.path.sep self.output_dir = output_directory self.max_file_size = max_file_size if not os.path.isdir(self.icon_dir): logging.error(u'Could not find icon directory ' + self.icon_dir) self.pathspec_helper = PathspecHelper(output_directory, max_file_size) # Create plugin manager and begin polling for changes to plugins self.plugin_manager = EfetchPluginManager(plugins_file, self.curr_dir) #self.poll = Poll(self.plugin_manager) #self.poll.start() self.standard_office_2007_extensions = [ 'xlsx', 'docx', 'pptx', 'dotx', 'docm', 'doct', 'xlsm', 'xltx', 'xltm', 'pptx', 'pptm', 'potx', 'ppam', 'ppsx', 'ppsm', 'sldx', 'sldm' ] # Elastic Search DB setup if es_url: self.db_util = DBUtil() else: self.db_util = DBUtil(es_url) def get_request_value(self, request, variable_name, default=None, raise_key_error=False): """Gets the value of a variable in either a GET or POST request""" if variable_name in request.args: return request.args[variable_name] elif variable_name in request.form: return request.form[variable_name] elif raise_key_error: logging.error('Required variable "' + variable_name + '" missing from request') raise KeyError('This request requires variable "' + variable_name + '"') return default def get_query_string(self, request, default_query=''): """Returns the query string of the given request""" if request.query_string: return "?" + request.query_string else: return default_query def get_query(self, request): """Gets the Kibana Query from the request""" return self.db_util.get_query( self.get_request_value(request, '_a', '()')) def get_theme(self, request): """Gets the Kibana Theme from the request""" return self.db_util.get_theme( self.get_request_value(request, '_a', '()')) def get_filters(self, request, must=[], must_not=[]): """Gets the Kibana Filter from the request""" return self.db_util.get_filters( self.get_request_value(request, '_a', '()'), self.get_request_value(request, '_g', '()'), self.get_request_value(request, 'timefield', 'datetime'), must, must_not) def is_expandable_evidence(self, evidence): """Returns True if evidence should be expandable and false if not""" # Separated for cleanliness, ordering matters here # Partitions and volume shadows are expandable if 'type_indicator' in evidence and evidence['type_indicator'] in [ 'TSK_PARTITION', 'VSHADOW' ]: return True # Volumes are expandable if 'volume_type' in evidence: return True # Most Storage types are expandable if 'storage_type' in evidence: # Not E02-E0N files if evidence['extension'].lower().startswith( 'e0') and evidence['extension'].lower() != 'e01': return False # All other storage types are expandable return True # Compression types are expandable if 'compression_type' in evidence: return True # Most archive types are expandable if 'archive_type' in evidence: # Not files with office 2007 mimetypes if evidence['mimetype'].startswith('application/vnd'): return False # Not files with office 2007 extensions if evidence['extension'].lower( ) in self.standard_office_2007_extensions: return False # All other archives are expandable return True # Everything else is not expandable return False def get_icon(self, evidence, resource=True): """Returns either an icon or thumbnail of the provided file""" if resource: curr_icon_dir = '/static/icons/' else: curr_icon_dir = self.icon_dir if 'volume_type' in evidence or 'storage_type' in evidence or 'compression_type' in evidence \ or 'archive_type' in evidence: if not evidence['mimetype_known']: evidence['mimetype'] = self.pathspec_helper.get_mimetype( evidence['pathspec']) if self.is_expandable_evidence(evidence): return curr_icon_dir + '_evidence.png' if not 'meta_type' in evidence: return curr_icon_dir + '_blank.png' # If it is folder just return the folder icon if evidence['meta_type'] == 'Directory' or unicode( evidence['file_name']).strip() == "." or unicode( evidence['file_name']).strip() == "..": return curr_icon_dir + '_folder.png' if evidence['meta_type'] != 'File' and evidence[ 'meta_type'] != 'Device': return curr_icon_dir + '_blank.png' # If the file is an image create a thumbnail if evidence['mimetype'].startswith('image') and resource: return '/plugins/thumbnail?' + evidence['url_query'] elif evidence['mimetype'].startswith( 'image') and self.pathspec_helper.use_thumbnails(): self.pathspec_helper.create_thumbnail(evidence) if os.path.isfile(evidence['thumbnail_cache_path']): return evidence['thumbnail_cache_path'] else: return curr_icon_dir + '_missing.png' # TODO if mimetype is known, perform a mimetype to extension lookup instead of using extension # If file is not an image return the icon associated with the files extension else: if not os.path.isfile(self.icon_dir + str(evidence['extension']).lower() + '.png'): return curr_icon_dir + '_blank.png' else: return curr_icon_dir + evidence['extension'].lower() + '.png' def action_get(self, evidence, request, display_name, function, term, update_term=False): """Runs a function that takes an evidence item, updates the term in elastic, and returns the results""" index = self.get_request_value(request, 'index', False) value = '' # Only needed when using elasticsearch just else just return the OCR if index: # If using elasticsearch get the first entry query = { '_source': [term, evidence['pathspec']], 'query': { 'term': { 'pathspec.raw': evidence['pathspec'] } } } first_elastic_entry = self.db_util.query_sources(query, index, 1) # If this plugin has not been run on this entry run it on all entries if term not in first_elastic_entry or update_term: # Not masking the exception, should be handled by plugin value = function(evidence, self) try: update = {term: value} events = self.db_util.scan(query, index) for item in events: logging.debug('Updating elasticsearch item: ' + str(item)) self.db_util.update(item['_id'], index, update, doc_type=item['_type']) except: logging.warn('Failed to update value in elasticsearch') else: value = first_elastic_entry[term] else: value = function(evidence, self) return value