def output_db(self, config, matches): """ Output any matches to the database. """ log = logging.getLogger('Mastiff.Plugins.' + self.name + '.output_db') db = DB.open_db_conf(config) if db is None: return False # add the table 'yara' if it doesn't exist if DB.check_table(db, 'yara') == False: fields = ['id INTEGER PRIMARY KEY', 'sid INTEGER DEFAULT NULL', 'rule_name TEXT DEFAULT NULL', 'meta TEXT DEFAULT NULL', 'tag TEXT DEFAULT NULL', 'rule_file TEXT DEFAULT NULL', 'file_offset INTEGER DEFAULT NULL', 'string_id TEXT DEFAULT NULL', 'data TEXT DEFAULT NULL' ] if not DB.add_table(db, 'yara', fields ): log.error('Unable to add "yara" database table.') return False sqlid = DB.get_id(db, config.get_var('Misc', 'hashes')) sel_query = 'SELECT count(*) FROM yara ' sel_query += 'WHERE sid=? AND rule_name=? AND meta=? AND tag=? AND ' sel_query += 'rule_file=? AND file_offset=? AND string_id=? AND data=? ' query = 'INSERT INTO yara ' query += '(sid, rule_name, meta, tag, rule_file, file_offset, string_id, data) ' query += 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)' cur = db.cursor() # go through all matches and insert into DB if needed try: for item in matches: for y_match in item.strings: match_insert = ( sqlid, item.rule, str(item.meta), \ str(item.tags), item.namespace, \ y_match[0], y_match[1], plugins.bin2hex(y_match[2]), ) # check to see if its already in there cur.execute(sel_query, match_insert) if cur.fetchone()[0] == 0: # not in the db already, add it in log.debug('Adding %s match to database.' % (item.rule)) cur.execute(query, match_insert) db.commit() except sqlite3.Error, err: log.error('SQL error when adding item to DB: %s' % err) return False
def output_db(self, config, data): """Print output from analysis to a file.""" log = logging.getLogger('Mastiff.Plugins.' + self.name) db = DB.open_db_conf(config) if db is None: return False db.text_factory = str # If the 'files' table does now exist, add it if DB.check_table(db, 'files') == False: log.debug('Adding table files') fields = [ 'id INTEGER PRIMARY KEY', 'sid INTEGER', 'filename TEXT', 'size INTEGER', 'firstseen INTEGER', 'lastseen INTEGER', 'times INTEGER' ] if DB.add_table(db, 'files', fields) is None: return False db.commit() cur = db.cursor() sqlid = DB.get_id(db, data['hashes']) if sqlid is None: log.error('%s hashes do not exist in the database', data['filename']) return False # see if the filename already exists in the db try: cur.execute( 'SELECT id, times FROM files WHERE filename=? AND sid=?', ( data['filename'], sqlid, )) except sqlite3.Error, err: log.error('Could not query filename table: %s', err) return None
def output_db(self, config, data): """Print output from analysis to a file.""" log = logging.getLogger('Mastiff.Plugins.' + self.name) db = DB.open_db_conf(config) if db is None: return False db.text_factory = str # If the 'files' table does now exist, add it if DB.check_table(db, 'files') == False: log.debug('Adding table files') fields = [ 'id INTEGER PRIMARY KEY', 'sid INTEGER', 'filename TEXT', 'size INTEGER', 'firstseen INTEGER', 'lastseen INTEGER', 'times INTEGER'] if DB.add_table(db, 'files', fields) is None: return False db.commit() cur = db.cursor() sqlid = DB.get_id(db, data['hashes']) if sqlid is None: log.error('%s hashes do not exist in the database', data['filename']) return False # see if the filename already exists in the db try: cur.execute('SELECT id, times FROM files WHERE filename=? AND sid=?', (data['filename'], sqlid, )) except sqlite3.Error, err: log.error('Could not query filename table: %s', err) return None
def compare_hashes(self, config, my_fuzzy): """ Compare the current hash to all of the fuzzy hashes already collected. """ log = logging.getLogger('Mastiff.Plugins.' + self.name + '.compare') db = DB.open_db_conf(config) conn = db.cursor() log.info('Comparing fuzzy hashes.') fuzz_results = list() my_md5 = config.get_var('Misc', 'hashes')[0] query = 'SELECT md5, fuzzy FROM mastiff WHERE fuzzy NOT NULL' try: # compare current hash for all fuzzy hashes for results in conn.execute(query): percent = pydeep.compare(my_fuzzy, results[1]) if percent > 0 and my_md5 != results[0]: fuzz_results.append([results[0], percent]) except sqlite3.OperationalError, err: log.error('Could not grab other fuzzy hashes: %s', err) return None
def output_db(self, config, my_fuzzy): """ Add fuzzy hash to the DB.""" log = logging.getLogger('Mastiff.Plugins.' + self.name + '.DB_output') # open up the DB and extend the mastiff table to include fuzzy hashes db = DB.open_db_conf(config) # there is a possibility the mastiff table is not available yet # check for that and add it if DB.check_table(db, 'files') == False: log.debug('Adding table "files"') fields = [ 'id INTEGER PRIMARY KEY', 'sid INTEGER', 'filename TEXT', 'size INTEGER', 'firstseen INTEGER', 'lastseen INTEGER', 'times INTEGER'] if DB.add_table(db, 'files', fields) is None: return False db.commit() if not DB.add_column(db, 'mastiff', 'fuzzy TEXT DEFAULT NULL'): log.error('Unable to add column.') return False conn = db.cursor() # update our hash sqlid = DB.get_id(db, config.get_var('Misc', 'Hashes')) query = 'UPDATE mastiff SET fuzzy=? WHERE id=?' try: conn.execute(query, (my_fuzzy, sqlid, )) db.commit() except sqlite3.OperationalError, err: log.error('Unable to add fuzzy hash: %s', err) return False
pluginInfo.plugin_object.__class__.__name__) cat_filter.update({pluginInfo.plugin_object.cat_name: cat_class}) #log.debug("Category Filters: %s", cat_filter) # Now collect and load all analysis plugins self.plugin_manager.setPluginPlaces(self.plugin_paths) self.plugin_manager.setCategoriesFilter(cat_filter) self.plugin_manager.collectPlugins() # Finally collect all output plugins self.output_manager.setPluginPlaces(self.output_paths) self.output_manager.collectPlugins() # set up database self.db = DB.open_db_conf(self.config) DB.create_mastiff_tables(self.db) # set up the output object self.output = dict() # init the filename if we have it if fname is not None: self.init_file(fname) def __del__(self): """ Class destructor. """ # Close down all logging file handles so we don't have any open file descriptors log = logging.getLogger("Mastiff")
# cat_filter will be a dict in the form: # { cat_name: cat_class } # and contains all the category plugins that have been activated cat_class = getattr(cat_mod, pluginInfo.plugin_object.__class__.__name__) cat_filter.update({pluginInfo.plugin_object.cat_name: cat_class}) #log.debug("Category Filters: %s", cat_filter) # Now collect and load all analysis plugins self.plugin_manager.setPluginPlaces(self.plugin_paths) self.plugin_manager.setCategoriesFilter( cat_filter ) self.plugin_manager.collectPlugins() # set up database self.db = DB.open_db_conf(self.config) DB.create_mastiff_tables(self.db) # init the filename if we have it if fname is not None: self.init_file(fname) def __del__(self): """ Class destructor. """ # Close down all logging file handles so we don't have any open file descriptors log = logging.getLogger("Mastiff") handles = list(log.handlers) for file_handle in handles: log.removeHandler(file_handle)