def output_db(self, config, matches):
""" Output any matches to the database. """
log = logging.getLogger('Mastiff.Plugins.' + self.name + '.output_db')
db = DB.open_db_conf(config)
if db is None:
return False
# add the table 'yara' if it doesn't exist
if DB.check_table(db, 'yara') == False:
fields = ['id INTEGER PRIMARY KEY',
'sid INTEGER DEFAULT NULL',
'rule_name TEXT DEFAULT NULL',
'meta TEXT DEFAULT NULL',
'tag TEXT DEFAULT NULL',
'rule_file TEXT DEFAULT NULL',
'file_offset INTEGER DEFAULT NULL',
'string_id TEXT DEFAULT NULL',
'data TEXT DEFAULT NULL' ]
if not DB.add_table(db, 'yara', fields ):
log.error('Unable to add "yara" database table.')
return False
sqlid = DB.get_id(db, config.get_var('Misc', 'hashes'))
sel_query = 'SELECT count(*) FROM yara '
sel_query += 'WHERE sid=? AND rule_name=? AND meta=? AND tag=? AND '
sel_query += 'rule_file=? AND file_offset=? AND string_id=? AND data=? '
query = 'INSERT INTO yara '
query += '(sid, rule_name, meta, tag, rule_file, file_offset, string_id, data) '
query += 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
cur = db.cursor()
# go through all matches and insert into DB if needed
try:
for item in matches:
for y_match in item.strings:
match_insert = ( sqlid, item.rule, str(item.meta), \
str(item.tags), item.namespace, \
y_match[0], y_match[1], plugins.bin2hex(y_match[2]), )
# check to see if its already in there
cur.execute(sel_query, match_insert)
if cur.fetchone()[0] == 0:
# not in the db already, add it in
log.debug('Adding %s match to database.' % (item.rule))
cur.execute(query, match_insert)
db.commit()
except sqlite3.Error, err:
log.error('SQL error when adding item to DB: %s' % err)
return False
def output_db(self, config, data):
"""Print output from analysis to a file."""
log = logging.getLogger('Mastiff.Plugins.' + self.name)
db = DB.open_db_conf(config)
if db is None:
return False
db.text_factory = str
# If the 'files' table does now exist, add it
if DB.check_table(db, 'files') == False:
log.debug('Adding table files')
fields = [
'id INTEGER PRIMARY KEY', 'sid INTEGER', 'filename TEXT',
'size INTEGER', 'firstseen INTEGER', 'lastseen INTEGER',
'times INTEGER'
]
if DB.add_table(db, 'files', fields) is None:
return False
db.commit()
cur = db.cursor()
sqlid = DB.get_id(db, data['hashes'])
if sqlid is None:
log.error('%s hashes do not exist in the database',
data['filename'])
return False
# see if the filename already exists in the db
try:
cur.execute(
'SELECT id, times FROM files WHERE filename=? AND sid=?', (
data['filename'],
sqlid,
))
except sqlite3.Error, err:
log.error('Could not query filename table: %s', err)
return None
def output_db(self, config, data):
"""Print output from analysis to a file."""
log = logging.getLogger('Mastiff.Plugins.' + self.name)
db = DB.open_db_conf(config)
if db is None:
return False
db.text_factory = str
# If the 'files' table does now exist, add it
if DB.check_table(db, 'files') == False:
log.debug('Adding table files')
fields = [ 'id INTEGER PRIMARY KEY',
'sid INTEGER',
'filename TEXT',
'size INTEGER',
'firstseen INTEGER',
'lastseen INTEGER',
'times INTEGER']
if DB.add_table(db, 'files', fields) is None:
return False
db.commit()
cur = db.cursor()
sqlid = DB.get_id(db, data['hashes'])
if sqlid is None:
log.error('%s hashes do not exist in the database', data['filename'])
return False
# see if the filename already exists in the db
try:
cur.execute('SELECT id, times FROM files WHERE filename=? AND sid=?',
(data['filename'], sqlid, ))
except sqlite3.Error, err:
log.error('Could not query filename table: %s', err)
return None
def compare_hashes(self, config, my_fuzzy):
"""
Compare the current hash to all of the fuzzy
hashes already collected.
"""
log = logging.getLogger('Mastiff.Plugins.' + self.name + '.compare')
db = DB.open_db_conf(config)
conn = db.cursor()
log.info('Comparing fuzzy hashes.')
fuzz_results = list()
my_md5 = config.get_var('Misc', 'hashes')[0]
query = 'SELECT md5, fuzzy FROM mastiff WHERE fuzzy NOT NULL'
try:
# compare current hash for all fuzzy hashes
for results in conn.execute(query):
percent = pydeep.compare(my_fuzzy, results[1])
if percent > 0 and my_md5 != results[0]:
fuzz_results.append([results[0], percent])
except sqlite3.OperationalError, err:
log.error('Could not grab other fuzzy hashes: %s', err)
return None
def output_db(self, config, my_fuzzy):
""" Add fuzzy hash to the DB."""
log = logging.getLogger('Mastiff.Plugins.' + self.name + '.DB_output')
# open up the DB and extend the mastiff table to include fuzzy hashes
db = DB.open_db_conf(config)
# there is a possibility the mastiff table is not available yet
# check for that and add it
if DB.check_table(db, 'files') == False:
log.debug('Adding table "files"')
fields = [ 'id INTEGER PRIMARY KEY',
'sid INTEGER',
'filename TEXT',
'size INTEGER',
'firstseen INTEGER',
'lastseen INTEGER',
'times INTEGER']
if DB.add_table(db, 'files', fields) is None:
return False
db.commit()
if not DB.add_column(db, 'mastiff', 'fuzzy TEXT DEFAULT NULL'):
log.error('Unable to add column.')
return False
conn = db.cursor()
# update our hash
sqlid = DB.get_id(db, config.get_var('Misc', 'Hashes'))
query = 'UPDATE mastiff SET fuzzy=? WHERE id=?'
try:
conn.execute(query, (my_fuzzy, sqlid, ))
db.commit()
except sqlite3.OperationalError, err:
log.error('Unable to add fuzzy hash: %s', err)
return False
pluginInfo.plugin_object.__class__.__name__)
cat_filter.update({pluginInfo.plugin_object.cat_name: cat_class})
#log.debug("Category Filters: %s", cat_filter)
# Now collect and load all analysis plugins
self.plugin_manager.setPluginPlaces(self.plugin_paths)
self.plugin_manager.setCategoriesFilter(cat_filter)
self.plugin_manager.collectPlugins()
# Finally collect all output plugins
self.output_manager.setPluginPlaces(self.output_paths)
self.output_manager.collectPlugins()
# set up database
self.db = DB.open_db_conf(self.config)
DB.create_mastiff_tables(self.db)
# set up the output object
self.output = dict()
# init the filename if we have it
if fname is not None:
self.init_file(fname)
def __del__(self):
"""
Class destructor.
"""
# Close down all logging file handles so we don't have any open file descriptors
log = logging.getLogger("Mastiff")
# cat_filter will be a dict in the form:
# { cat_name: cat_class }
# and contains all the category plugins that have been activated
cat_class = getattr(cat_mod,
pluginInfo.plugin_object.__class__.__name__)
cat_filter.update({pluginInfo.plugin_object.cat_name: cat_class})
#log.debug("Category Filters: %s", cat_filter)
# Now collect and load all analysis plugins
self.plugin_manager.setPluginPlaces(self.plugin_paths)
self.plugin_manager.setCategoriesFilter( cat_filter )
self.plugin_manager.collectPlugins()
# set up database
self.db = DB.open_db_conf(self.config)
DB.create_mastiff_tables(self.db)
# init the filename if we have it
if fname is not None:
self.init_file(fname)
def __del__(self):
"""
Class destructor.
"""
# Close down all logging file handles so we don't have any open file descriptors
log = logging.getLogger("Mastiff")
handles = list(log.handlers)
for file_handle in handles:
log.removeHandler(file_handle)