def test01RunScanners(self): """ Running Logical Index Scanner """ ## Make sure the word secret is in there. pdbh = DB.DBO() pdbh.execute("select * from dictionary where word='secret' limit 1") row = pdbh.fetch() if not row: pdbh.insert('dictionary', **{'word':'secret', 'class':'English', 'type':'word'}) env = pyflagsh.environment(case=self.test_case) pyflagsh.shell_execv(env=env, command="scan", argv=["*",'IndexScan']) dbh = DB.DBO(self.test_case) dbh2 = DB.DBO(self.test_case) fsfd = DBFS(self.test_case) dbh.execute("select inode_id, word,offset,length from LogicalIndexOffsets join %s.dictionary on LogicalIndexOffsets.word_id=%s.dictionary.id where word='secret'", (config.FLAGDB,config.FLAGDB)) count = 0 for row in dbh: count += 1 path, inode, inode_id = fsfd.lookup(inode_id = row['inode_id']) fd = fsfd.open(inode=inode) fd.overread = True fd.slack = True fd.seek(row['offset']) data = fd.read(row['length']) print "Looking for %s: Found in %s at offset %s length %s %r" % ( row['word'], inode, row['offset'], row['length'],data) self.assertEqual(data.lower(), row['word'].lower()) ## Did we find all the secrets? self.assertEqual(count,2)
def execute(self): if len(self.args) < 2: yield self.help() return ## Try to glob the inode list: dbh = DB.DBO(self.environment._CASE) dbh.execute( "select inode_id from vfs where !isnull(inode_id) and path rlike %r", (fnmatch.translate(self.args[0]))) pdbh = DB.DBO() pdbh.mass_insert_start('jobs') ## This is a cookie used to identify our requests so that we ## can check they have been done later. cookie = time.time() scanners = [] for i in range(1, len(self.args)): scanners.extend( fnmatch.filter(Registry.SCANNERS.scanners, self.args[i])) scanners = ScannerUtils.fill_in_dependancies(scanners) for row in dbh: Scanner.scan_inode_distributed(dbh.case, row['inode_id'], scanners, cookie=cookie) self.wait_for_scan(cookie) yield "Scanning complete"
def execute(self): try: dbh = DB.DBO() except: dbh = DB.DBO('mysql') dbh.execute("create database `%s`" % config.FLAGDB) dbh = DB.DBO() FlagFramework.post_event("init_default_db", None) yield "Done"
def startup(self): ## Check to see if the nsrl db exists try: dbh = DB.DBO(config.HASHDB) dbh.execute("select * from meta limit 1") dbh.fetch() except Exception, e: try: dbh = DB.DBO() self.init_default_db(dbh, None) except: pass
def execute(self): scanners = [] if len(self.args) < 2: yield self.help() return elif type(self.args[1]) == types.ListType: scanners = self.args[1] else: for i in range(1, len(self.args)): scanners.extend( fnmatch.filter(Registry.SCANNERS.scanners, self.args[i])) ## Assume that people always want recursive - I think this makes sense path = self.args[0] if not path.endswith("*"): path = path + "*" ## FIXME For massive images this should be broken up, as in the old GUI method dbh = DB.DBO(self.environment._CASE) dbh.execute( "select inode.inode from inode join file on file.inode = inode.inode where file.path rlike %r", fnmatch.translate(path)) pdbh = DB.DBO() pdbh.mass_insert_start('jobs') ## This is a cookie used to identify our requests so that we ## can check they have been done later. cookie = int(time.time()) for row in dbh: inode = row['inode'] pdbh.mass_insert( command='Scan', arg1=self.environment._CASE, arg2=row['inode'], arg3=','.join(scanners), cookie=cookie, ) # pdbh.mass_insert_commit() ## Wait for the scanners to finish: self.wait_for_scan(cookie) yield "Scanning complete"
def execute(self): if len(self.args) < 2: yield self.help() return pdbh = DB.DBO() pdbh.mass_insert_start('jobs') cookie = int(time.time()) scanners = [] for i in range(1, len(self.args)): scanners.extend( fnmatch.filter(Registry.SCANNERS.scanners, self.args[i])) for path in self.glob_files(self.args[:1]): path, inode, inode_id = self.environment._FS.lookup(path=path) ## This is a cookie used to identify our requests so that we ## can check they have been done later. pdbh.mass_insert( command='Scan', arg1=self.environment._CASE, arg2=inode, arg3=','.join(scanners), cookie=cookie, ) pdbh.mass_insert_commit() ## Wait for the scanners to finish: if 1 or self.environment.interactive: self.wait_for_scan(cookie) yield "Scanning complete"
def CaseTableTests(self, tablename): ## Apply each column's test filters: t = Registry.CASE_TABLES.dispatch(tablename)() result = HTMLUI.HTMLUI(initial=True) dbh = DB.DBO(self.test_case) ## For each column run all its test cases: elements = [c for c in t.bind_columns(self.test_case)] ## Create a renderer: r = UI.TableRenderer(elements=elements, table=tablename) for c in elements: for operator, arg, e in c.tests: try: ## Get the SQL: r.filter_str = "'%s' %s '%s'" % (c.name, operator, arg) query = FlagFramework.query_type(direction=1) sql = r._make_sql(query) print "%s: Testing %s: %s" % (tablename, c.__class__, r.filter_str) dbh.execute(sql + " limit 1") dbh.fetch() except Exception: if not e: raise continue if e: raise Exception( "Expected an exception but did not receive one on filter string %s. SQL was %s" % (r.filter_str, sql))
def test01Mounted(self): """ Test that mounted images work """ ## We should be able to see test_file in the file table: dbh=DB.DBO(self.test_case) dbh.execute("select * from file where name = %r",self.test_file) row = dbh.fetch() self.assert_(row,"Unable to find the file in the VFS???")
def start_workers(): if config.FLUSH: dbh = DB.DBO() pyflaglog.log(pyflaglog.WARNING, "Deleting job queue and killing workers") #dbh.execute("select max(id) as max from jobs") #row = dbh.fetch() #broadcast_id = row['max'] or 0 dbh.execute("delete from jobs") #dbh.insert("jobs", _fast=True, # command='Exit', state='broadcast', # ) if config.WORKERS == 0: return for i in range(config.WORKERS): try: r, w = os.pipe() pid = os.fork() except AttributeError: ## When running under windows we can not fork... We must ## launch this module by itself instead - this is very ## suboptimal because we will be performing all startup ## code (registry parsing etc) for each worker. If you want ## performance you would not choose windows anyway, ## though. The following is windows specific: ## First find the name of the interpreter: import ctypes, sys name = ctypes.create_string_buffer(255) length = ctypes.windll.kernel32.GetModuleFileNameA(None, name, 255) interpreter = name.raw[:length] ## This encloses at least the file path in quotes just in ## case we are installed to somewhere with spaces - It ## seems that on windows argvs are not processed correctly ## because the below array ends up as a single command line ## string WTF? This is very dodgy... os.spawnv( os.P_NOWAIT, interpreter, ['"%s"' % interpreter, '"%s"' % __file__] + sys.argv[1:]) pid = 1 ## Parents: if pid: os.close(r) children.append(pid) else: os.close(w) nanny(worker_run, keepalive=r) atexit.register(terminate_children) ## The parent now calls the startup method on each of the events: for event in Registry.EVENT_HANDLERS.classes: try: event().startup() except Exception, e: pyflaglog.log(pyflaglog.WARNING, "Error: %s" % e)
def tree_cb(path): if path=='/': dbh = DB.DBO(query['case']) dbh.cached_execute("select tld from http where content_type like 'text/html%%' group by tld order by tld") for row in dbh: tld = row['tld'] yield ((tld,tld,'leaf'))
def test03FilteredLoad(self): """ Test that filter expressions work when loading """ filter = ' "rhost" maxmind_country AUS and "status" = 404 ' ## See if we can load the preset again: log = LogFile.load_preset(self.test_case, self.log_preset, [self.datafile]) t = time.time() table_name = self.test_table + "filtered" ## Load the data: for a in log.load(table_name, filter=filter): print a print "Took %s seconds to load log" % (time.time()-t) ## How many rows were inserted dbh = DB.DBO(self.test_case) dbh.execute("select count(*) as c from `%s_log`", table_name) row = dbh.fetch() ## Now try to enforce the same filter on the original table ## (which has all the rows in it): for f in log.fields: f.table = self.test_table+"_log" filter_sql = parser.parse_to_sql(filter, log.fields, None) dbh.execute("select count(*) as c from `%s_log` where %s", (self.test_table, filter_sql)) row2 = dbh.fetch() self.assertEqual(row['c'], row2['c'])
def casetable(self, field, query): """ Checks that field is a table within the case given as query[case]. This is not a fatal error, we just return false if not. """ dbh = DB.DBO(query['case']) try: dbh.execute("select * from `%s_log` limit 1", query[field]) except DB.DBError: return False
def run_analysis(self, report, query): """ Run the analysis """ print query try: canonical_query = self.flag.canonicalise(query) thread_name = threading.currentThread().getName() print "Current thread is %s" % thread_name try: report.analyse(query) print "analysed report" except Exception, e: gtk.gdk.threads_enter() self.error_popup(e) gtk.gdk.threads_leave() return dbh = DB.DBO(query['case']) dbh.execute("insert into meta set property=%r,value=%r", ('report_executed', canonical_query)) ## This thread must never touch GTK stuff or dead lock ## will occur. We must signal the other threads that we ## have finished analysis. del self.running_threads[query.__str__()] return
def test01CreatePreset(self): """ Test that Advanced Presets can be created """ dbh = DB.DBO(self.test_case) log = AdvancedLog(case=self.test_case) query = query_type(datafile = self.test_file, log_preset=self.log_preset, field_param_1_name="Timestamp", field_param_1_column="time", field_param_1_format="%b %d %H:%M:%S", field_param_1_override_year="2007", field_types1="TimestampType", field_types2="StringType", field_param_2_name = "Host", field_param_2_column = 'host', field_types3="StringType", field_param_3_name ="Service", field_param_3_column = "service", field_types4="PadType", field_param_4_regex = "(\[[^\]]+\])?", field_types5="StringType", field_param_5_name = "Message", field_param_5_column = 'messages', field_param_5_regex = '.*', ) log.parse(query) log.store(self.log_preset)
def wait_for_scan(self, cookie): """ Waits for scanners to complete """ import pyflag.Farm as Farm while Farm.get_cookie_reference(cookie) > 0: time.sleep(0.5) return print "Waiting for cookie %s" % cookie pdbh = DB.DBO() ## Often this process owns a worker as well. In that case we can wake it up: import pyflag.Farm as Farm #Farm.wake_workers() ## Wait until there are no more jobs left. while 1: pdbh.execute("select * from jobs where cookie=%r limit 1", (cookie)) row = pdbh.fetch() if not row: break time.sleep(1)
def startup(self): try: dbh = DB.DBO() dbh.check_index("whois_routes", "netmask") dbh.check_index("whois_routes", "network") except: pass
def __init__(self, fsfd): GenScanFactory.__init__(self, fsfd) ## Make sure we have indexes on the NSRL tables: dbh_flag = DB.DBO(config.HASHDB) dbh_flag.check_index("NSRL_hashes", "md5", 4) dbh_flag.check_index("NSRL_products", "Code")
def test03CreateAnotherPreset(self): """ Another setup test for Simple log file preset ... """ dbh = DB.DBO(self.test_case) myLog = SimpleLog(case=self.test_case) myQuery = query_type(datafile=self.test_file_two, log_preset=self.log_preset_two, delimiter=",", field0="AString", field1="SomeNumber", field2="FirstIP", field3="BString", field4="SecondIP", field5="OtherNumber", field6="ThirdIP", type0="StringType", type1="IntegerType", type2="IPType", type3="StringType", type4="IPType", type5="IntegerType", type6="IPType") myLog.parse(myQuery) myLog.store(self.log_preset_two)
def test01CreatePreset(self): """ Test that Simple Presets can be created """ dbh = DB.DBO(self.test_case) log = SimpleLog(case=self.test_case) query = query_type(datafile=self.test_file, log_preset=self.log_preset, delimiter="\s+", field0="Time", field1="Protocol", field2="SourceIP", field3="SourcePort", field4="DestIP", field5="DestPort", field6="Bytes", type0="EpochTimestamp", type1="IntegerType", type2="IPType", type3="IntegerType", type4="IPType", type5="IntegerType", type6="IntegerType", index0="yes", index1="yes", index2="yes", index4="yes", index5="yes") log.parse(query) log.store(self.log_preset)
def display(self, query, result): dbh = DB.DBO(query['case']) dbh.execute( "select unix_timestamp(create_time) as time from INFORMATION_SCHEMA.TABLES where table_schema = %r and table_name = 'top_talkers'", query['case']) last_updated = dbh.fetch() if not last_updated or last_updated['time'] + 3600 < time.time(): dbh.execute("drop table if exists top_talkers") dbh.execute( "create table top_talkers select dest_ip,sum(length) as bytes from connection join connection_details using(inode_id) group by dest_ip" ) #__target__ = "filter", link = FlagFramework.query_type( case=query['case'], family='Network Forensics', report='/Network Forensics/View Connections', __target__='filter', __target_format__="'Destination IP'='%s'") result.table(elements=[ IPType('Destination IP', 'dest_ip', link=link), IntegerType('Bytes Transfered', 'bytes') ], table='top_talkers', case=query['case'], order=1, direction=0)
def test02LoadFile(self): """ Test that IIS Log files can be loaded """ dbh = DB.DBO(self.test_case) log = LogFile.load_preset(self.test_case, self.log_preset, [self.test_file]) t = time.time() ## Load the data: for a in log.load(self.test_table): pass print "Took %s seconds to load log" % (time.time() - t) ## Check that all rows were uploaded: dbh.execute("select count(*) as c from `%s_log`", self.test_table) row = dbh.fetch() self.assertEqual(row['c'], 8334) ## More specific tests dbh.execute( "select count(*) as c from `%s_log` where `IP Address` = 2921232307", self.test_table) row = dbh.fetch() self.assertEqual(row['c'], 129) dbh.execute( "select count(*) as c from `%s_log` where cs_username = '******'", self.test_table) row = dbh.fetch() self.assertEqual(row['c'], 7898)
def find_fields_line(self): # Find the fields line: count = 0 fields = None for row in self.read_record(ignore_comment=False): count += 1 if row.startswith('#Fields: '): dbh = DB.DBO() fields = [dbh.MakeSQLSafe(i) for i in row.split()[1:]] # Coallesc the date and time field together: try: i = fields.index('date') del fields[i] except ValueError: pass break ## couldnt we find the field header? if count > 15: break if not fields: raise RuntimeError( "Error parsing IIS log file (I can't find a #Fields header line.)\nMaybe you may be able to use the Simple or Advanced log driver for this log?" ) return fields
def execute(self): if len(self.args) < 2: yield self.help() return pdbh = DB.DBO() pdbh.mass_insert_start('jobs') cookie = int(time.time()) scanners = [] for i in range(1, len(self.args)): scanners.extend( fnmatch.filter(Registry.SCANNERS.scanners, self.args[i])) for path in self.glob_files(self.args[:1]): try: path, inode, inode_id = self.environment._FS.lookup(path=path) except Exception, e: continue ## This is a cookie used to identify our requests so that we ## can check they have been done later. pdbh.mass_insert( command='Scan', arg1=self.environment._CASE, arg2=inode_id, arg3=','.join(scanners), cookie=cookie, )
def test01CreatePreset(self): """ Test that EventLog Presets can be created """ dbh = DB.DBO(self.test_case) log = EventLogLog(case=self.test_case) query = query_type(datafile=self.test_file, log_preset=self.log_preset) log.parse(query) log.store(self.log_preset)
def display(self, query, result): path = query['path'] key = query['key'] result.heading("Registry Key Contents") result.text(DB.expand("Key %s/%s:", (path, key)), style='red', font='typewriter') dbh = DB.DBO(query['case']) def hexdump(query, out): """ Show the hexdump for the key """ dbh.execute( "select value from reg where path=%r and reg_key=%r limit 1", (path, key)) row = dbh.fetch() if row: HexDump(row['value'], out).dump() return out def strings(query, out): """ Draw the strings in the key """ out.para("not implimented yet") return out def stats(query, out): """ display stats on a key """ out.para("not implemented yet") return out result.notebook(names=["HexDump", "Strings", "Statistics"], callbacks=[hexdump, strings, stats], context="display_mode")
def test01RunScanner(self): """ Running scanners """ ## Populate the key words into the dictionary: dbh = DB.DBO() for row in self.case_sensitive_keywords: id = row[0] w = row[1] dbh.delete('dictionary', 'id=%r' % (id + 1000), _fast=True) dbh.insert('dictionary', _fast=True, **{ 'id': id + 1000, 'class': "DFTT", 'type': 'literal', 'word': w }) for row in self.regex_keywords: id = row[0] w = row[1] dbh.delete('dictionary', 'id=%r' % (id + 1000), _fast=True) dbh.insert('dictionary', _fast=True, **{ 'id': id + 1000, 'class': "DFTT", 'type': 'regex', 'word': w }) env = pyflagsh.environment(case=self.test_case) pyflagsh.shell_execv(env=env, command="scan", argv=["*", 'IndexScan'])
def analyse(self, query): context = self.get_context(query) word_id = Indexing.insert_dictionary_word(query['word'], query['type']) pdbh = DB.DBO() sql = DB.expand("select inode.inode_id as `inode_id` "\ "%s where (%s) and (%s)", (context.get('tables',''), context.get('inode_sql','1'), context.get('where','1'))) Indexing.schedule_inode_index_sql(query['case'], sql, word_id, query['cookie'], unique=True) ## Now wait here until everyone is finished: while 1: pdbh.execute("select count(*) as c from jobs where cookie=%r", query['cookie']) row = pdbh.fetch() self.rows_left = row['c'] if row['c'] == 0: break time.sleep(1) return 1
def startup(self): print "Checking schema for compliance" ## Make sure that the schema conforms dbh = DB.DBO() dbh.execute("select value from meta where property='flag_db'") DB.check_column_in_table(None, 'sql_cache', 'status', 'enum("progress","dirty","cached")') for row in dbh: try: DB.check_column_in_table(row['value'], 'sql_cache', 'status', 'enum("progress","dirty","cached")') except: continue ## Check the schema: dbh.check_index("jobs", "state") DB.check_column_in_table(None, 'jobs', 'priority', 'int default 10') DB.check_column_in_table(None, 'jobs', 'pid', 'int default 0') DB.check_column_in_table(None, 'jobs', 'when_valid', 'TIMESTAMP ON UPDATE CURRENT_TIMESTAMP NOT NULL') ## Check for the high_priority_jobs table (its basically ## another jobs table for high priority jobs - so workers ## first check this table before the main jobs table). try: dbh.execute("select * from high_priority_jobs limit 1") except: dbh.execute("create table if not exists high_priority_jobs like jobs") ## Schedule the first periodic task: task = Periodic() task.run()
def render_html(self, inode_id, table_renderer): dbh = DB.DBO() case = table_renderer.case dbh.insert("jobs", command="Export", arg1=case, arg2=inode_id, cookie=int(time.time())) filename, content_type, fd = table_renderer.make_archive_filename(inode_id) result = "<a href='%s'>%s</a><br />" % (filename, fd.inode) try: filename = "inodes/%s_summary.html" % inode_id fd.html_export result += "<a href='%s'><img src=images/browse.png /></a>" % ( filename, ) except AttributeError: pass #if table_renderer.explain_inodes: ## Add a link to the explaination page: filename = "inodes/%s_explain.html" % inode_id result += "<a href='%s'><img src=images/question.png /></a>" % (filename, ) ## Check if there are annotations for this dbh = DB.DBO(case) dbh.execute("select * from annotate where inode_id=%r", inode_id) for row in dbh: result += "<br>%s" % row['note'] return result
def execute(self): start_time = time.time() report, query = self.prepare() pyflaglog.log(pyflaglog.VERBOSE_DEBUG, "Flash will execute the following query: %s" % query) ## Instantiate the report report = report(self.environment._flag) if self.environment._flag.is_cached(query): ## Run the display method result = TEXTUI.TEXTUI(query=query) report.display(query, result) yield result.display() return ## Execute the report: try: report.analyse(query) try: dbh = DB.DBO(query['case']) except KeyError: dbh = DB.DBO() canonical_query = FlagFramework.canonicalise(query) ## We call the display method just in case this report ## does something in the display result = TEXTUI.TEXTUI(query=query) report.display(query, result) yield result.display() dbh.execute("insert into meta set property=%r,value=%r", ('report_executed', canonical_query)) yield "Execution of %s successful in %s sec" % ( self.args[1], time.time() - start_time) pyflaglog.log( pyflaglog.VERBOSE_DEBUG, "Flash successfully ran the following query: %s" % query) except Exception, e: pyflaglog.log( pyflaglog.WARNING, "Flash encountered the following error: %s when running query: %s" % (e, query)) print FlagFramework.get_bt_string(e) raise raise RuntimeError("%s: %s after %s sec" % (sys.exc_info()[0], sys.exc_info()[1], time.time() - start_time))