def test01RunScanners(self):
""" Running Logical Index Scanner """
## Make sure the word secret is in there.
pdbh = DB.DBO()
pdbh.execute("select * from dictionary where word='secret' limit 1")
row = pdbh.fetch()
if not row:
pdbh.insert('dictionary', **{'word':'secret', 'class':'English', 'type':'word'})
env = pyflagsh.environment(case=self.test_case)
pyflagsh.shell_execv(env=env, command="scan",
argv=["*",'IndexScan'])
dbh = DB.DBO(self.test_case)
dbh2 = DB.DBO(self.test_case)
fsfd = DBFS(self.test_case)
dbh.execute("select inode_id, word,offset,length from LogicalIndexOffsets join %s.dictionary on LogicalIndexOffsets.word_id=%s.dictionary.id where word='secret'", (config.FLAGDB,config.FLAGDB))
count = 0
for row in dbh:
count += 1
path, inode, inode_id = fsfd.lookup(inode_id = row['inode_id'])
fd = fsfd.open(inode=inode)
fd.overread = True
fd.slack = True
fd.seek(row['offset'])
data = fd.read(row['length'])
print "Looking for %s: Found in %s at offset %s length %s %r" % (
row['word'], inode, row['offset'], row['length'],data)
self.assertEqual(data.lower(), row['word'].lower())
## Did we find all the secrets?
self.assertEqual(count,2)
def execute(self):
if len(self.args) < 2:
yield self.help()
return
## Try to glob the inode list:
dbh = DB.DBO(self.environment._CASE)
dbh.execute(
"select inode_id from vfs where !isnull(inode_id) and path rlike %r",
(fnmatch.translate(self.args[0])))
pdbh = DB.DBO()
pdbh.mass_insert_start('jobs')
## This is a cookie used to identify our requests so that we
## can check they have been done later.
cookie = time.time()
scanners = []
for i in range(1, len(self.args)):
scanners.extend(
fnmatch.filter(Registry.SCANNERS.scanners, self.args[i]))
scanners = ScannerUtils.fill_in_dependancies(scanners)
for row in dbh:
Scanner.scan_inode_distributed(dbh.case,
row['inode_id'],
scanners,
cookie=cookie)
self.wait_for_scan(cookie)
yield "Scanning complete"
def execute(self):
try:
dbh = DB.DBO()
except:
dbh = DB.DBO('mysql')
dbh.execute("create database `%s`" % config.FLAGDB)
dbh = DB.DBO()
FlagFramework.post_event("init_default_db", None)
yield "Done"
def startup(self):
## Check to see if the nsrl db exists
try:
dbh = DB.DBO(config.HASHDB)
dbh.execute("select * from meta limit 1")
dbh.fetch()
except Exception, e:
try:
dbh = DB.DBO()
self.init_default_db(dbh, None)
except:
pass
def execute(self):
scanners = []
if len(self.args) < 2:
yield self.help()
return
elif type(self.args[1]) == types.ListType:
scanners = self.args[1]
else:
for i in range(1, len(self.args)):
scanners.extend(
fnmatch.filter(Registry.SCANNERS.scanners, self.args[i]))
## Assume that people always want recursive - I think this makes sense
path = self.args[0]
if not path.endswith("*"):
path = path + "*"
## FIXME For massive images this should be broken up, as in the old GUI method
dbh = DB.DBO(self.environment._CASE)
dbh.execute(
"select inode.inode from inode join file on file.inode = inode.inode where file.path rlike %r",
fnmatch.translate(path))
pdbh = DB.DBO()
pdbh.mass_insert_start('jobs')
## This is a cookie used to identify our requests so that we
## can check they have been done later.
cookie = int(time.time())
for row in dbh:
inode = row['inode']
pdbh.mass_insert(
command='Scan',
arg1=self.environment._CASE,
arg2=row['inode'],
arg3=','.join(scanners),
cookie=cookie,
) #
pdbh.mass_insert_commit()
## Wait for the scanners to finish:
self.wait_for_scan(cookie)
yield "Scanning complete"
def execute(self):
if len(self.args) < 2:
yield self.help()
return
pdbh = DB.DBO()
pdbh.mass_insert_start('jobs')
cookie = int(time.time())
scanners = []
for i in range(1, len(self.args)):
scanners.extend(
fnmatch.filter(Registry.SCANNERS.scanners, self.args[i]))
for path in self.glob_files(self.args[:1]):
path, inode, inode_id = self.environment._FS.lookup(path=path)
## This is a cookie used to identify our requests so that we
## can check they have been done later.
pdbh.mass_insert(
command='Scan',
arg1=self.environment._CASE,
arg2=inode,
arg3=','.join(scanners),
cookie=cookie,
)
pdbh.mass_insert_commit()
## Wait for the scanners to finish:
if 1 or self.environment.interactive:
self.wait_for_scan(cookie)
yield "Scanning complete"
def CaseTableTests(self, tablename):
## Apply each column's test filters:
t = Registry.CASE_TABLES.dispatch(tablename)()
result = HTMLUI.HTMLUI(initial=True)
dbh = DB.DBO(self.test_case)
## For each column run all its test cases:
elements = [c for c in t.bind_columns(self.test_case)]
## Create a renderer:
r = UI.TableRenderer(elements=elements, table=tablename)
for c in elements:
for operator, arg, e in c.tests:
try:
## Get the SQL:
r.filter_str = "'%s' %s '%s'" % (c.name, operator, arg)
query = FlagFramework.query_type(direction=1)
sql = r._make_sql(query)
print "%s: Testing %s: %s" % (tablename, c.__class__,
r.filter_str)
dbh.execute(sql + " limit 1")
dbh.fetch()
except Exception:
if not e: raise
continue
if e:
raise Exception(
"Expected an exception but did not receive one on filter string %s. SQL was %s"
% (r.filter_str, sql))
def test01Mounted(self):
""" Test that mounted images work """
## We should be able to see test_file in the file table:
dbh=DB.DBO(self.test_case)
dbh.execute("select * from file where name = %r",self.test_file)
row = dbh.fetch()
self.assert_(row,"Unable to find the file in the VFS???")
def start_workers():
if config.FLUSH:
dbh = DB.DBO()
pyflaglog.log(pyflaglog.WARNING,
"Deleting job queue and killing workers")
#dbh.execute("select max(id) as max from jobs")
#row = dbh.fetch()
#broadcast_id = row['max'] or 0
dbh.execute("delete from jobs")
#dbh.insert("jobs", _fast=True,
# command='Exit', state='broadcast',
# )
if config.WORKERS == 0:
return
for i in range(config.WORKERS):
try:
r, w = os.pipe()
pid = os.fork()
except AttributeError:
## When running under windows we can not fork... We must
## launch this module by itself instead - this is very
## suboptimal because we will be performing all startup
## code (registry parsing etc) for each worker. If you want
## performance you would not choose windows anyway,
## though. The following is windows specific:
## First find the name of the interpreter:
import ctypes, sys
name = ctypes.create_string_buffer(255)
length = ctypes.windll.kernel32.GetModuleFileNameA(None, name, 255)
interpreter = name.raw[:length]
## This encloses at least the file path in quotes just in
## case we are installed to somewhere with spaces - It
## seems that on windows argvs are not processed correctly
## because the below array ends up as a single command line
## string WTF? This is very dodgy...
os.spawnv(
os.P_NOWAIT, interpreter,
['"%s"' % interpreter, '"%s"' % __file__] + sys.argv[1:])
pid = 1
## Parents:
if pid:
os.close(r)
children.append(pid)
else:
os.close(w)
nanny(worker_run, keepalive=r)
atexit.register(terminate_children)
## The parent now calls the startup method on each of the events:
for event in Registry.EVENT_HANDLERS.classes:
try:
event().startup()
except Exception, e:
pyflaglog.log(pyflaglog.WARNING, "Error: %s" % e)
def tree_cb(path):
if path=='/':
dbh = DB.DBO(query['case'])
dbh.cached_execute("select tld from http where content_type like 'text/html%%' group by tld order by tld")
for row in dbh:
tld = row['tld']
yield ((tld,tld,'leaf'))
def test03FilteredLoad(self):
""" Test that filter expressions work when loading """
filter = ' "rhost" maxmind_country AUS and "status" = 404 '
## See if we can load the preset again:
log = LogFile.load_preset(self.test_case, self.log_preset, [self.datafile])
t = time.time()
table_name = self.test_table + "filtered"
## Load the data:
for a in log.load(table_name, filter=filter):
print a
print "Took %s seconds to load log" % (time.time()-t)
## How many rows were inserted
dbh = DB.DBO(self.test_case)
dbh.execute("select count(*) as c from `%s_log`", table_name)
row = dbh.fetch()
## Now try to enforce the same filter on the original table
## (which has all the rows in it):
for f in log.fields: f.table = self.test_table+"_log"
filter_sql = parser.parse_to_sql(filter, log.fields, None)
dbh.execute("select count(*) as c from `%s_log` where %s", (self.test_table, filter_sql))
row2 = dbh.fetch()
self.assertEqual(row['c'], row2['c'])
def casetable(self, field, query):
""" Checks that field is a table within the case given as query[case]. This is not a fatal error, we just return false if not. """
dbh = DB.DBO(query['case'])
try:
dbh.execute("select * from `%s_log` limit 1", query[field])
except DB.DBError:
return False
def run_analysis(self, report, query):
""" Run the analysis """
print query
try:
canonical_query = self.flag.canonicalise(query)
thread_name = threading.currentThread().getName()
print "Current thread is %s" % thread_name
try:
report.analyse(query)
print "analysed report"
except Exception, e:
gtk.gdk.threads_enter()
self.error_popup(e)
gtk.gdk.threads_leave()
return
dbh = DB.DBO(query['case'])
dbh.execute("insert into meta set property=%r,value=%r",
('report_executed', canonical_query))
## This thread must never touch GTK stuff or dead lock
## will occur. We must signal the other threads that we
## have finished analysis.
del self.running_threads[query.__str__()]
return
def test01CreatePreset(self):
""" Test that Advanced Presets can be created """
dbh = DB.DBO(self.test_case)
log = AdvancedLog(case=self.test_case)
query = query_type(datafile = self.test_file, log_preset=self.log_preset,
field_param_1_name="Timestamp",
field_param_1_column="time",
field_param_1_format="%b %d %H:%M:%S",
field_param_1_override_year="2007",
field_types1="TimestampType",
field_types2="StringType",
field_param_2_name = "Host",
field_param_2_column = 'host',
field_types3="StringType",
field_param_3_name ="Service",
field_param_3_column = "service",
field_types4="PadType",
field_param_4_regex = "(\[[^\]]+\])?",
field_types5="StringType",
field_param_5_name = "Message",
field_param_5_column = 'messages',
field_param_5_regex = '.*',
)
log.parse(query)
log.store(self.log_preset)
def wait_for_scan(self, cookie):
""" Waits for scanners to complete """
import pyflag.Farm as Farm
while Farm.get_cookie_reference(cookie) > 0:
time.sleep(0.5)
return
print "Waiting for cookie %s" % cookie
pdbh = DB.DBO()
## Often this process owns a worker as well. In that case we can wake it up:
import pyflag.Farm as Farm
#Farm.wake_workers()
## Wait until there are no more jobs left.
while 1:
pdbh.execute("select * from jobs where cookie=%r limit 1",
(cookie))
row = pdbh.fetch()
if not row: break
time.sleep(1)
def startup(self):
try:
dbh = DB.DBO()
dbh.check_index("whois_routes", "netmask")
dbh.check_index("whois_routes", "network")
except:
pass
def __init__(self, fsfd):
GenScanFactory.__init__(self, fsfd)
## Make sure we have indexes on the NSRL tables:
dbh_flag = DB.DBO(config.HASHDB)
dbh_flag.check_index("NSRL_hashes", "md5", 4)
dbh_flag.check_index("NSRL_products", "Code")
def test03CreateAnotherPreset(self):
""" Another setup test for Simple log file preset ... """
dbh = DB.DBO(self.test_case)
myLog = SimpleLog(case=self.test_case)
myQuery = query_type(datafile=self.test_file_two,
log_preset=self.log_preset_two,
delimiter=",",
field0="AString",
field1="SomeNumber",
field2="FirstIP",
field3="BString",
field4="SecondIP",
field5="OtherNumber",
field6="ThirdIP",
type0="StringType",
type1="IntegerType",
type2="IPType",
type3="StringType",
type4="IPType",
type5="IntegerType",
type6="IPType")
myLog.parse(myQuery)
myLog.store(self.log_preset_two)
def test01CreatePreset(self):
""" Test that Simple Presets can be created """
dbh = DB.DBO(self.test_case)
log = SimpleLog(case=self.test_case)
query = query_type(datafile=self.test_file,
log_preset=self.log_preset,
delimiter="\s+",
field0="Time",
field1="Protocol",
field2="SourceIP",
field3="SourcePort",
field4="DestIP",
field5="DestPort",
field6="Bytes",
type0="EpochTimestamp",
type1="IntegerType",
type2="IPType",
type3="IntegerType",
type4="IPType",
type5="IntegerType",
type6="IntegerType",
index0="yes",
index1="yes",
index2="yes",
index4="yes",
index5="yes")
log.parse(query)
log.store(self.log_preset)
def display(self, query, result):
dbh = DB.DBO(query['case'])
dbh.execute(
"select unix_timestamp(create_time) as time from INFORMATION_SCHEMA.TABLES where table_schema = %r and table_name = 'top_talkers'",
query['case'])
last_updated = dbh.fetch()
if not last_updated or last_updated['time'] + 3600 < time.time():
dbh.execute("drop table if exists top_talkers")
dbh.execute(
"create table top_talkers select dest_ip,sum(length) as bytes from connection join connection_details using(inode_id) group by dest_ip"
)
#__target__ = "filter",
link = FlagFramework.query_type(
case=query['case'],
family='Network Forensics',
report='/Network Forensics/View Connections',
__target__='filter',
__target_format__="'Destination IP'='%s'")
result.table(elements=[
IPType('Destination IP', 'dest_ip', link=link),
IntegerType('Bytes Transfered', 'bytes')
],
table='top_talkers',
case=query['case'],
order=1,
direction=0)
def test02LoadFile(self):
""" Test that IIS Log files can be loaded """
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset,
[self.test_file])
t = time.time()
## Load the data:
for a in log.load(self.test_table):
pass
print "Took %s seconds to load log" % (time.time() - t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from `%s_log`", self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 8334)
## More specific tests
dbh.execute(
"select count(*) as c from `%s_log` where `IP Address` = 2921232307",
self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 129)
dbh.execute(
"select count(*) as c from `%s_log` where cs_username = '******'",
self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 7898)
def find_fields_line(self):
# Find the fields line:
count = 0
fields = None
for row in self.read_record(ignore_comment=False):
count += 1
if row.startswith('#Fields: '):
dbh = DB.DBO()
fields = [dbh.MakeSQLSafe(i) for i in row.split()[1:]]
# Coallesc the date and time field together:
try:
i = fields.index('date')
del fields[i]
except ValueError:
pass
break
## couldnt we find the field header?
if count > 15:
break
if not fields:
raise RuntimeError(
"Error parsing IIS log file (I can't find a #Fields header line.)\nMaybe you may be able to use the Simple or Advanced log driver for this log?"
)
return fields
def execute(self):
if len(self.args) < 2:
yield self.help()
return
pdbh = DB.DBO()
pdbh.mass_insert_start('jobs')
cookie = int(time.time())
scanners = []
for i in range(1, len(self.args)):
scanners.extend(
fnmatch.filter(Registry.SCANNERS.scanners, self.args[i]))
for path in self.glob_files(self.args[:1]):
try:
path, inode, inode_id = self.environment._FS.lookup(path=path)
except Exception, e:
continue
## This is a cookie used to identify our requests so that we
## can check they have been done later.
pdbh.mass_insert(
command='Scan',
arg1=self.environment._CASE,
arg2=inode_id,
arg3=','.join(scanners),
cookie=cookie,
)
def test01CreatePreset(self):
""" Test that EventLog Presets can be created """
dbh = DB.DBO(self.test_case)
log = EventLogLog(case=self.test_case)
query = query_type(datafile=self.test_file, log_preset=self.log_preset)
log.parse(query)
log.store(self.log_preset)
def display(self, query, result):
path = query['path']
key = query['key']
result.heading("Registry Key Contents")
result.text(DB.expand("Key %s/%s:", (path, key)),
style='red',
font='typewriter')
dbh = DB.DBO(query['case'])
def hexdump(query, out):
""" Show the hexdump for the key """
dbh.execute(
"select value from reg where path=%r and reg_key=%r limit 1",
(path, key))
row = dbh.fetch()
if row:
HexDump(row['value'], out).dump()
return out
def strings(query, out):
""" Draw the strings in the key """
out.para("not implimented yet")
return out
def stats(query, out):
""" display stats on a key """
out.para("not implemented yet")
return out
result.notebook(names=["HexDump", "Strings", "Statistics"],
callbacks=[hexdump, strings, stats],
context="display_mode")
def test01RunScanner(self):
""" Running scanners """
## Populate the key words into the dictionary:
dbh = DB.DBO()
for row in self.case_sensitive_keywords:
id = row[0]
w = row[1]
dbh.delete('dictionary', 'id=%r' % (id + 1000), _fast=True)
dbh.insert('dictionary',
_fast=True,
**{
'id': id + 1000,
'class': "DFTT",
'type': 'literal',
'word': w
})
for row in self.regex_keywords:
id = row[0]
w = row[1]
dbh.delete('dictionary', 'id=%r' % (id + 1000), _fast=True)
dbh.insert('dictionary',
_fast=True,
**{
'id': id + 1000,
'class': "DFTT",
'type': 'regex',
'word': w
})
env = pyflagsh.environment(case=self.test_case)
pyflagsh.shell_execv(env=env, command="scan", argv=["*", 'IndexScan'])
def analyse(self, query):
context = self.get_context(query)
word_id = Indexing.insert_dictionary_word(query['word'], query['type'])
pdbh = DB.DBO()
sql = DB.expand("select inode.inode_id as `inode_id` "\
"%s where (%s) and (%s)", (context.get('tables',''),
context.get('inode_sql','1'),
context.get('where','1')))
Indexing.schedule_inode_index_sql(query['case'],
sql,
word_id,
query['cookie'],
unique=True)
## Now wait here until everyone is finished:
while 1:
pdbh.execute("select count(*) as c from jobs where cookie=%r",
query['cookie'])
row = pdbh.fetch()
self.rows_left = row['c']
if row['c'] == 0: break
time.sleep(1)
return 1
def startup(self):
print "Checking schema for compliance"
## Make sure that the schema conforms
dbh = DB.DBO()
dbh.execute("select value from meta where property='flag_db'")
DB.check_column_in_table(None, 'sql_cache', 'status',
'enum("progress","dirty","cached")')
for row in dbh:
try:
DB.check_column_in_table(row['value'], 'sql_cache', 'status',
'enum("progress","dirty","cached")')
except: continue
## Check the schema:
dbh.check_index("jobs", "state")
DB.check_column_in_table(None, 'jobs', 'priority', 'int default 10')
DB.check_column_in_table(None, 'jobs', 'pid', 'int default 0')
DB.check_column_in_table(None, 'jobs', 'when_valid',
'TIMESTAMP ON UPDATE CURRENT_TIMESTAMP NOT NULL')
## Check for the high_priority_jobs table (its basically
## another jobs table for high priority jobs - so workers
## first check this table before the main jobs table).
try:
dbh.execute("select * from high_priority_jobs limit 1")
except:
dbh.execute("create table if not exists high_priority_jobs like jobs")
## Schedule the first periodic task:
task = Periodic()
task.run()
def render_html(self, inode_id, table_renderer):
dbh = DB.DBO()
case = table_renderer.case
dbh.insert("jobs",
command="Export",
arg1=case,
arg2=inode_id,
cookie=int(time.time()))
filename, content_type, fd = table_renderer.make_archive_filename(inode_id)
result = "<a href='%s'>%s</a><br />" % (filename, fd.inode)
try:
filename = "inodes/%s_summary.html" % inode_id
fd.html_export
result += "<a href='%s'><img src=images/browse.png /></a>" % (
filename, )
except AttributeError:
pass
#if table_renderer.explain_inodes:
## Add a link to the explaination page:
filename = "inodes/%s_explain.html" % inode_id
result += "<a href='%s'><img src=images/question.png /></a>" % (filename, )
## Check if there are annotations for this
dbh = DB.DBO(case)
dbh.execute("select * from annotate where inode_id=%r", inode_id)
for row in dbh:
result += "<br>%s" % row['note']
return result
def execute(self):
start_time = time.time()
report, query = self.prepare()
pyflaglog.log(pyflaglog.VERBOSE_DEBUG,
"Flash will execute the following query: %s" % query)
## Instantiate the report
report = report(self.environment._flag)
if self.environment._flag.is_cached(query):
## Run the display method
result = TEXTUI.TEXTUI(query=query)
report.display(query, result)
yield result.display()
return
## Execute the report:
try:
report.analyse(query)
try:
dbh = DB.DBO(query['case'])
except KeyError:
dbh = DB.DBO()
canonical_query = FlagFramework.canonicalise(query)
## We call the display method just in case this report
## does something in the display
result = TEXTUI.TEXTUI(query=query)
report.display(query, result)
yield result.display()
dbh.execute("insert into meta set property=%r,value=%r",
('report_executed', canonical_query))
yield "Execution of %s successful in %s sec" % (
self.args[1], time.time() - start_time)
pyflaglog.log(
pyflaglog.VERBOSE_DEBUG,
"Flash successfully ran the following query: %s" % query)
except Exception, e:
pyflaglog.log(
pyflaglog.WARNING,
"Flash encountered the following error: %s when running query: %s"
% (e, query))
print FlagFramework.get_bt_string(e)
raise
raise RuntimeError("%s: %s after %s sec" %
(sys.exc_info()[0], sys.exc_info()[1],
time.time() - start_time))
