def pane_cb(path, result):
query['order'] = 'Filename'
if path == '': path = '/'
## If we are asked to show a file, we will show the
## contents of the directory the file is in:
fsfd = FileSystem.DBFS(query["case"])
if not fsfd.isdir(path):
path = os.path.dirname(path)
self.make_table_widget(
['URN', 'Name', 'Size', 'Modified'],
query,
result,
where=DB.expand(
"path=%r and (isnull(type) or type!='directory')",
(path)),
)
result.toolbar(text=DB.expand("Scan %s", path),
icon="examine.png",
link=query_type(family="Load Data",
report="ScanFS",
path=path,
case=query['case']),
pane='popup')
def test02catTests(self):
""" Test the cat command """
self.env = pyflagsh.environment(case=self.test_case)
pyflagsh.shell_execv(env=self.env,
command="load",
argv=[
self.test_case,
])
self.fsfd = FileSystem.DBFS(self.test_case)
fd = self.fsfd.open("/dscf1080.jpg")
data1 = fd.read()
fd = self.fsfd.open("/dscf1081.jpg")
data2 = fd.read()
fd = self.fsfd.open("/dscf1082.jpg")
data3 = fd.read()
result = ''
for l in pyflagsh.shell_execv_iter(env=self.env,
command="cat",
argv=["/dscf1081.jpg"]):
result += l
self.assertEqual(result, data2)
result = ''
for l in pyflagsh.shell_execv_iter(env=self.env,
command="cat",
argv=["/dscf108*"]):
result += l
self.assertEqual(len(result), len(data1) + len(data2) + len(data3))
self.assert_(result == data1 + data2 + data3)
def pane_cb(path, tmp):
query['order'] = 'Filename'
## If we are asked to show a file, we will show the
## contents of the directory the file is in:
fsfd = FileSystem.DBFS(query["case"])
if not fsfd.isdir(path):
path = os.path.dirname(path)
tmp.table(
elements=[
InodeIDType(case=query['case']),
FilenameType(basename=True, case=query['case']),
DeletedType(),
IntegerType('File Size', 'size'),
TimestampType('Last Modified', 'mtime'),
StringType('Mode', 'mode', table='file')
],
table='inode',
where=DB.expand("file.path=%r and file.mode!='d/d'",
(path + '/')),
case=query['case'],
pagesize=10,
filter="filter2",
)
target = tmp.defaults.get('open_tree', '/')
tmp.toolbar(text=DB.expand("Scan %s", target),
icon="examine.png",
link=query_type(family="Load Data",
report="ScanFS",
path=target,
case=query['case']),
pane='popup')
def display(self, query, result):
new_q = result.make_link(query, '')
if not query.has_key('limit'): query['limit'] = 0
dbh = self.DBO(query['case'])
fsfd = FileSystem.DBFS(query["case"])
## If this is a directory, only show the stats
if query.has_key('inode_id'):
fd = fsfd.open(inode_id=query['inode_id'])
fd.inode_id = query['inode_id']
query['inode'] = fd.inode
else:
fd = fsfd.open(inode=query['inode'])
if not fd: return
image = Graph.Thumbnailer(fd, 300)
## Make a series of links to each level of this inode - this
## way we can view parents of this inode.
tmp = result.__class__(result)
tmp.text("Viewing file in inode ", make_inode_link(fd, query, result))
result.heading(tmp)
try:
result.text("Classified as %s by magic" % image.GetMagic())
except IOError, e:
result.text("Unable to classify file, no blocks: %s" % e)
image = None
def tree_cb(path):
## We expect a directory here:
if not path.endswith('/'): path = path + '/'
## We need to wait for this directory to load:
case = query['case']
dbh = DB.DBO(case)
count = 0
while 1:
dbh.execute("select * from file where path=%r and name=''",
path)
if dbh.fetch(): break
count += 1
time.sleep(1)
## FIXME: what is a good time to decide when to give up?
if count > 3600:
raise RuntimeError("Unable to load the filesystem?")
## We need a local copy of the filesystem factory so
## as not to affect other instances!!!
fsfd = FileSystem.DBFS(query["case"])
for i in fsfd.dent_walk(path):
if i['mode'] == "d/d" and i['status'] == 'alloc':
yield (([i['name'], i['name'], 'branch']))
def tree_cb(path):
fsfd = FileSystem.DBFS(query['case'])
query.default("path", '/')
dirs = []
for i in fsfd.longls(path):
if i['type'] == "directory":
dirs.append(i['name'])
yield (([i['name'], i['name'], 'branch']))
def tree_cb(path):
fsfd = FileSystem.DBFS(query['case'])
query.default("path",'/')
if not path.endswith('/'): path=path+'/'
dirs = []
for i in fsfd.dent_walk(path):
if i['mode']=="d/d" and i['status']=='alloc' and i['name'] not in dirs:
dirs.append(i['name'])
yield(([i['name'],i['name'],'branch']))
def tree_cb(path):
## We expect a directory here:
if not path.endswith('/'): path = path + '/'
## We need a local copy of the filesystem factory so
## as not to affect other instances!!!
fsfd = FileSystem.DBFS(query["case"])
dirs = []
for i in fsfd.dent_walk(path):
if i['mode'] == "d/d" and i['status'] == 'alloc' and i[
'name'] not in dirs:
dirs.append(i['name'])
yield (([i['name'], i['name'], 'branch']))
def check_for_file(self, sql='1'):
dbh = DB.DBO(self.test_case)
dbh.execute(
"select inode.inode as inode,type.type,path,name from file,type,inode where file.inode_id=type.inode_id and inode.inode_id=type.inode_id and type.type like '%%JPEG%%' and %s limit 1",
sql)
row = dbh.fetch()
if not row: return None
## Check that its a real file:
fsfd = FileSystem.DBFS(self.test_case)
fd = fsfd.open(inode=row['inode'])
data = fd.read()
if len(data) == 0:
raise IOError("Can not read file %s%s (%s) %r" %
(row['path'], row['name'], row['inode'], data))
return row
def pane_cb(path,tmp):
fsfd = FileSystem.DBFS( query["case"])
if not fsfd.isdir(path):
path=posixpath.dirname(path)
new_query = make_new_query(query, path + '/')
tmp.table(
elements = [ InodeIDType(case=query['case']),
FilenameType(basename=True, case=query['case'],
link = new_query,
link_pane = 'parent'),
IntegerType('File Size','size'),
],
table='inode',
where=DB.expand("file.path=%r and file.mode!='d/d'", (path+'/')),
case=query['case'],
pagesize=10,
filter="filter2",
)
def display(self, query, result):
new_q = result.make_link(query, '')
if not query.has_key('limit'): query['limit'] = 0
dbh = self.DBO(query['case'])
fsfd = FileSystem.DBFS(query["case"])
## If this is a directory, only show the stats
fd = fsfd.open(inode_id=query['inode_id'])
if not fd: return
tmp = result.__class__(result)
tmp.text(fd.urn)
result.heading(tmp)
try:
m = Magic.MagicResolver()
type, mime = m.find_inode_magic(query['case'], fd.inode_id)
result.text("Classified as %s by magic" % type)
except IOError, e:
result.text("Unable to classify file, no blocks: %s" % e)
def test02TestOutput(self):
""" Testing output """
dbh = DB.DBO(self.test_case)
fsfd = FileSystem.DBFS(self.test_case)
dbh.execute(
"select inode_id, word_id, word,offset,length from LogicalIndexOffsets join %s.dictionary on LogicalIndexOffsets.word_id=%s.dictionary.id where id>1000 and id<1020",
(config.FLAGDB, config.FLAGDB))
for row in dbh:
patg, inode, inode_id = fsfd.lookup(inode_id=row['inode_id'])
fd = fsfd.open(inode=inode)
fd.overread = True
fd.slack = True
fd.seek(row['offset'])
data = fd.read(row['length'])
filename, inode, inode_id = fsfd.lookup(inode=inode)
print "Looking for %s: Found in %s (%s) at offset %s length %s %r" % (
row['word'], filename, inode, row['offset'], row['length'],
data)
#self.assertEqual(data.lower(), row['word'].lower())
self.find_expected_output(row['word'], row['word_id'] - 1000,
filename, row['offset'],
self.case_sensitive_keywords, data)
print "Left over %s" % self.case_sensitive_keywords
def tree_cb(path):
fsfd = FileSystem.DBFS(query["case"])
for i in fsfd.longls(path):
if i['type'] == 'directory':
yield (([i['name'], i['name'], 'branch']))
def open(self):
fsfd = FileSystem.DBFS(self.name)
return fsfd.open(path = self.path)
class RevEng_GUI(Reports.report):
""" Allows us to manipulate data structures in reverse engineering efforts """
name = "DAFT"
family = "Misc"
description = "Data Analysis Facilitation Tool (Reverse Engineering)"
# parameters = { "foo": "any"}
def analyse(self, query):
pass
def display(self, query, result):
result.start_form(query)
def settings_cb(query, ui):
ui.decoration = "naked"
try:
if query['finish'] and query['MaxRows'] and query[
'StartOffset']:
del query['finish']
del query['submit']
ui.refresh(0, query, parent=1)
except KeyError:
pass
ui.start_form(query)
ui.start_table()
ui.textfield("Starting Offset", "StartOffset", size=20)
ui.textfield("Maximum Rows", "MaxRows")
ui.checkbox("Click here to finish", "finish", "yes")
ui.end_table()
ui.end_form()
return ui
def popup_cb(query, ui, column_number=None, mode=''):
"""Popup for defining column attributes"""
## print "I am here"
ui.decoration = "naked"
if mode == 'insert':
pre = 'insert_'
else:
pre = ''
try:
if query['finish'] and query['%sname_%s' %
(pre, column_number)]:
del query['finish']
del query['submit']
ui.refresh(0, query, parent=1)
except KeyError:
pass
ui.start_form(query)
ui.start_table()
if mode == 'insert':
ui.heading("Inserting Column number %s" % column_number)
else:
ui.heading("Column number %s" % column_number)
names = [
x.__name__ for x in Registry.FILEFORMATS.classes if x.visible
]
ui.textfield("Name for this field",
"%sname_%s" % (pre, column_number))
ui.const_selector("Data Type",
'%sdata_type_%s' % (pre, column_number), names,
names)
try:
temp = Registry.FILEFORMATS[query['%sdata_type_%s' %
(pre, column_number)]]("",
None)
ui.row("Description", temp.__doc__)
temp.form("%sparameter_%s_" % (pre, column_number), query, ui)
except KeyError, e:
print 'KeyError: %s' % e
ui.checkbox("Visible",
"%svisible_%s" % (pre, column_number),
"yes",
checked=True)
ui.checkbox("Click here to finish", "finish", "yes")
ui.end_table()
ui.end_form()
return ui
def delete_col_cb(query, ui, column_number=None):
"""Popup to confirm deletion of column"""
ui.decoration = "naked"
ui.heading("Delete column number %s?" % column_number)
try:
if query['submit']:
del query['submit']
ui.refresh(0, query, parent=1)
except KeyError:
pass
ui.start_form(query)
ui.checkbox("Click here to delete", "delete_%s" % column_number,
"yes")
ui.end_form()
return ui
def processquery(query):
delcol = -1
insvalues = {}
for k in query.keys():
if k.startswith('delete_'):
delcol = int(k[7:])
del query[k]
break
elif k.startswith('insert_'):
insvalues[k[7:]] = query[k]
if k.startswith('insert_name_'):
inscol = int(k[12:])
del query[k]
continue
elif k.startswith('savenow'):
savelayout(query)
del query[k]
break
elif k.startswith('loadlayout'):
openlayout(query)
del query[k]
### other stuff for ins col parameters
if delcol >= 0:
count = delcol
while 1:
try:
query['name_%s' % count]
count += 1
except KeyError:
break
for i in range(delcol + 1, count):
del query['name_%s' % (i - 1)]
del query['data_type_%s' % (i - 1)]
del query['visible_%s' % (i - 1)]
params = [
k for k in query.keys()
if k.startswith('parameters_%s_' % (i - 1))
]
for parameter in params:
del query[parameter]
query['name_%s' % (i - 1)] = query['name_%s' % i]
query['data_type_%s' % (i - 1)] = query['data_type_%s' % i]
query['visible_%s' % (i - 1)] = query['visible_%s' % i]
key = 'parameter_'
params = [
k[11 + len('%s' % i):] for k in query.keys()
if k.startswith('%s%s_' % (key, i))
]
for parameter in params:
query['%s%s_%s' %
(key, (i - 1), parameter)] = query['%s%s_%s' %
(key, i,
parameter)]
del query['name_%s' % (count - 1)]
del query['data_type_%s' % (count - 1)]
del query['visible_%s' % (count - 1)]
params = [
k for k in query.keys()
if k.startswith('parameter_%s_' % (count - 1))
]
for parameter in params:
del query[params]
elif len(insvalues) > 0:
count = inscol
while 1:
try:
query['name_%s' % count]
count += 1
except KeyError:
break
for i in range(count, inscol, -1):
query['name_%s' % i] = query['name_%s' % (i - 1)]
query['data_type_%s' % i] = query['data_type_%s' % (i - 1)]
query['visible_%s' % i] = query['visible_%s' % (i - 1)]
key = 'parameter_'
params = [
k[11 + len('%s' % (i - 1)):] for k in query.keys()
if k.startswith('%s%s_' % (key, (i - 1)))
]
for parameter in params:
query['%s%s_%s' %
(key, i, parameter)] = query['%s%s_%s' %
(key, (i - 1),
parameter)]
del query['name_%s' % (i - 1)]
del query['data_type_%s' % (i - 1)]
del query['visible_%s' % (i - 1)]
params = [
k for k in query.keys()
if k.startswith('parameter_%s_' % (i - 1))
]
for parameter in params:
del query[parameter]
for k in insvalues.keys():
query[k] = insvalues[k]
def open_cb(query, ui):
"""Popup for loading a layout"""
ui.decoration = "naked"
dbh = self.DBO(query['case'])
try:
if query['finish']:
del query['finish']
del query['submit']
ui.refresh(0, query, parent=1)
except KeyError:
pass
ui.start_form(query)
ui.start_table()
ui.heading("Load a saved layout")
try:
dbh.execute('select name from DAFTLayouts')
rows = []
for row in dbh:
rows.append(row['name'])
ui.const_selector("Layout Name", "loadlayout", rows, rows)
except DB.DBError:
dbh.execute(
'create table DAFTLayouts (`name` text, `layout` text)')
ui.const_selector("Layout Name", "loadlayout", [''], [''])
ui.checkbox("Click here to finish", "finish", "yes")
ui.end_table()
ui.end_form()
return ui
### Create a list of saved layouts
def openlayout(query):
dbh = self.DBO(query['case'])
keylist = [
'name_', 'data_type_', 'visible_', 'parameter_', 'fileselect',
'MaxRows', 'StartOffset', 'savelayout'
]
try:
dbh.execute("select layout from DAFTLayouts where name='%s'" %
query['loadlayout'])
except DB.DBError:
pass
rows = []
oldkeys = []
for row in dbh:
rows.append(row)
try:
if len(rows) < 1:
raise ValueError
for k in keylist:
oldkeys = [x for x in query.keys() if x.startswith(k)]
for key in oldkeys:
del query[key]
for kvpair in rows[0]['layout'].split(','):
key, value = kvpair.split('=')
query[key] = value
except ValueError:
pass
def save_cb(query, ui):
"""Popup for saving layout"""
ui.decoration = "naked"
try:
if query['finish'] and query['savelayout']:
query['savenow'] = 'yes'
del query['finish']
del query['submit']
ui.refresh(0, query, parent=1)
except KeyError:
pass
ui.start_form(query)
ui.start_table()
ui.heading("Save current layout")
ui.textfield("Layout name", "savelayout")
ui.checkbox("Click here to finish", "finish", "yes")
ui.end_table()
ui.end_form()
return ui
def savelayout(query):
"""Saves the current layout into the DAFTLayouts table"""
dbh = self.DBO(query['case'])
keylist = [
'name_', 'data_type_', 'visible_', 'parameter_', 'fileselect',
'MaxRows', 'StartOffset', 'savelayout'
]
try:
dbh.execute(
"select name, layout from DAFTLayouts where name='%s'" %
query['savelayout'])
except DB.DBError:
dbh.execute(
"create table DAFTLayouts (`name` text, `layout` text)")
dbh.execute(
"select name, layout from DAFTLayouts where name='%s'" %
query['savelayout'])
rows = []
keys = []
for row in dbh:
rows.append(row)
for k in keylist:
keys = keys + [x for x in query.keys() if x.startswith(k)]
value = ','.join('%s=%s' % (x, query[x]) for x in keys)
if rows == []:
dbh.execute(
"insert into DAFTLayouts set name='%s', layout='%s'" %
(query['savelayout'], value))
else:
dbh.execute(
"update DAFTLayouts set layout='%s' where name='%s'" %
(value, query['savelayout']))
def filelist_cb(query, ui):
"""Popup to select files to analyse"""
ui.decoration = "naked"
try:
if query['finish'] and query['fileselect']:
del query['finish']
del query['submit']
ui.refresh(0, query, parent=1)
except KeyError:
pass
ui.start_form(query)
ui.start_table()
ui.heading("Select files")
values = []
keys = []
dbh.execute("select name, path, inode from file where inode != ''")
for row in dbh:
values.append('%s%s' % (row['path'], row['name']))
keys.append(row['inode'])
ui.const_selector("File", "fileselect", keys, values)
ui.checkbox("Click here to finish", "finish", "yes")
ui.end_table()
ui.end_form()
def render_HTMLUI(data):
"""Callback to render mysql stored data in HTML"""
tmp = result.__class__(result)
tmp.result = data
return tmp
##### Display starts here
try:
result.heading("Data Analysis Facilitation Tool")
dbh = DB.DBO(query['case'])
processquery(query)
## Build a struct to work from:
try:
startoffset = DAFTFormats.numeric(query['StartOffset'])
except KeyError:
startoffset = 0
try:
maxrows = DAFTFormats.numeric(query['MaxRows'])
except KeyError:
maxrows = 10
fsfd = FileSystem.DBFS(query["case"])
try:
fd = fsfd.open(inode=query['fileselect'])
fdsize = fsfd.istat(inode=query['fileselect'])
fd.block_size = dbh.get_meta('block_size')
buf = format.Buffer(fd=fd)[startoffset:]
except IOError, e:
print 'IOError: %s' % e
fd = None
fdsize = 0
s = '\x00' * 1024
buf = format.Buffer(string=s)
except KeyError, e:
print 'KeyError: %s' % e
s = '\x00' * 1024
buf = format.Buffer(string=s)
fd = None
fdsize = 0
