def test02catTests(self): """ Test the cat command """ self.env = pyflagsh.environment(case=self.test_case) pyflagsh.shell_execv(env=self.env, command="load", argv=[ self.test_case, ]) self.fsfd = FileSystem.DBFS(self.test_case) fd = self.fsfd.open("/dscf1080.jpg") data1 = fd.read() fd = self.fsfd.open("/dscf1081.jpg") data2 = fd.read() fd = self.fsfd.open("/dscf1082.jpg") data3 = fd.read() result = '' for l in pyflagsh.shell_execv_iter(env=self.env, command="cat", argv=["/dscf1081.jpg"]): result += l self.assertEqual(result, data2) result = '' for l in pyflagsh.shell_execv_iter(env=self.env, command="cat", argv=["/dscf108*"]): result += l self.assertEqual(len(result), len(data1) + len(data2) + len(data3)) self.assert_(result == data1 + data2 + data3)
def tree_cb(path): ## We expect a directory here: if not path.endswith('/'): path = path + '/' ## We need to wait for this directory to load: case = query['case'] dbh = DB.DBO(case) count = 0 while 1: dbh.execute("select * from file where path=%r and name=''", path) if dbh.fetch(): break count += 1 time.sleep(1) ## FIXME: what is a good time to decide when to give up? if count > 3600: raise RuntimeError("Unable to load the filesystem?") ## We need a local copy of the filesystem factory so ## as not to affect other instances!!! fsfd = FileSystem.DBFS(query["case"]) for i in fsfd.dent_walk(path): if i['mode'] == "d/d" and i['status'] == 'alloc': yield (([i['name'], i['name'], 'branch']))
def display(self, query, result): new_q = result.make_link(query, '') if not query.has_key('limit'): query['limit'] = 0 dbh = self.DBO(query['case']) fsfd = FileSystem.DBFS(query["case"]) ## If this is a directory, only show the stats if query.has_key('inode_id'): fd = fsfd.open(inode_id=query['inode_id']) fd.inode_id = query['inode_id'] query['inode'] = fd.inode else: fd = fsfd.open(inode=query['inode']) if not fd: return image = Graph.Thumbnailer(fd, 300) ## Make a series of links to each level of this inode - this ## way we can view parents of this inode. tmp = result.__class__(result) tmp.text("Viewing file in inode ", make_inode_link(fd, query, result)) result.heading(tmp) try: result.text("Classified as %s by magic" % image.GetMagic()) except IOError, e: result.text("Unable to classify file, no blocks: %s" % e) image = None
def pane_cb(path, result): query['order'] = 'Filename' if path == '': path = '/' ## If we are asked to show a file, we will show the ## contents of the directory the file is in: fsfd = FileSystem.DBFS(query["case"]) if not fsfd.isdir(path): path = os.path.dirname(path) self.make_table_widget( ['URN', 'Name', 'Size', 'Modified'], query, result, where=DB.expand( "path=%r and (isnull(type) or type!='directory')", (path)), ) result.toolbar(text=DB.expand("Scan %s", path), icon="examine.png", link=query_type(family="Load Data", report="ScanFS", path=path, case=query['case']), pane='popup')
def display(self, query, result): result.decoration = 'naked' self.case = query['case'] fsfd = FileSystem.DBFS(self.case) if query.has_key('inode_id'): fd = fsfd.open(inode_id=query['inode_id']) inode = fd.inode inode_id = query['inode_id'] elif query.has_key("sundry_id"): fd = FileSystem.File(query['case'], None, "HTTP%s" % query['sundry_id']) inode_id = -1 else: fd = fsfd.open(inode=query['inode']) inode_id = fd.lookup_id() content_type = self.guess_content_type(fd, query, inode_id) result.generator.content_type = content_type ## Now establish the dispatcher for it for k, v in self.dispatcher.items(): if k.search(content_type): return v(self, fd, result) return self.default_handler(fd, result)
def pane_cb(path, tmp): query['order'] = 'Filename' ## If we are asked to show a file, we will show the ## contents of the directory the file is in: fsfd = FileSystem.DBFS(query["case"]) if not fsfd.isdir(path): path = os.path.dirname(path) tmp.table( elements=[ InodeIDType(case=query['case']), FilenameType(basename=True, case=query['case']), DeletedType(), IntegerType('File Size', 'size'), TimestampType('Last Modified', 'mtime'), StringType('Mode', 'mode', table='file') ], table='inode', where=DB.expand("file.path=%r and file.mode!='d/d'", (path + '/')), case=query['case'], pagesize=10, filter="filter2", ) target = tmp.defaults.get('open_tree', '/') tmp.toolbar(text=DB.expand("Scan %s", target), icon="examine.png", link=query_type(family="Load Data", report="ScanFS", path=target, case=query['case']), pane='popup')
def tree_cb(path): fsfd = FileSystem.DBFS(query['case']) query.default("path", '/') dirs = [] for i in fsfd.longls(path): if i['type'] == "directory": dirs.append(i['name']) yield (([i['name'], i['name'], 'branch']))
def tree_cb(path): fsfd = FileSystem.DBFS(query['case']) query.default("path",'/') if not path.endswith('/'): path=path+'/' dirs = [] for i in fsfd.dent_walk(path): if i['mode']=="d/d" and i['status']=='alloc' and i['name'] not in dirs: dirs.append(i['name']) yield(([i['name'],i['name'],'branch']))
def glob_files(self, args): ## Glob the path if possible: files = {} for arg in args: ## Add the implied CWD: if not arg.startswith("/"): arg=FlagFramework.normpath(self.environment.CWD+"/"+arg) for path in FileSystem.glob(arg, case=self.environment._CASE): files[path]=True ## This is used to collate files which may appear in multiple globs files = files.keys() files.sort() return files
def tree_cb(path): ## We expect a directory here: if not path.endswith('/'): path = path + '/' ## We need a local copy of the filesystem factory so ## as not to affect other instances!!! fsfd = FileSystem.DBFS(query["case"]) dirs = [] for i in fsfd.dent_walk(path): if i['mode'] == "d/d" and i['status'] == 'alloc' and i[ 'name'] not in dirs: dirs.append(i['name']) yield (([i['name'], i['name'], 'branch']))
def check_for_file(self, sql='1'): dbh = DB.DBO(self.test_case) dbh.execute( "select inode.inode as inode,type.type,path,name from file,type,inode where file.inode_id=type.inode_id and inode.inode_id=type.inode_id and type.type like '%%JPEG%%' and %s limit 1", sql) row = dbh.fetch() if not row: return None ## Check that its a real file: fsfd = FileSystem.DBFS(self.test_case) fd = fsfd.open(inode=row['inode']) data = fd.read() if len(data) == 0: raise IOError("Can not read file %s%s (%s) %r" % (row['path'], row['name'], row['inode'], data)) return row
def pane_cb(path,tmp): fsfd = FileSystem.DBFS( query["case"]) if not fsfd.isdir(path): path=posixpath.dirname(path) new_query = make_new_query(query, path + '/') tmp.table( elements = [ InodeIDType(case=query['case']), FilenameType(basename=True, case=query['case'], link = new_query, link_pane = 'parent'), IntegerType('File Size','size'), ], table='inode', where=DB.expand("file.path=%r and file.mode!='d/d'", (path+'/')), case=query['case'], pagesize=10, filter="filter2", )
def display(self, query, result): new_q = result.make_link(query, '') if not query.has_key('limit'): query['limit'] = 0 dbh = self.DBO(query['case']) fsfd = FileSystem.DBFS(query["case"]) ## If this is a directory, only show the stats fd = fsfd.open(inode_id=query['inode_id']) if not fd: return tmp = result.__class__(result) tmp.text(fd.urn) result.heading(tmp) try: m = Magic.MagicResolver() type, mime = m.find_inode_magic(query['case'], fd.inode_id) result.text("Classified as %s by magic" % type) except IOError, e: result.text("Unable to classify file, no blocks: %s" % e)
def test02TestOutput(self): """ Testing output """ dbh = DB.DBO(self.test_case) fsfd = FileSystem.DBFS(self.test_case) dbh.execute( "select inode_id, word_id, word,offset,length from LogicalIndexOffsets join %s.dictionary on LogicalIndexOffsets.word_id=%s.dictionary.id where id>1000 and id<1020", (config.FLAGDB, config.FLAGDB)) for row in dbh: patg, inode, inode_id = fsfd.lookup(inode_id=row['inode_id']) fd = fsfd.open(inode=inode) fd.overread = True fd.slack = True fd.seek(row['offset']) data = fd.read(row['length']) filename, inode, inode_id = fsfd.lookup(inode=inode) print "Looking for %s: Found in %s (%s) at offset %s length %s %r" % ( row['word'], filename, inode, row['offset'], row['length'], data) #self.assertEqual(data.lower(), row['word'].lower()) self.find_expected_output(row['word'], row['word_id'] - 1000, filename, row['offset'], self.case_sensitive_keywords, data) print "Left over %s" % self.case_sensitive_keywords
def tree_cb(path): fsfd = FileSystem.DBFS(query["case"]) for i in fsfd.longls(path): if i['type'] == 'directory': yield (([i['name'], i['name'], 'branch']))
def open(self): fsfd = FileSystem.DBFS(self.name) return fsfd.open(path = self.path)
class RevEng_GUI(Reports.report): """ Allows us to manipulate data structures in reverse engineering efforts """ name = "DAFT" family = "Misc" description = "Data Analysis Facilitation Tool (Reverse Engineering)" # parameters = { "foo": "any"} def analyse(self, query): pass def display(self, query, result): result.start_form(query) def settings_cb(query, ui): ui.decoration = "naked" try: if query['finish'] and query['MaxRows'] and query[ 'StartOffset']: del query['finish'] del query['submit'] ui.refresh(0, query, parent=1) except KeyError: pass ui.start_form(query) ui.start_table() ui.textfield("Starting Offset", "StartOffset", size=20) ui.textfield("Maximum Rows", "MaxRows") ui.checkbox("Click here to finish", "finish", "yes") ui.end_table() ui.end_form() return ui def popup_cb(query, ui, column_number=None, mode=''): """Popup for defining column attributes""" ## print "I am here" ui.decoration = "naked" if mode == 'insert': pre = 'insert_' else: pre = '' try: if query['finish'] and query['%sname_%s' % (pre, column_number)]: del query['finish'] del query['submit'] ui.refresh(0, query, parent=1) except KeyError: pass ui.start_form(query) ui.start_table() if mode == 'insert': ui.heading("Inserting Column number %s" % column_number) else: ui.heading("Column number %s" % column_number) names = [ x.__name__ for x in Registry.FILEFORMATS.classes if x.visible ] ui.textfield("Name for this field", "%sname_%s" % (pre, column_number)) ui.const_selector("Data Type", '%sdata_type_%s' % (pre, column_number), names, names) try: temp = Registry.FILEFORMATS[query['%sdata_type_%s' % (pre, column_number)]]("", None) ui.row("Description", temp.__doc__) temp.form("%sparameter_%s_" % (pre, column_number), query, ui) except KeyError, e: print 'KeyError: %s' % e ui.checkbox("Visible", "%svisible_%s" % (pre, column_number), "yes", checked=True) ui.checkbox("Click here to finish", "finish", "yes") ui.end_table() ui.end_form() return ui def delete_col_cb(query, ui, column_number=None): """Popup to confirm deletion of column""" ui.decoration = "naked" ui.heading("Delete column number %s?" % column_number) try: if query['submit']: del query['submit'] ui.refresh(0, query, parent=1) except KeyError: pass ui.start_form(query) ui.checkbox("Click here to delete", "delete_%s" % column_number, "yes") ui.end_form() return ui def processquery(query): delcol = -1 insvalues = {} for k in query.keys(): if k.startswith('delete_'): delcol = int(k[7:]) del query[k] break elif k.startswith('insert_'): insvalues[k[7:]] = query[k] if k.startswith('insert_name_'): inscol = int(k[12:]) del query[k] continue elif k.startswith('savenow'): savelayout(query) del query[k] break elif k.startswith('loadlayout'): openlayout(query) del query[k] ### other stuff for ins col parameters if delcol >= 0: count = delcol while 1: try: query['name_%s' % count] count += 1 except KeyError: break for i in range(delcol + 1, count): del query['name_%s' % (i - 1)] del query['data_type_%s' % (i - 1)] del query['visible_%s' % (i - 1)] params = [ k for k in query.keys() if k.startswith('parameters_%s_' % (i - 1)) ] for parameter in params: del query[parameter] query['name_%s' % (i - 1)] = query['name_%s' % i] query['data_type_%s' % (i - 1)] = query['data_type_%s' % i] query['visible_%s' % (i - 1)] = query['visible_%s' % i] key = 'parameter_' params = [ k[11 + len('%s' % i):] for k in query.keys() if k.startswith('%s%s_' % (key, i)) ] for parameter in params: query['%s%s_%s' % (key, (i - 1), parameter)] = query['%s%s_%s' % (key, i, parameter)] del query['name_%s' % (count - 1)] del query['data_type_%s' % (count - 1)] del query['visible_%s' % (count - 1)] params = [ k for k in query.keys() if k.startswith('parameter_%s_' % (count - 1)) ] for parameter in params: del query[params] elif len(insvalues) > 0: count = inscol while 1: try: query['name_%s' % count] count += 1 except KeyError: break for i in range(count, inscol, -1): query['name_%s' % i] = query['name_%s' % (i - 1)] query['data_type_%s' % i] = query['data_type_%s' % (i - 1)] query['visible_%s' % i] = query['visible_%s' % (i - 1)] key = 'parameter_' params = [ k[11 + len('%s' % (i - 1)):] for k in query.keys() if k.startswith('%s%s_' % (key, (i - 1))) ] for parameter in params: query['%s%s_%s' % (key, i, parameter)] = query['%s%s_%s' % (key, (i - 1), parameter)] del query['name_%s' % (i - 1)] del query['data_type_%s' % (i - 1)] del query['visible_%s' % (i - 1)] params = [ k for k in query.keys() if k.startswith('parameter_%s_' % (i - 1)) ] for parameter in params: del query[parameter] for k in insvalues.keys(): query[k] = insvalues[k] def open_cb(query, ui): """Popup for loading a layout""" ui.decoration = "naked" dbh = self.DBO(query['case']) try: if query['finish']: del query['finish'] del query['submit'] ui.refresh(0, query, parent=1) except KeyError: pass ui.start_form(query) ui.start_table() ui.heading("Load a saved layout") try: dbh.execute('select name from DAFTLayouts') rows = [] for row in dbh: rows.append(row['name']) ui.const_selector("Layout Name", "loadlayout", rows, rows) except DB.DBError: dbh.execute( 'create table DAFTLayouts (`name` text, `layout` text)') ui.const_selector("Layout Name", "loadlayout", [''], ['']) ui.checkbox("Click here to finish", "finish", "yes") ui.end_table() ui.end_form() return ui ### Create a list of saved layouts def openlayout(query): dbh = self.DBO(query['case']) keylist = [ 'name_', 'data_type_', 'visible_', 'parameter_', 'fileselect', 'MaxRows', 'StartOffset', 'savelayout' ] try: dbh.execute("select layout from DAFTLayouts where name='%s'" % query['loadlayout']) except DB.DBError: pass rows = [] oldkeys = [] for row in dbh: rows.append(row) try: if len(rows) < 1: raise ValueError for k in keylist: oldkeys = [x for x in query.keys() if x.startswith(k)] for key in oldkeys: del query[key] for kvpair in rows[0]['layout'].split(','): key, value = kvpair.split('=') query[key] = value except ValueError: pass def save_cb(query, ui): """Popup for saving layout""" ui.decoration = "naked" try: if query['finish'] and query['savelayout']: query['savenow'] = 'yes' del query['finish'] del query['submit'] ui.refresh(0, query, parent=1) except KeyError: pass ui.start_form(query) ui.start_table() ui.heading("Save current layout") ui.textfield("Layout name", "savelayout") ui.checkbox("Click here to finish", "finish", "yes") ui.end_table() ui.end_form() return ui def savelayout(query): """Saves the current layout into the DAFTLayouts table""" dbh = self.DBO(query['case']) keylist = [ 'name_', 'data_type_', 'visible_', 'parameter_', 'fileselect', 'MaxRows', 'StartOffset', 'savelayout' ] try: dbh.execute( "select name, layout from DAFTLayouts where name='%s'" % query['savelayout']) except DB.DBError: dbh.execute( "create table DAFTLayouts (`name` text, `layout` text)") dbh.execute( "select name, layout from DAFTLayouts where name='%s'" % query['savelayout']) rows = [] keys = [] for row in dbh: rows.append(row) for k in keylist: keys = keys + [x for x in query.keys() if x.startswith(k)] value = ','.join('%s=%s' % (x, query[x]) for x in keys) if rows == []: dbh.execute( "insert into DAFTLayouts set name='%s', layout='%s'" % (query['savelayout'], value)) else: dbh.execute( "update DAFTLayouts set layout='%s' where name='%s'" % (value, query['savelayout'])) def filelist_cb(query, ui): """Popup to select files to analyse""" ui.decoration = "naked" try: if query['finish'] and query['fileselect']: del query['finish'] del query['submit'] ui.refresh(0, query, parent=1) except KeyError: pass ui.start_form(query) ui.start_table() ui.heading("Select files") values = [] keys = [] dbh.execute("select name, path, inode from file where inode != ''") for row in dbh: values.append('%s%s' % (row['path'], row['name'])) keys.append(row['inode']) ui.const_selector("File", "fileselect", keys, values) ui.checkbox("Click here to finish", "finish", "yes") ui.end_table() ui.end_form() def render_HTMLUI(data): """Callback to render mysql stored data in HTML""" tmp = result.__class__(result) tmp.result = data return tmp ##### Display starts here try: result.heading("Data Analysis Facilitation Tool") dbh = DB.DBO(query['case']) processquery(query) ## Build a struct to work from: try: startoffset = DAFTFormats.numeric(query['StartOffset']) except KeyError: startoffset = 0 try: maxrows = DAFTFormats.numeric(query['MaxRows']) except KeyError: maxrows = 10 fsfd = FileSystem.DBFS(query["case"]) try: fd = fsfd.open(inode=query['fileselect']) fdsize = fsfd.istat(inode=query['fileselect']) fd.block_size = dbh.get_meta('block_size') buf = format.Buffer(fd=fd)[startoffset:] except IOError, e: print 'IOError: %s' % e fd = None fdsize = 0 s = '\x00' * 1024 buf = format.Buffer(string=s) except KeyError, e: print 'KeyError: %s' % e s = '\x00' * 1024 buf = format.Buffer(string=s) fd = None fdsize = 0