def pane_cb(path, result): query['order'] = 'Filename' if path == '': path = '/' ## If we are asked to show a file, we will show the ## contents of the directory the file is in: fsfd = FileSystem.DBFS(query["case"]) if not fsfd.isdir(path): path = os.path.dirname(path) self.make_table_widget( ['URN', 'Name', 'Size', 'Modified'], query, result, where=DB.expand( "path=%r and (isnull(type) or type!='directory')", (path)), ) result.toolbar(text=DB.expand("Scan %s", path), icon="examine.png", link=query_type(family="Load Data", report="ScanFS", path=path, case=query['case']), pane='popup')
def longls(self,path='/', dirs = None): dbh=DB.DBO(self.case) if self.isdir(path): ## If we are listing a directory, we list the files inside the directory if not path.endswith('/'): path=path+'/' where = DB.expand(" path=%r " ,path) else: ## We are listing the exact file specified: where = DB.expand(" path=%r and name=%r", ( FlagFramework.normpath(posixpath.dirname(path)+'/'), posixpath.basename(path))) mode ='' if(dirs == 1): mode=" and file.mode like 'd%'" elif(dirs == 0): mode=" and file.mode like 'r%'" dbh.execute("select * from file where %s %s", (where, mode)) result = [dent for dent in dbh] for dent in result: if dent['inode']: dbh.execute("select * from inode where inode = %r", dent['inode']) data = dbh.fetch() if data: dent.update(data) return result
def pane_cb(path, tmp): query['order'] = 'Filename' ## If we are asked to show a file, we will show the ## contents of the directory the file is in: fsfd = FileSystem.DBFS(query["case"]) if not fsfd.isdir(path): path = os.path.dirname(path) tmp.table( elements=[ InodeIDType(case=query['case']), FilenameType(basename=True, case=query['case']), DeletedType(), IntegerType('File Size', 'size'), TimestampType('Last Modified', 'mtime'), StringType('Mode', 'mode', table='file') ], table='inode', where=DB.expand("file.path=%r and file.mode!='d/d'", (path + '/')), case=query['case'], pagesize=10, filter="filter2", ) target = tmp.defaults.get('open_tree', '/') tmp.toolbar(text=DB.expand("Scan %s", target), icon="examine.png", link=query_type(family="Load Data", report="ScanFS", path=target, case=query['case']), pane='popup')
def longls(self, path='/', dirs=None): dbh = DB.DBO(self.case) if self.isdir(path): ## If we are listing a directory, we list the files inside the directory if not path.endswith('/'): path = path + '/' where = DB.expand(" path=%r ", path) else: ## We are listing the exact file specified: where = DB.expand( " path=%r and name=%r", (FlagFramework.normpath(posixpath.dirname(path) + '/'), posixpath.basename(path))) mode = '' if (dirs == 1): mode = " and file.mode like 'd%'" elif (dirs == 0): mode = " and file.mode like 'r%'" dbh.execute("select * from file where %s %s", (where, mode)) result = [dent for dent in dbh] for dent in result: if dent['inode']: dbh.execute("select * from inode where inode = %r", dent['inode']) data = dbh.fetch() if data: dent.update(data) return result
def pane_cb(path,tmp): query['order']='Filename' ## If we are asked to show a file, we will show the ## contents of the directory the file is in: fsfd = FileSystem.DBFS( query["case"]) if not fsfd.isdir(path): path=os.path.dirname(path) tmp.table( elements = [ InodeIDType(case=query['case']), FilenameType(basename=True, case=query['case']), DeletedType(), IntegerType('File Size','size'), TimestampType('Last Modified','mtime'), StringType('Mode','mode', table='file') ], table='inode', where=DB.expand("file.path=%r and file.mode!='d/d'", (path+'/')), case=query['case'], pagesize=10, filter="filter2", ) target = tmp.defaults.get('open_tree','/') tmp.toolbar(text=DB.expand("Scan %s",target), icon="examine.png", link=query_type(family="Load Data", report="ScanFS", path=target, case=query['case']), pane='popup' )
def __str__(self): postfix = '' ## Some tags are never allowed to be outputted if self.name not in self.allowable_tags: if self.name in self.forbidden_tag: return '' #print "Rejected tag %s" % self.name return self.innerHTML() if self.name == 'head': self.children = [self.header,] + self.children elif self.name =='body': self.children = [self.body_extra, ] + self.children ## Frames without src are filtered because IE Whinges: if self.name == 'iframe' and 'src' not in self.attributes: return '' attributes = "".join([" %s='%s'" % (k,v) for k,v \ in self.attributes.items() if k in \ self.allowable_attributes]) if 'style' in self.attributes: attributes += ' style=%r' % self.css_filter(self.attributes['style'] or '') if 'http-equiv' in self.attributes: if self.attributes['http-equiv'].lower() == "content-type": ## PF _always_ outputs in utf8 attributes += ' http-equiv = "Content-Type" content="text/html; charset=UTF-8"' if 'src' in self.attributes: attributes += ' src=%s' % self.resolve_reference(self.attributes['src']) try: if 'href' in self.attributes: if self.name == 'link': attributes += " href=%s" % self.resolve_reference(self.attributes['href'], 'text/css') else: attributes += DB.expand(' href="javascript: alert(%r)"', iri_to_uri(DB.expand("%s",self.attributes['href'])[:100])) postfix = self.mark_link(self.attributes['href']) except: pass ## CSS needs to be filtered extra well if self.name == 'style': return expand("<style %s>%s</style>" , (attributes, self.css_filter(self.innerHTML()))) if self.type == 'selfclose': return expand("<%s%s/>%s" , (self.name, attributes, postfix)) else: return expand("<%s%s>%s</%s>%s", (self.name, attributes, self.innerHTML(), self.name,postfix))
class Index(Farm.Task): """ A task to index an inode with the dictionary """ def run(self, case, inode_id, *args): global INDEX if not INDEX: reindex() try: desired_version = int(args[0]) except: desired_version = INDEX_VERSION ## Did they want a detailed index or a unique index? unique = desired_version < 2**30 ## In unique mode we want to generate one hit per scan job per ## word if unique: INDEX.clear_set() pyflaglog.log( pyflaglog.VERBOSE_DEBUG, "Indexing inode_id %s (version %s)" % (inode_id, desired_version)) fsfd = FileSystem.DBFS(case) fd = fsfd.open(inode_id=inode_id) buff_offset = 0 dbh = DB.DBO(case) ## Clear old hits: dbh.check_index("LogicalIndexOffsets", "inode_id") dbh.delete("LogicalIndexOffsets", where=DB.expand("inode_id = %r", inode_id)) ## Get ready for scan dbh.mass_insert_start("LogicalIndexOffsets") while 1: data = fd.read(1024 * 1024) if len(data) == 0: break for offset, matches in INDEX.index_buffer(data, unique=unique): for id, length in matches: dbh.mass_insert(inode_id=inode_id, word_id=id, offset=offset + buff_offset, length=length) buff_offset += len(data) dbh.mass_insert_commit() ## Update the version dbh.update("inode", where=DB.expand('inode_id = %r', inode_id), version=desired_version)
def execute(self): for iosource in self.args: dbh = DB.DBO(self.environment._CASE) dbh2 = dbh.clone() dbh.delete('inode', where=DB.expand("inode like 'I%s|%%'", iosource)) dbh.execute("select * from filesystems where iosource = %r", iosource) for row in dbh: dbh2.delete('file', where=DB.expand("path like '%s%%'", iosource)) dbh.delete("iosources", where=DB.expand("name=%r", iosource)) yield "Removed IOSource %s" % iosource
def run(self, case, inode_id, *args): global INDEX if not INDEX: reindex() try: desired_version = int(args[0]) except: desired_version = INDEX_VERSION ## Did they want a detailed index or a unique index? unique = desired_version < 2**30 ## In unique mode we want to generate one hit per scan job per ## word if unique: INDEX.clear_set() pyflaglog.log(pyflaglog.VERBOSE_DEBUG, "Indexing inode_id %s (version %s)" % (inode_id, desired_version)) fsfd = FileSystem.DBFS(case) fd = fsfd.open(inode_id=inode_id) buff_offset = 0 dbh = DB.DBO(case) ## Clear old hits: dbh.check_index("LogicalIndexOffsets", "inode_id") dbh.delete("LogicalIndexOffsets", where=DB.expand("inode_id = %r", inode_id)) ## Get ready for scan dbh.mass_insert_start("LogicalIndexOffsets") while 1: data = fd.read(1024*1024) if len(data)==0: break for offset, matches in INDEX.index_buffer(data, unique = unique): for id, length in matches: dbh.mass_insert( inode_id = inode_id, word_id = id, offset = offset + buff_offset, length = length) buff_offset += len(data) dbh.mass_insert_commit() ## Update the version dbh.update("inode", where = DB.expand('inode_id = %r', inode_id), version = desired_version)
class ThumbnailType(InodeIDType): """ A Column showing thumbnails of inodes """ def __init__(self, name='Thumbnail', **args ): InodeIDType.__init__(self, **args) self.fsfd = FileSystem.DBFS(self.case) self.name = name def select(self): return "%s.inode_id" % self.table ## When exporting to html we need to export the thumbnail too: def render_html(self, inode_id, table_renderer): ct='' try: fd = self.fsfd.open(inode_id = inode_id) image = Graph.Thumbnailer(fd, 200) inode_filename, ct, fd = table_renderer.make_archive_filename(inode_id) filename, ct, fd = table_renderer.make_archive_filename(inode_id, directory = "thumbnails/") table_renderer.add_file_from_string(filename, image.display()) except IOError,e: print e return "<a href=%r ><img src='images/broken.png' /></a>" % inode_filename InodeIDType.render_html(self, inode_id, table_renderer) table_renderer.add_file_to_archive(inode_id) return DB.expand("<a href=%r type=%r ><img src=%r /></a>", (inode_filename, ct, filename))
def form(self, query, result): result.textfield("Inode ID", 'inode_id') dbh = DB.DBO(query['case']) try: result.selector("Table Name", 'table_name', DB.expand('select name as `key`,name as value from sqlite where inode_id=%r', query['inode_id']), case=query['case']) except KeyError, e: pass
def display(self, query, result): path = query['path'] key = query['key'] result.heading("Registry Key Contents") result.text(DB.expand("Key %s/%s:", (path, key)), style='red', font='typewriter') dbh = DB.DBO(query['case']) def hexdump(query, out): """ Show the hexdump for the key """ dbh.execute( "select value from reg where path=%r and reg_key=%r limit 1", (path, key)) row = dbh.fetch() if row: HexDump(row['value'], out).dump() return out def strings(query, out): """ Draw the strings in the key """ out.para("not implimented yet") return out def stats(query, out): """ display stats on a key """ out.para("not implemented yet") return out result.notebook(names=["HexDump", "Strings", "Statistics"], callbacks=[hexdump, strings, stats], context="display_mode")
def drop_table(case, name): """ Drops the log table tablename """ if not name: return dbh = DB.DBO(case) pyflaglog.log(pyflaglog.DEBUG, "Dropping log table %s in case %s" % (name, case)) dbh.execute("select * from log_tables where table_name = %r limit 1" , name) row = dbh.fetch() ## Table not found if not row: return preset = row['preset'] ## Get the driver for this table: log = load_preset(case, preset) log.drop(name) ## Ask the driver to remove its table: dbh.delete("log_tables", where= DB.expand("table_name = %r ", name)); ## Make sure that the reports get all reset FlagFramework.reset_all(family='Load Data', report="Load Preset Log File", table = name, case=case)
def __init__(self, case, fd, inode): File.__init__(self, case, fd, inode) # strategy: must determine basepath from parent, get our path # from db and then return the file: ## Note this _must_ work because we can only ever be called on ## a mounted iosource - it is an error otherwise: basepath = fd.io.directory self.case = case dbh = DB.DBO(case) dbh.check_index("file", "inode") dbh.execute("select path,name from file where inode=%r limit 1", (inode)) row = dbh.fetch() path = row["path"] mount_point = fd.io.mount_point ## Prune the path down to the mount point: if path[: len(mount_point)] != mount_point: raise RuntimeError(DB.expand("Something went wrong - %s should be mounted on %s", (path, mount_point))) path = path[len(mount_point) :] path = basepath + "/" + path + "/" + row["name"] if not path.startswith(posixpath.normpath(config.UPLOADDIR)): path = FlagFramework.sane_join(config.UPLOADDIR, path) if os.path.isdir(path): self.fd = StringIO.StringIO("") else: self.fd = open(path, "r") s = os.stat(path) self.size = s.st_size
def pane_cb(path, result): tlds = path.split("/") try: result.defaults.set('filter', DB.expand('TLD = %r and "Content Type" contains html',tlds[1])) Reports.CaseTableReports.display(self, query, result) except IndexError: result.para("Click on a TLD to view all URLs from that TLD")
def pane_cb(path, result): if not path.endswith('/'): path=path+'/' result.heading("Path is %s" % path) case = query['case'] dbh = DB.DBO(case) fsfd = Registry.FILESYSTEMS.dispatch(query['fstype'])(case) ## Try to see if the directory is already loaded: dbh.execute("select * from file where path=%r and name=''", path) if not dbh.fetch(): fsfd.load(mount_point = query['mount_point'], iosource_name= query['iosource'], directory = path) ## Now display the table result.table( elements = [ InodeIDType(case=query['case']), FilenameType(case=query['case']), DeletedType(), IntegerType(name='File Size',column='size'), TimestampType(name = 'Last Modified',column = 'mtime'), ], table='inode', where=DB.expand("file.path=%r and file.mode!='d/d'",(path)), case = query['case'], pagesize=10, )
def glob_sql(pattern): path, name = posixpath.split(pattern) if globbing_re.search(path): path_sql = "path rlike '^%s/?$'" % translate(path) else: ## Ensure that path has a / at the end: if not path.endswith("/"): path = path + '/' path_sql = "path='%s'" % path if globbing_re.search(name): name_sql = "name rlike '^%s$'" % translate(name) else: name_sql = DB.expand("name=%r", name) if name and path: sql = "select concat(path,name) as path from file where %s and %s group by file.path,file.name" % ( path_sql, name_sql) elif name: sql = "select concat(path,name) as path from file where %s group by file.path,file.name" % name_sql elif path: #sql = "%s and name=''" % path_sql sql = "select path from file where %s group by file.path" % path_sql else: ## Dont return anything for an empty glob sql = "select * from file where 1=0" return sql
def explain(self, query, result): name = self.fd.name ## Trim the upload directory if present if name.startswith(config.UPLOADDIR): name = name[len(config.UPLOADDIR) :] result.row("Filename", DB.expand("%s", name), **{"class": "explainrow"})
def get_factories(case, scanners): """ Scanner factories are obtained from the Store or created as required. Scanners is a list in the form case:scanner """ ## Ensure dependencies are satisfied scanners = ScannerUtils.fill_in_dependancies(scanners) ## First prepare the required factories: result = [] for scanner in scanners: key = DB.expand("%s:%s", (case, scanner)) try: f = factories.get(key) except KeyError: try: cls = Registry.SCANNERS.dispatch(scanner) except: # pyflaglog.log(pyflaglog.WARNING, "Unable to find scanner for %s", scanner) continue # Instatiate it: import pyflag.FileSystem as FileSystem f = cls(FileSystem.DBFS(case)) ## Initialise it: f.prepare() ## Store it: factories.put(f, key=key) result.append(f) return result
def analyse(self, query): context = self.get_context(query) word_id = Indexing.insert_dictionary_word(query['word'], query['type']) pdbh = DB.DBO() sql = DB.expand("select inode.inode_id as `inode_id` "\ "%s where (%s) and (%s)", (context.get('tables',''), context.get('inode_sql','1'), context.get('where','1'))) Indexing.schedule_inode_index_sql(query['case'], sql, word_id, query['cookie'], unique=True) ## Now wait here until everyone is finished: while 1: pdbh.execute("select count(*) as c from jobs where cookie=%r", query['cookie']) row = pdbh.fetch() self.rows_left = row['c'] if row['c'] == 0: break time.sleep(1) return 1
def glob_sql(pattern): path,name = posixpath.split(pattern) if globbing_re.search(path): path_sql = "path rlike '^%s/?$'" % translate(path) else: ## Ensure that path has a / at the end: if not path.endswith("/"): path=path+'/' path_sql = "path='%s'" % path if globbing_re.search(name): name_sql = "name rlike '^%s$'" % translate(name) else: name_sql = DB.expand("name=%r", name) if name and path: sql = "select concat(path,name) as path from file where %s and %s group by file.path,file.name" % (path_sql,name_sql) elif name: sql = "select concat(path,name) as path from file where %s group by file.path,file.name" % name_sql elif path: #sql = "%s and name=''" % path_sql sql = "select path from file where %s group by file.path" % path_sql else: ## Dont return anything for an empty glob sql = "select * from file where 1=0" return sql
def display(self,query,result): path=query['path'] key=query['key'] result.heading("Registry Key Contents") result.text(DB.expand("Key %s/%s:", (path,key)),style='red',font='typewriter') dbh=DB.DBO(query['case']) def hexdump(query,out): """ Show the hexdump for the key """ dbh.execute("select value from reg where path=%r and reg_key=%r limit 1",(path,key)) row=dbh.fetch() if row: HexDump(row['value'],out).dump() return out def strings(query,out): """ Draw the strings in the key """ out.para("not implimented yet") return out def stats(query,out): """ display stats on a key """ out.para("not implemented yet") return out result.notebook( names=["HexDump","Strings","Statistics"], callbacks=[hexdump,strings,stats], context="display_mode" )
def explain(self, query, result): name = self.fd.name ## Trim the upload directory if present if name.startswith(config.UPLOADDIR): name = name[len(config.UPLOADDIR):] result.row("Filename",DB.expand("%s", name), **{'class':'explainrow'})
def finish(self): self.dbh.mass_insert_commit() ## Update the version self.dbh.update("inode", where = DB.expand('inode_id = %r', self.inode_id), version = INDEX_VERSION) del self.dbh
def finish(self): self.dbh.mass_insert_commit() ## Update the version self.dbh.update("inode", where=DB.expand('inode_id = %r', self.inode_id), version=INDEX_VERSION) del self.dbh
def insert_into_table(mode ,root ,name): rel_root = FlagFramework.normpath(DB.expand("%s/%s/" , (mount_point, root[len(path):]))) try: s=os.lstat(os.path.join(root,name)) except OSError: pyflaglog.log(pyflaglog.WARNING, DB.expand("Unable to stat %s - mount the directory with the uid option", root)) return inode = DB.expand("I%s|M%s", (iosource_name, s.st_ino)) dbh_inode.insert('inode', inode = inode, uid = s.st_uid, gid = s.st_gid, _mtime = "from_unixtime(%s)" % s.st_mtime, _atime = "from_unixtime(%s)" % s.st_atime, _ctime = "from_unixtime(%s)" % s.st_ctime, status = 'alloc', mode = str(oct(s.st_mode)), size = s.st_size, _fast=True) inode_id = dbh_inode.autoincrement() dbh_file.mass_insert(inode_id = inode_id, inode = inode, mode = mode, status = 'alloc', path = rel_root, name = name) ## If needed schedule inode for scanning: if scanners and mode=='r/r': pdbh.mass_insert( command = 'Scan', arg1 = self.case, arg2 = inode, arg3= scanner_string, cookie=cookie, ) ## Fixme - handle symlinks try: link=os.readlink(DB.expand("%s/%s", (root,name)).encode("utf8")) except OSError: link=''
def drop_preset(preset): """ Drops the specified preset name """ pyflaglog.log(pyflaglog.DEBUG, "Droppping preset %s" % preset) for case, table in find_tables(preset): drop_table(case, table) dbh = DB.DBO() if preset: dbh.delete("log_presets", where=DB.expand("name = %r",preset))
def insert_whois_cache(sql_ip, id, ipinfo): dbh = DB.DBO() dbh.insert("whois_cache", _ip=sql_ip, id=id, _geoip_city=DB.expand( "(select id from geoip_city where city=%r " "limit 1)", (ipinfo.get('city', 'Unknown'), )) or '', _geoip_country=DB.expand( "(select id from geoip_country where country" "=%r limit 1)", (ipinfo.get("country_code3", "---"), )), _geoip_org=DB.expand( "(select id from geoip_org where org" "=%r limit 1)", (ipinfo.get("org", "Unknown"), )), _geoip_isp=DB.expand( "(select id from geoip_isp where isp" "=%r limit 1)", (ipinfo.get("isp", "Unknown"), )), _fast=True)
def drop_preset(preset): """ Drops the specified preset name """ pyflaglog.log(pyflaglog.DEBUG, "Droppping preset %s" % preset) for case, table in find_tables(preset): drop_table(case, table) dbh = DB.DBO() if preset: dbh.delete("log_presets", where=DB.expand("name = %r", preset))
def operator_hit(self, column, operator, arg): """ Search for a hit in the dictionary """ ## Try to work out if we need to reindex: reindex = False dbh = DB.DBO() dbh.execute("select id from dictionary where word = %r limit 1", arg) row = dbh.fetch() if self.ui: ## If the word is not in the dictionary, we definitely want to reindex if not row: count, total, tables, sql = self.outstanding_inodes() message = "Word %s is not in the dictionary" % arg ## If the word is in the dictionary we want to know how may ## inodes are outdated else: count, total, tables, sql = self.outstanding_inodes( word_id=row['id']) message = "There are some inodes which are not up to date" ## Any inodes to process? if count > 0: reindex = True ## We do not need to reindex - just do it if not reindex: return DB.expand( "(%s = %s)", (self.escape_column_name(self.column), row.get('id', 0))) ## Allow the user to reindex the currently selected set of ## inodes with a new dictionary based on the new word self.ui.heading(message) self.ui.para( "This will affect %s inodes and require rescanning %s bytes" % (count, total)) ## Make up the link for the use: context = FlagFramework.STORE.put( dict(tables=tables, inode_sql=sql, previous_query=self.ui.defaults, target='parent_pane', where=self.table_where_clause)) link = query_type(report="Add Word", family="Keyword Indexing", case=self.case, context=context, word=arg) self.ui.link("Click here to scan these inodes", link, pane='self') ## Ok - Show the error to the user: raise self.ui
def insert_whois_cache(sql_ip, id, ipinfo): dbh = DB.DBO() dbh.insert("whois_cache", _ip = sql_ip, id = id, _geoip_city = DB.expand("(select id from geoip_city where city=%r " "limit 1)", (ipinfo.get('city','Unknown'),)) or '', _geoip_country = DB.expand("(select id from geoip_country where country" "=%r limit 1)", (ipinfo.get("country_code3","---"),)), _geoip_org = DB.expand("(select id from geoip_org where org" "=%r limit 1)", (ipinfo.get("org","Unknown"),)), _geoip_isp = DB.expand("(select id from geoip_isp where isp" "=%r limit 1)", (ipinfo.get("isp","Unknown"),)), _fast = True )
def pane_cb(path,result): query['order']='Filename' if path=='': path='/' ## If we are asked to show a file, we will show the ## contents of the directory the file is in: fsfd = FileSystem.DBFS( query["case"]) if not fsfd.isdir(path): path=os.path.dirname(path) self.make_table_widget(['URN','Name', 'Size','Modified'], query, result, where=DB.expand("path=%r and (isnull(type) or type!='directory')", (path)),) result.toolbar(text=DB.expand("Scan %s",path), icon="examine.png", link=query_type(family="Load Data", report="ScanFS", path=path, case=query['case']), pane='popup' )
def form(self, query, result): result.textfield("Inode ID", "inode_id") dbh = DB.DBO(query["case"]) try: result.selector( "Table Name", "table_name", DB.expand("select name as `key`,name as value from sqlite where inode_id=%r", query["inode_id"]), case=query["case"], ) except KeyError, e: pass
def longls(self,path='/', dirs = None): dbh=DB.DBO(self.case) if self.isdir(path): ## If we are listing a directory, we list the files inside the directory where = DB.expand(" path=%r " ,path) else: ## We are listing the exact file specified: where = DB.expand(" path=%r and name=%r", ( FlagFramework.normpath(posixpath.dirname(path)), posixpath.basename(path))) ## Only list directories if dirs: where += " and isnull(inode_id) " else: where += " and not isnull(inode_id) " dbh.execute("select * from vfs where %s group by inode_id,path,name", (where)) result = [dent for dent in dbh] return result
def operator_hit(self, column, operator, arg): """ Search for a hit in the dictionary """ ## Try to work out if we need to reindex: reindex = False dbh = DB.DBO() dbh.execute("select id from dictionary where word = %r limit 1", arg) row = dbh.fetch() if self.ui: ## If the word is not in the dictionary, we definitely want to reindex if not row: count, total, tables, sql = self.outstanding_inodes() message = "Word %s is not in the dictionary" % arg ## If the word is in the dictionary we want to know how may ## inodes are outdated else: count, total, tables, sql = self.outstanding_inodes(word_id = row['id']) message = "There are some inodes which are not up to date" ## Any inodes to process? if count > 0: reindex = True ## We do not need to reindex - just do it if not reindex: return DB.expand("(%s = %s)", (self.escape_column_name(self.column), row.get('id',0))) ## Allow the user to reindex the currently selected set of ## inodes with a new dictionary based on the new word self.ui.heading(message) self.ui.para("This will affect %s inodes and require rescanning %s bytes" % (count,total)) ## Make up the link for the use: context = FlagFramework.STORE.put(dict(tables = tables, inode_sql = sql, previous_query = self.ui.defaults, target = 'parent_pane', where = self.table_where_clause )) link = query_type(report = "Add Word", family = "Keyword Indexing", case = self.case, context = context, word = arg) self.ui.link("Click here to scan these inodes", link, pane = 'self') ## Ok - Show the error to the user: raise self.ui
def delete_case(case): """ A helper function which deletes the case """ dbh = DB.DBO(None) ## Broadcast that the case is about to be dropped (This broadcasts ## to the workers) dbh.insert('jobs',command = "DropCase", state='broadcast', arg1=case, cookie=0, _fast = True) ## This sends an event to our process: post_event('reset', case) ## Remove any jobs that may be outstanding (dont touch the ## currently processing jobs) dbh.delete('jobs',DB.expand("arg1=%r and state='pending' " , case), _fast= True) ## Now wait until there are no more processing jobs: total_time = 0 while 1: dbh.execute("select * from jobs where arg1=%r and state='processing' limit 1", case) row = dbh.fetch() if row: time.sleep(2) total_time += 2 if total_time > 20: pyflaglog.log(pyflaglog.WARNING,"Outstanding jobs remain in %s. Removing the case anyway." % case) dbh.execute("delete from jobs where arg1=%r and state='processing'",case) break pyflaglog.log(pyflaglog.INFO, "Waiting for outstanding jobs in case %r to be completed" % case) else: break try: #Delete the case from the database dbh.delete('meta',DB.expand("property='flag_db' and value=%r" , case), _fast=True) dbh.execute("drop database if exists `%s`" ,case) except DB.DBError,e: pass
def form(self, query, result): result.textfield("Inode ID", 'inode_id') dbh = DB.DBO(query['case']) try: result.selector( "Table Name", 'table_name', DB.expand( 'select name as `key`,name as value from sqlite where inode_id=%r', query['inode_id']), case=query['case']) except KeyError, e: pass
def add_inode(self, fd, offset): """ This is called to allow the Carver to add VFS inodes. Returns the new inode_id. """ ## Calculate the length of the new file length = self.get_length(fd,offset) new_inode = "%s|o%s:%s" % (self.fd.inode, offset, length) path, inode, inode_id = self.fsfd.lookup(inode_id = self.fd.inode_id) name = DB.expand("%s/%s", (path, self.make_filename(offset))) ## By default we just add a VFS Inode for it. new_inode_id = self.fsfd.VFSCreate(None, new_inode, name, size = length, ) pyflaglog.log(pyflaglog.DEBUG, DB.expand("Added Carved inode %s (id %s) as %s", (new_inode, new_inode_id, name))) self.add_type_info(new_inode_id)
def list_hits(case, inode_id, word, start=None, end=None): """ Returns a generator of hits of the word within the inode between offset start and end (these are inode offsets.""" dbh = DB.DBO(case) pdbh = DB.DBO() pdbh.execute("select id from dictionary where word = %r limit 1" , word) row = pdbh.fetch() if not row: raise RuntimeError("Word queried (%s) is not in the dictionary???" % word) ranges = '' if start!=None: ranges += DB.expand("and offset >= %r",(start,)) if end!=None: ranges += DB.expand("and offset < %r", (end,)) id = row['id'] dbh.execute("select offset,length from LogicalIndexOffsets where " "inode_id = %r and word_id = %r %s order by offset", inode_id, id, ranges) return dbh
def render_page(self, page_name, page_number, elements, row_generator): ## We must have an InodeID in there inode = None for e in elements: if isinstance(e, InodeIDType): inode = e break ## Should not happen if not inode: raise RuntimeError("You must have Inodes in your table") start_value = None end_value = None self.row_count = 0 tmp = [] result = '<table>' for row in row_generator: value = row[inode.name] cell_ui = value tmp.append(cell_ui.__str__()) end_value = row[elements[self.order].name] if not start_value: start_value = end_value if len(tmp) >= 5: result += self.render_row(tmp) tmp = [] self.row_count +=1 if self.row_count > self.pagesize: break result += self.render_row( tmp) dbh = DB.DBO(self.case) dbh.delete("reporting", where=DB.expand("page_name = %r", page_name)) dbh.insert("reporting", start_value = start_value, end_value = end_value, page_name = page_name, description = self.description) return self.header % {'toolbar': self.navigation_buttons(page_number), 'title': self.description or "PyFlag Gallery Export", } + \ result + """</tbody></table>
def render_page(self, page_name, page_number, elements, row_generator): ## We must have an InodeID in there inode = None for e in elements: if isinstance(e, AFF4URN): inode = e break ## Should not happen if not inode: raise RuntimeError("You must have Inodes in your table") start_value = None end_value = None self.row_count = 0 tmp = [] result = '<table>' for row in row_generator: value = row[inode.name] cell_ui = value tmp.append(cell_ui.__str__()) end_value = row[elements[self.order].name] if not start_value: start_value = end_value if len(tmp) >= 5: result += self.render_row(tmp) tmp = [] self.row_count +=1 if self.row_count > self.pagesize: break result += self.render_row( tmp) dbh = DB.DBO(self.case) dbh.delete("reporting", where=DB.expand("page_name = %r", page_name)) dbh.insert("reporting", start_value = start_value, end_value = end_value, page_name = page_name, description = self.description) return self.header % {'toolbar': self.navigation_buttons(page_number), 'title': self.description or "PyFlag Gallery Export", } + \ result + """</tbody></table>
def mass_annotate(new_query, result): if not new_query.has_key("annotate_inode"): result.heading("Error") result.para("You must select some inodes to annotate by checking their checkboxes") return if new_query.has_key("__submit__"): tag = new_query.get("annotate_text","Tag") category = new_query.get("annotate_category", "Note") if new_query.has_key("new_annotate_category"): category = new_query['new_annotate_category'] dbh = DB.DBO(case) ## First delete old annotations if present for inode_id in new_query.getarray("annotate_inode"): dbh.delete("annotate", where=DB.expand("inode_id=%r", inode_id)) ## Now insert new ones for inode_id in new_query.getarray("annotate_inode"): dbh.insert("annotate", inode_id = inode_id, note = tag, category = category, ) del query["annotate_inode"] query.set("annotate_text", tag) query.set("annotate_category", category) result.refresh(0, query, pane="parent_pane") fsfd = FileSystem.DBFS(case) new_query.default('annotate_text',query.get("annotate_text","Tag")) new_query.default("annotate_category", query.get("annotate_category","Note")) result.decoration='naked' result.heading("Set Annotation Text") result.start_form(new_query, pane='self') result.textarea("Annotation Text",'annotate_text') TableActions.selector_display(None, "Category", "annotate_category", result=result, table = 'annotate', field='category', case=case, default='Note') result.end_table() result.end_form() result.para("The following inodes will be annotated:") result.start_table(**{'class':'GeneralTable'}) for inode_id in new_query.getarray("annotate_inode"): path, inode, inode_id = fsfd.lookup(inode_id = inode_id) result.row(inode, path)
def insert_into_table(mode, root, name): rel_root = FlagFramework.normpath(DB.expand("%s/%s/", (mount_point, root[len(path) :]))) try: s = os.lstat(os.path.join(root, name)) except OSError: pyflaglog.log( pyflaglog.WARNING, DB.expand("Unable to stat %s - mount the directory with the uid option", root) ) return inode = DB.expand("I%s|M%s", (iosource_name, s.st_ino)) dbh_inode.insert( "inode", inode=inode, uid=s.st_uid, gid=s.st_gid, _mtime="from_unixtime(%s)" % s.st_mtime, _atime="from_unixtime(%s)" % s.st_atime, _ctime="from_unixtime(%s)" % s.st_ctime, status="alloc", mode=str(oct(s.st_mode)), size=s.st_size, _fast=True, ) inode_id = dbh_inode.autoincrement() dbh_file.mass_insert(inode_id=inode_id, inode=inode, mode=mode, status="alloc", path=rel_root, name=name) ## If needed schedule inode for scanning: if scanners and mode == "r/r": pdbh.mass_insert(command="Scan", arg1=self.case, arg2=inode, arg3=scanner_string, cookie=cookie) ## Fixme - handle symlinks try: link = os.readlink(DB.expand("%s/%s", (root, name)).encode("utf8")) except OSError: link = ""
def external_process(self, fd): pyflaglog.log(pyflaglog.DEBUG, "Opening %s for SQLite scanning", self.inode) filename = CacheManager.MANAGER.provide_cache_filename( self.case, self.fd.inode) db = sqlite.connect(filename) ldbh = db.cursor() ldbh2 = db.cursor() ldbh.execute("select * From sqlite_master") dbh = DB.DBO(self.case) dbh.cursor.ignore_warnings = True for row in ldbh: if row[0] == 'table': dbh.insert('sqlite', name=row[1], inode_id=self.fd.inode_id, definition=row[4]) try: case_table = build_case_table("sqlite", row[4]) except RuntimeError, e: pyflaglog.log(pyflaglog.WARNING, e) continue ## Create our copy of this table (if its not ## already there try: case_table.create(dbh) except Exception, e: pass ## Insert all the data into our copy of the table ldbh2.execute("select * from %s" % row[1]) dbh.mass_insert_start(case_table.name) for row in ldbh2: args = {'inode_id': self.fd.inode_id} for i in range(len(row)): if row[i] != None: if case_table.sql_filters[i]: args["_" + case_table. column_names[i]] = DB.expand( case_table.sql_filters[i], row[i]) else: args[case_table.column_names[i]] = row[i] dbh.mass_insert(**args) dbh.mass_insert_commit()
def log(level,message, *args): """ Prints the message out only if the configured verbosity is higher than the message's level.""" if config.LOG_LEVEL >= level: try: log_fd = open(config.LOGFILE,"ab") except Exception,e: log_fd = sys.stderr import pyflag.DB as DB try: string = DB.expand("%s(%s): %s" % (os.getpid(),lookup[level],message), args) except Exception,e: log_fd.write("%s\n" % e) string = message
def set_filter(self, query,result): ## If we get here - the word is ok filter_expression = DB.expand("Word = '%s'", (query['indexing_word'])) try: query.set(self.filter, "%s and %s" % (query[self.filter], filter_expression)) except KeyError: query.set(self.filter, filter_expression) ## Check that we can filter on the word as all: element = WordColumn(case = self.case, filter=self.filter, where = self.where) element.ui = result element.elements = self.elements element.operator_hit("Word", "=", query['indexing_word'])
def form(self,query,result): try: ## Draw the form for each scan group: result.text(DB.expand("Scanning Inode %s", (query['inode']))) groups = [] for cls in ScannerUtils.scan_groups_gen(): try: drawer = cls.Drawer() if drawer.group in groups: continue groups.append(drawer.group) drawer.form(query,result) except RuntimeError: pass result.checkbox('Click here when finished','final','ok') except KeyError: return result
def external_process(self, fd): pyflaglog.log(pyflaglog.DEBUG, "Opening %s for SQLite scanning", self.inode) filename = CacheManager.MANAGER.provide_cache_filename(self.case, self.fd.inode) db = sqlite.connect(filename) ldbh = db.cursor() ldbh2 = db.cursor() ldbh.execute("select * From sqlite_master") dbh = DB.DBO(self.case) dbh.cursor.ignore_warnings = True for row in ldbh: if row[0]=='table': dbh.insert('sqlite', name = row[1], inode_id = self.fd.inode_id, definition = row[4]) try: case_table = build_case_table("sqlite", row[4]) except RuntimeError,e: pyflaglog.log(pyflaglog.WARNING, e) continue ## Create our copy of this table (if its not ## already there try: case_table.create(dbh) except Exception,e: pass ## Insert all the data into our copy of the table ldbh2.execute("select * from %s" % row[1]) dbh.mass_insert_start(case_table.name) for row in ldbh2: args = {'inode_id': self.fd.inode_id} for i in range(len(row)): if row[i]!=None: if case_table.sql_filters[i]: args["_"+case_table.column_names[i]] = DB.expand( case_table.sql_filters[i],row[i]) else: args[case_table.column_names[i]] = row[i] dbh.mass_insert(**args) dbh.mass_insert_commit()
def display(self,query,result): ## This indexing will be done in process (i.e. not ## distributable) because its exactly one job: task = Index() task.run(query['case'], query['inode_id'], 2**30 + int(query['word_id'])) case = query['case'] result.table( elements = [ InodeIDType(case=case), OffsetType(case=case), DataPreview(name='Preview', case=case), ], table = 'LogicalIndexOffsets', where = DB.expand('inode.inode_id = %r and LogicalIndexOffsets.word_id= %r', (query['inode_id'],query['word_id'])), case =case, )
def display(self, query, result): ## The dictionary is site wide and lives in the FlagDB dbh = DB.DBO() ## class_override the class variable: try: if len(query['class_override']) > 3: del query['class'] query['class'] = query['class_override'] del query['class_override'] except KeyError: pass status = '' ## Do we need to add a new entry: try: if len(query['word']) < 3: raise DB.DBError( "Word is too short to index, minimum of 3 letter words") if query['action'] == 'insert': try: if len(query['class']) < 3: raise DB.DBError( "Class name is too short, minimum of 3 letter words are used as class names" ) except KeyError: status = "Classification missing or too short" raise Indexing.insert_dictionary_word(query['word'], query['type'], query['class']) status = "Added word %s to dictionary" % query['word'] elif query['action'] == 'delete': dbh.delete("dictionary", where=DB.expand("word=%b and type=%r", (query['word'], query['type']))) status = "Deleted word %s from dictionary" % query['word'] except KeyError, e: pass
def pane_cb(path,tmp): fsfd = FileSystem.DBFS( query["case"]) if not fsfd.isdir(path): path=posixpath.dirname(path) new_query = make_new_query(query, path + '/') tmp.table( elements = [ InodeIDType(case=query['case']), FilenameType(basename=True, case=query['case'], link = new_query, link_pane = 'parent'), IntegerType('File Size','size'), ], table='inode', where=DB.expand("file.path=%r and file.mode!='d/d'", (path+'/')), case=query['case'], pagesize=10, filter="filter2", )
def render_pane(self, branch, query, result): ## We may only draw on the pane that belongs to us: if branch[0] != self.name: return if len(branch)==1: result.heading("Show file types") result.text("This statistic allows different file types to be examined") else: t = branch[1].replace("__",'/') result.table( elements = [ AFF4URN(case = self.case), FilenameType(case = self.case, link_pane='main'), IntegerType('Size','size', table='vfs'), TimestampType('Timestamp','mtime', table='vfs'), StringType('Mime', 'mime', table='type')], table = 'type', where = DB.expand('type.type=%r ', t), case = self.case, )
def store_key(nk_key, path): if not nk_key: return key_name = nk_key['key_name'].__unicode__() regi_handle.mass_insert(dirname=path, basename=key_name) new_path = DB.expand("%s/%s/", (path, nk_key['key_name'])) new_path = FlagFramework.normpath(new_path) ## Store all the values: for v in nk_key.values(): reg_handle.mass_insert(inode_id=inode_id, path=new_path, offset=v['data']['offs_data'], _modified="from_unixtime(%d)" % nk_key['WriteTS'].get_value(), type=v['data']['val_type'], reg_key=v['keyname'], value=v['data']) for k in nk_key.keys(): store_key(k, new_path)