def analyse(self, query): context = self.get_context(query) word_id = Indexing.insert_dictionary_word(query['word'], query['type']) pdbh = DB.DBO() sql = DB.expand("select inode.inode_id as `inode_id` "\ "%s where (%s) and (%s)", (context.get('tables',''), context.get('inode_sql','1'), context.get('where','1'))) Indexing.schedule_inode_index_sql(query['case'], sql, word_id, query['cookie'], unique=True) ## Now wait here until everyone is finished: while 1: pdbh.execute("select count(*) as c from jobs where cookie=%r", query['cookie']) row = pdbh.fetch() self.rows_left = row['c'] if row['c'] == 0: break time.sleep(1) return 1
def form(self, query, result): ## If the new word is already in the dictionary just analyse ## it try: if Indexing.is_word_in_dict(query['word']): query['cookie'] = time.time() query.default('class', "English") query.default('type', "word") result.refresh(0, query) except KeyError: pass result.textfield("Word to add", "word") result.selector( 'Classification:', 'class', 'select class as `key`,class as `value` from dictionary where left(class,1) != \'_\' group by class order by class', ()) result.textfield('(Or create a new class:)', 'class_override') result.const_selector('Type:', 'type', ('word', 'literal', 'regex'), ('Word', 'Literal', 'RegEx')) result.hidden("cookie", time.time(), exclusive=True) if query.has_key("class_override") and len( query['class_override']) > 2: query['class'] = query['class_override'] ## Refresh ourselves to update to the new class name result.refresh(0, query)
def ensure_carver_signatures_in_dictionary(carver): carver.ids = [] for word in carver.regexs: id = Indexing.insert_dictionary_word(word, word_type='regex', classification='_Carver', binary=True) ## Make sure the carver knows about it carver.ids.append(id)
def display(self, query, result): ## The dictionary is site wide and lives in the FlagDB dbh = DB.DBO() ## class_override the class variable: try: if len(query['class_override']) > 3: del query['class'] query['class'] = query['class_override'] del query['class_override'] except KeyError: pass status = '' ## Do we need to add a new entry: try: if len(query['word']) < 3: raise DB.DBError( "Word is too short to index, minimum of 3 letter words") if query['action'] == 'insert': try: if len(query['class']) < 3: raise DB.DBError( "Class name is too short, minimum of 3 letter words are used as class names" ) except KeyError: status = "Classification missing or too short" raise Indexing.insert_dictionary_word(query['word'], query['type'], query['class']) status = "Added word %s to dictionary" % query['word'] elif query['action'] == 'delete': dbh.delete("dictionary", where=DB.expand("word=%b and type=%r", (query['word'], query['type']))) status = "Deleted word %s from dictionary" % query['word'] except KeyError, e: pass
def display(self,query,result): ## The dictionary is site wide and lives in the FlagDB dbh=DB.DBO() ## class_override the class variable: try: if len(query['class_override'])>3: del query['class'] query['class']=query['class_override'] del query['class_override'] except KeyError: pass status = '' ## Do we need to add a new entry: try: if len(query['word'])<3: raise DB.DBError("Word is too short to index, minimum of 3 letter words") if query['action']=='insert': try: if len(query['class'])<3: raise DB.DBError("Class name is too short, minimum of 3 letter words are used as class names") except KeyError: status = "Classification missing or too short" raise Indexing.insert_dictionary_word(query['word'], query['type'], query['class']) status = "Added word %s to dictionary" % query['word'] elif query['action']=='delete': dbh.delete("dictionary", where=DB.expand("word=%b and type=%r", (query['word'],query['type']))) status = "Deleted word %s from dictionary" % query['word'] except KeyError,e: pass
def analyse(self, query): context = self.get_context(query) word_id = Indexing.insert_dictionary_word(query['word'], query['type']) pdbh = DB.DBO() sql = DB.expand("select inode.inode_id as `inode_id` "\ "%s where (%s) and (%s)", (context.get('tables',''), context.get('inode_sql','1'), context.get('where','1'))) Indexing.schedule_inode_index_sql(query['case'], sql, word_id, query['cookie'], unique=True) ## Now wait here until everyone is finished: while 1: pdbh.execute("select count(*) as c from jobs where cookie=%r", query['cookie']) row = pdbh.fetch() self.rows_left = row['c'] if row['c']==0: break time.sleep(1) return 1
def form(self, query,result): ## If the new word is already in the dictionary just analyse ## it try: if Indexing.is_word_in_dict(query['word']): query['cookie'] = time.time() query.default('class', "English") query.default('type', "word") result.refresh(0, query) except KeyError: pass result.textfield("Word to add", "word") result.selector('Classification:','class','select class as `key`,class as `value` from dictionary where left(class,1) != \'_\' group by class order by class',()) result.textfield('(Or create a new class:)','class_override') result.const_selector('Type:','type',('word','literal','regex'),('Word','Literal','RegEx')) result.hidden("cookie", time.time(), exclusive=True) if query.has_key("class_override") and len(query['class_override'])>2: query['class'] = query['class_override'] ## Refresh ourselves to update to the new class name result.refresh(0, query)
def reindex(): global INDEX, INDEX_VERSION pyflaglog.log(pyflaglog.DEBUG, "Index manager: Building index trie") start_time = time.time() dbh = DB.DBO() INDEX_VERSION = Indexing.get_dict_version() dbh.execute("select word,id,type,class from dictionary") INDEX = index.Index() for row in dbh: ## Classes starting with _ are private classes and want to ## return all hits. if row['class'].startswith("_"): id = row['id'] + 2**30 else: id = row['id'] t = row['type'] ## Literal and extended are encoded using latin if t == 'literal': INDEX.add_word(row['word'].decode("latin").encode("latin"), id, index.WORD_LITERAL) elif t == 'regex': if type(row['word']) == str: word = row['word'].decode('latin') else: word = row['word'] INDEX.add_word(word.encode("latin"), id, index.WORD_EXTENDED) elif t == 'word': try: word = row['word'].decode("UTF-8").lower() for e in config.INDEX_ENCODINGS.split(","): w = word.encode(e) if len(w) >= 3: INDEX.add_word(w, id, index.WORD_ENGLISH) except UnicodeDecodeError, error: pyflaglog.log( pyflaglog.ERROR, "Unable to encode in encoding %e: %s" % (e, error))
def outstanding_inodes(self, word_id=None): ## Remove any references to the LogicalIndexOffsets table ## - this will allow us to count how many inodes are ## involved in the re-indexing without making references ## to this column type. elements = [ e for e in self.elements if e.table != "LogicalIndexOffsets" ] ## Calculate the new filter string try: new_filter_string = self.ui.defaults[self.filter] sql = parser.parse_to_sql(new_filter_string, elements, None) except KeyError: sql = '1' tables = UI._make_join_clause(elements) final_sql = DB.expand("%s where (%s) and (%s)", (tables, sql, self.table_where_clause)) count, total = Indexing.count_outdated_inodes(self.case, final_sql, word_id=word_id) return count, total, tables, sql
def reindex(): global INDEX, INDEX_VERSION pyflaglog.log(pyflaglog.DEBUG,"Index manager: Building index trie") start_time = time.time() dbh = DB.DBO() INDEX_VERSION = Indexing.get_dict_version() dbh.execute("select word,id,type,class from dictionary") INDEX = index.Index() for row in dbh: ## Classes starting with _ are private classes and want to ## return all hits. if row['class'].startswith("_"): id = row['id'] + 2**30 else: id = row['id'] t = row['type'] ## Literal and extended are encoded using latin if t == 'literal': INDEX.add_word(row['word'].decode("latin").encode("latin"),id, index.WORD_LITERAL) elif t == 'regex': if type(row['word'])==str: word = row['word'].decode('latin') else: word = row['word'] INDEX.add_word(word.encode("latin"),id, index.WORD_EXTENDED) elif t=='word': try: word = row['word'].decode("UTF-8").lower() for e in config.INDEX_ENCODINGS.split(","): w = word.encode(e) if len(w)>=3: INDEX.add_word(w,id, index.WORD_ENGLISH) except UnicodeDecodeError,error: pyflaglog.log(pyflaglog.ERROR, "Unable to encode in encoding %e: %s" % (e,error))