def analyse(self, query):
context = self.get_context(query)
word_id = Indexing.insert_dictionary_word(query['word'], query['type'])
pdbh = DB.DBO()
sql = DB.expand("select inode.inode_id as `inode_id` "\
"%s where (%s) and (%s)", (context.get('tables',''),
context.get('inode_sql','1'),
context.get('where','1')))
Indexing.schedule_inode_index_sql(query['case'],
sql,
word_id,
query['cookie'],
unique=True)
## Now wait here until everyone is finished:
while 1:
pdbh.execute("select count(*) as c from jobs where cookie=%r",
query['cookie'])
row = pdbh.fetch()
self.rows_left = row['c']
if row['c'] == 0: break
time.sleep(1)
return 1
def form(self, query, result):
## If the new word is already in the dictionary just analyse
## it
try:
if Indexing.is_word_in_dict(query['word']):
query['cookie'] = time.time()
query.default('class', "English")
query.default('type', "word")
result.refresh(0, query)
except KeyError:
pass
result.textfield("Word to add", "word")
result.selector(
'Classification:', 'class',
'select class as `key`,class as `value` from dictionary where left(class,1) != \'_\' group by class order by class',
())
result.textfield('(Or create a new class:)', 'class_override')
result.const_selector('Type:', 'type', ('word', 'literal', 'regex'),
('Word', 'Literal', 'RegEx'))
result.hidden("cookie", time.time(), exclusive=True)
if query.has_key("class_override") and len(
query['class_override']) > 2:
query['class'] = query['class_override']
## Refresh ourselves to update to the new class name
result.refresh(0, query)
def ensure_carver_signatures_in_dictionary(carver):
carver.ids = []
for word in carver.regexs:
id = Indexing.insert_dictionary_word(word, word_type='regex',
classification='_Carver',
binary=True)
## Make sure the carver knows about it
carver.ids.append(id)
def display(self, query, result):
## The dictionary is site wide and lives in the FlagDB
dbh = DB.DBO()
## class_override the class variable:
try:
if len(query['class_override']) > 3:
del query['class']
query['class'] = query['class_override']
del query['class_override']
except KeyError:
pass
status = ''
## Do we need to add a new entry:
try:
if len(query['word']) < 3:
raise DB.DBError(
"Word is too short to index, minimum of 3 letter words")
if query['action'] == 'insert':
try:
if len(query['class']) < 3:
raise DB.DBError(
"Class name is too short, minimum of 3 letter words are used as class names"
)
except KeyError:
status = "Classification missing or too short"
raise
Indexing.insert_dictionary_word(query['word'], query['type'],
query['class'])
status = "Added word %s to dictionary" % query['word']
elif query['action'] == 'delete':
dbh.delete("dictionary",
where=DB.expand("word=%b and type=%r",
(query['word'], query['type'])))
status = "Deleted word %s from dictionary" % query['word']
except KeyError, e:
pass
def display(self,query,result):
## The dictionary is site wide and lives in the FlagDB
dbh=DB.DBO()
## class_override the class variable:
try:
if len(query['class_override'])>3:
del query['class']
query['class']=query['class_override']
del query['class_override']
except KeyError:
pass
status = ''
## Do we need to add a new entry:
try:
if len(query['word'])<3:
raise DB.DBError("Word is too short to index, minimum of 3 letter words")
if query['action']=='insert':
try:
if len(query['class'])<3:
raise DB.DBError("Class name is too short, minimum of 3 letter words are used as class names")
except KeyError:
status = "Classification missing or too short"
raise
Indexing.insert_dictionary_word(query['word'], query['type'],
query['class'])
status = "Added word %s to dictionary" % query['word']
elif query['action']=='delete':
dbh.delete("dictionary",
where=DB.expand("word=%b and type=%r",
(query['word'],query['type'])))
status = "Deleted word %s from dictionary" % query['word']
except KeyError,e:
pass
def analyse(self, query):
context = self.get_context(query)
word_id = Indexing.insert_dictionary_word(query['word'], query['type'])
pdbh = DB.DBO()
sql = DB.expand("select inode.inode_id as `inode_id` "\
"%s where (%s) and (%s)", (context.get('tables',''),
context.get('inode_sql','1'),
context.get('where','1')))
Indexing.schedule_inode_index_sql(query['case'],
sql, word_id, query['cookie'], unique=True)
## Now wait here until everyone is finished:
while 1:
pdbh.execute("select count(*) as c from jobs where cookie=%r",
query['cookie'])
row = pdbh.fetch()
self.rows_left = row['c']
if row['c']==0: break
time.sleep(1)
return 1
def form(self, query,result):
## If the new word is already in the dictionary just analyse
## it
try:
if Indexing.is_word_in_dict(query['word']):
query['cookie'] = time.time()
query.default('class', "English")
query.default('type', "word")
result.refresh(0, query)
except KeyError: pass
result.textfield("Word to add", "word")
result.selector('Classification:','class','select class as `key`,class as `value` from dictionary where left(class,1) != \'_\' group by class order by class',())
result.textfield('(Or create a new class:)','class_override')
result.const_selector('Type:','type',('word','literal','regex'),('Word','Literal','RegEx'))
result.hidden("cookie", time.time(), exclusive=True)
if query.has_key("class_override") and len(query['class_override'])>2:
query['class'] = query['class_override']
## Refresh ourselves to update to the new class name
result.refresh(0, query)
def reindex():
global INDEX, INDEX_VERSION
pyflaglog.log(pyflaglog.DEBUG, "Index manager: Building index trie")
start_time = time.time()
dbh = DB.DBO()
INDEX_VERSION = Indexing.get_dict_version()
dbh.execute("select word,id,type,class from dictionary")
INDEX = index.Index()
for row in dbh:
## Classes starting with _ are private classes and want to
## return all hits.
if row['class'].startswith("_"):
id = row['id'] + 2**30
else:
id = row['id']
t = row['type']
## Literal and extended are encoded using latin
if t == 'literal':
INDEX.add_word(row['word'].decode("latin").encode("latin"), id,
index.WORD_LITERAL)
elif t == 'regex':
if type(row['word']) == str:
word = row['word'].decode('latin')
else:
word = row['word']
INDEX.add_word(word.encode("latin"), id, index.WORD_EXTENDED)
elif t == 'word':
try:
word = row['word'].decode("UTF-8").lower()
for e in config.INDEX_ENCODINGS.split(","):
w = word.encode(e)
if len(w) >= 3:
INDEX.add_word(w, id, index.WORD_ENGLISH)
except UnicodeDecodeError, error:
pyflaglog.log(
pyflaglog.ERROR,
"Unable to encode in encoding %e: %s" % (e, error))
def outstanding_inodes(self, word_id=None):
## Remove any references to the LogicalIndexOffsets table
## - this will allow us to count how many inodes are
## involved in the re-indexing without making references
## to this column type.
elements = [ e for e in self.elements if e.table != "LogicalIndexOffsets" ]
## Calculate the new filter string
try:
new_filter_string = self.ui.defaults[self.filter]
sql = parser.parse_to_sql(new_filter_string,
elements, None)
except KeyError:
sql = '1'
tables = UI._make_join_clause(elements)
final_sql = DB.expand("%s where (%s) and (%s)",
(tables, sql,
self.table_where_clause))
count, total = Indexing.count_outdated_inodes(self.case, final_sql, word_id=word_id)
return count, total, tables, sql
def reindex():
global INDEX, INDEX_VERSION
pyflaglog.log(pyflaglog.DEBUG,"Index manager: Building index trie")
start_time = time.time()
dbh = DB.DBO()
INDEX_VERSION = Indexing.get_dict_version()
dbh.execute("select word,id,type,class from dictionary")
INDEX = index.Index()
for row in dbh:
## Classes starting with _ are private classes and want to
## return all hits.
if row['class'].startswith("_"):
id = row['id'] + 2**30
else:
id = row['id']
t = row['type']
## Literal and extended are encoded using latin
if t == 'literal':
INDEX.add_word(row['word'].decode("latin").encode("latin"),id, index.WORD_LITERAL)
elif t == 'regex':
if type(row['word'])==str:
word = row['word'].decode('latin')
else:
word = row['word']
INDEX.add_word(word.encode("latin"),id, index.WORD_EXTENDED)
elif t=='word':
try:
word = row['word'].decode("UTF-8").lower()
for e in config.INDEX_ENCODINGS.split(","):
w = word.encode(e)
if len(w)>=3:
INDEX.add_word(w,id, index.WORD_ENGLISH)
except UnicodeDecodeError,error:
pyflaglog.log(pyflaglog.ERROR, "Unable to encode in encoding %e: %s" % (e,error))
