def __get_contexts(self, titles):
contexts = []
misses = []
for title, context in self.__contexts_cache.get(titles).items():
if context is not None:
contexts.append(context)
else:
misses.append(title)
if self.__on_miss_backoff is True:
for title in misses:
SQLiteDict.storage(
Config.get('CTI')['backed_off_search']
['storage_name'])[title] = title
else:
new_contexts = self.__search_engine.contexts(misses)
contexts += list(new_contexts.values())
self.__contexts_cache.set_many(new_contexts)
return contexts
def get(self, titles):
contexts = {}
for title in titles:
contexts[title] = SQLiteDict.storage(self.__cache_name).get(title, default=None)
return contexts
def backed_off_search():
config = Config.get('CTI')['backed_off_search']
contexts_cache = ContextsCache.create(
Config.get('CTI')['knowledge_base']['contexts_cache'])
search_engine = SearchEngine()
Logger.log(__name__, 'backed off search process started')
Logger.log(
__name__, 'backed off search storage has ' +
str(len(SQLiteDict.storage(config['storage_name']))) + ' items')
c = 0
while True:
try:
title = SQLiteDict.storage(config['storage_name']).popitem()[0]
except KeyError:
sleep(config['empty_storage_wait_seconds'])
continue
contexts_cache.set(title, search_engine.context(title))
Logger.log(__name__, 'backed off search got context for: ' + title)
c += 1
if c > 30:
c = 0
Logger.log(
__name__, 'backed off search storage has ' +
str(len(SQLiteDict.storage(config['storage_name']))) +
' items')
sleep(config['seconds_between_searches'])
def set(self, query, response):
SQLiteDict.storage(self.__cache_name)[query] = response
def get(self, query):
return SQLiteDict.storage(self.__cache_name).get(query, default=None)
def set_many(self, contexts_dict):
for title, context in contexts_dict.items():
SQLiteDict.storage(self.__cache_name)[title] = context
def set(self, title, context):
SQLiteDict.storage(self.__cache_name)[title] = context