def __init__(self, cache_name):
self.__cache_name = cache_name
if ES.connection('es') is None:
Logger.log(__name__, 'could not connect to elasticsearch', type='error')
return
ESService.create_index(self.__cache_name, Config.get('elasticsearch')['indices_settings'][self.__cache_name])
def clear_backedoff_storage():
storage_path = Data.get(
Config.get('CTI')['backed_off_search']['storage_name'])
if os.path.isfile(storage_path):
os.remove(storage_path)
Logger.log(__name__, 'the backedoff search storage is now empty')
else:
Logger.log(__name__, 'the backedoff search storage is already empty')
def bootstrap_knowledge_base(dump_directory):
Logger.log(__name__, 'bootstrapping knowledge base...')
contexts_cache = ContextsCache.create(
Config.get('CTI')['knowledge_base']['contexts_cache'])
for article in WikipediaUtils.get_articles(dump_directory,
attributes=['title',
'context']):
contexts_cache.set(article['title'], article['context'])
Logger.log(__name__,
'bootstrapping got context for: ' + article['title'])
def delete_index(name):
try:
if not ES.connection('es').indices.exists(name):
Logger.log(__name__,
'index ' + name + ' does not exist',
type='error')
return False
ES.connection('es').indices.delete(index=name)
except Exception as ex:
Logger.log(__name__,
'could not delete index ' + name + '\t' + str(ex),
type='error')
return False
def create_index(name, config):
try:
if ES.connection('es').indices.exists(name):
Logger.log(__name__,
'index ' + name + ' already exists',
type='warning')
return True
ES.connection('es').indices.create(index=name, body=config)
except Exception as ex:
Logger.log(__name__,
'could not create index ' + name + '\t' + str(ex),
type='error')
return False
return True
def backed_off_search():
config = Config.get('CTI')['backed_off_search']
contexts_cache = ContextsCache.create(
Config.get('CTI')['knowledge_base']['contexts_cache'])
search_engine = SearchEngine()
Logger.log(__name__, 'backed off search process started')
Logger.log(
__name__, 'backed off search storage has ' +
str(len(SQLiteDict.storage(config['storage_name']))) + ' items')
c = 0
while True:
try:
title = SQLiteDict.storage(config['storage_name']).popitem()[0]
except KeyError:
sleep(config['empty_storage_wait_seconds'])
continue
contexts_cache.set(title, search_engine.context(title))
Logger.log(__name__, 'backed off search got context for: ' + title)
c += 1
if c > 30:
c = 0
Logger.log(
__name__, 'backed off search storage has ' +
str(len(SQLiteDict.storage(config['storage_name']))) +
' items')
sleep(config['seconds_between_searches'])
def clear_knowledge_base():
ESService.delete_index(
Config.get('CTI')['knowledge_base']['query_cache']['name'])
ESService.delete_index(
Config.get('CTI')['knowledge_base']['contexts_cache']['name'])
Logger.log(__name__, 'the knowledge base is now empty')