def load_from_db_or_from_galago(table_name, key, galago_fn):
if has_key(table_name, key):
return load(table_name, key)
r = galago_fn()
if not has_key(table_name, key):
save(table_name, key, r)
flush()
return r
def commit_buffer_to_db(buffer):
cnt = 0
skipped_keys = 0
for e in buffer:
table_name, key, value = e
if has_key(table_name, key):
skipped_keys += 1
pass
else:
if skipped_keys:
print("Skipped {} existing keys".format(skipped_keys))
skipped_keys = 0
cnt += 1
save(table_name, key, value)
if cnt > 100:
flush()
def insert_ranked_list(q_res_id: QueryResultID,
ranked_list: List[SimpleRankedListEntry]):
save(QueryResult, q_res_id, ranked_list)
flush()
def add_to_db(query_id: str):
if query_id in ranked_list_dict:
r = ranked_list_dict[query_id]
q_res_id: str = "{}_{}".format(query_id, q_config_id)
if not has_key(QueryResult, q_res_id):
save(QueryResult, q_res_id, r)
def flush(self):
pass
for table_name, key, value in self.buffer:
save(table_name, key, value)
flush()
self.buffer = []
def save_tf(doc_id, tf):
return save(CluewebDocTF, doc_id, tf)