def load_from_db_or_from_galago(table_name, key, galago_fn): if has_key(table_name, key): return load(table_name, key) r = galago_fn() if not has_key(table_name, key): save(table_name, key, r) flush() return r
def commit_buffer_to_db(buffer): cnt = 0 skipped_keys = 0 for e in buffer: table_name, key, value = e if has_key(table_name, key): skipped_keys += 1 pass else: if skipped_keys: print("Skipped {} existing keys".format(skipped_keys)) skipped_keys = 0 cnt += 1 save(table_name, key, value) if cnt > 100: flush()
def insert_ranked_list(q_res_id: QueryResultID, ranked_list: List[SimpleRankedListEntry]): save(QueryResult, q_res_id, ranked_list) flush()
def add_to_db(query_id: str): if query_id in ranked_list_dict: r = ranked_list_dict[query_id] q_res_id: str = "{}_{}".format(query_id, q_config_id) if not has_key(QueryResult, q_res_id): save(QueryResult, q_res_id, r)
def flush(self): pass for table_name, key, value in self.buffer: save(table_name, key, value) flush() self.buffer = []
def save_tf(doc_id, tf): return save(CluewebDocTF, doc_id, tf)