def rabbit_with_log(papers, check_invalid_papers, log_comment, partial=False): from bibauthorid_rabbit import rabbit starting_time = get_sql_time() rabbit(papers, check_invalid_papers) if partial: action = 'PID_UPDATE_PARTIAL' else: action = 'PID_UPDATE' insert_user_log('daemon', '-1', action, 'bibsched', 'status', comment=log_comment, timestamp=starting_time)
def rabbit_with_log(papers, check_invalid_papers, log_comment, partial=False): from bibauthorid_rabbit import rabbit starting_time = get_sql_time() rabbit(papers, check_invalid_papers) if partial: action = "PID_UPDATE_PARTIAL" else: action = "PID_UPDATE" insert_user_log("daemon", "-1", action, "bibsched", "status", comment=log_comment, timestamp=starting_time)
def run_tortoise(from_scratch): from bibauthorid_tortoise import tortoise, tortoise_from_scratch if from_scratch: tortoise_from_scratch() else: start_time = get_sql_time() tortoise_db_name = 'tortoise' last_run = get_user_log(userinfo=tortoise_db_name, only_most_recent=True) if last_run: modified = get_recently_modified_record_ids(last_run[0][2]) else: modified = [] tortoise(modified) insert_user_log(tortoise_db_name, '-1', '', '', '', timestamp=start_time)
def __init__(self, cluster_set, use_cache=False, save_cache=False): ''' Constructs probability matrix. If use_cache is true, it will try to load old computations from the database. If save cache is true it will save the current results into the database. @param cluster_set: A cluster set object, used to initialize the matrix. ''' def check_for_cleaning(cur_calc): if cur_calc % 10000000 == 0: clear_comparison_caches() self._bib_matrix = bib_matrix(cluster_set) old_matrix = bib_matrix() ncl = sum(len(cl.bibs) for cl in cluster_set.clusters) expected = ((ncl * (ncl - 1)) / 2) if expected == 0: expected = 1 if use_cache and old_matrix.load(cluster_set.last_name): cached_bibs = set(filter_modified_record_ids( old_matrix.get_keys(), old_matrix.creation_time)) else: cached_bibs = set() if save_cache: creation_time = get_sql_time() cur_calc, opti = 0, 0 for cl1 in cluster_set.clusters: update_status((float(opti) + cur_calc) / expected, "Prob matrix: calc %d, opti %d." % (cur_calc, opti)) for cl2 in cluster_set.clusters: if id(cl1) < id(cl2) and not cl1.hates(cl2): for bib1 in cl1.bibs: for bib2 in cl2.bibs: if bib1 in cached_bibs and bib2 in cached_bibs: val = old_matrix[bib1, bib2] if not val: cur_calc += 1 check_for_cleaning(cur_calc) val = compare_bibrefrecs(bib1, bib2) else: opti += 1 if bconfig.DEBUG_CHECKS: assert _debug_is_eq_v(val, compare_bibrefrecs(bib1, bib2)) else: cur_calc += 1 check_for_cleaning(cur_calc) val = compare_bibrefrecs(bib1, bib2) self._bib_matrix[bib1, bib2] = val clear_comparison_caches() if save_cache: update_status(1., "saving...") self._bib_matrix.store(cluster_set.last_name, creation_time) update_status_final("Matrix done. %d calc, %d opt." % (cur_calc, opti))