def rabbit_with_log(papers, check_invalid_papers, log_comment, partial=False):
from bibauthorid_rabbit import rabbit
starting_time = get_sql_time()
rabbit(papers, check_invalid_papers)
if partial:
action = 'PID_UPDATE_PARTIAL'
else:
action = 'PID_UPDATE'
insert_user_log('daemon', '-1', action, 'bibsched', 'status', comment=log_comment, timestamp=starting_time)
def rabbit_with_log(papers, check_invalid_papers, log_comment, partial=False):
from bibauthorid_rabbit import rabbit
starting_time = get_sql_time()
rabbit(papers, check_invalid_papers)
if partial:
action = "PID_UPDATE_PARTIAL"
else:
action = "PID_UPDATE"
insert_user_log("daemon", "-1", action, "bibsched", "status", comment=log_comment, timestamp=starting_time)
def rabbit_with_log(papers, check_invalid_papers, log_comment, partial=False):
from bibauthorid_rabbit import rabbit
starting_time = get_sql_time()
rabbit(papers, check_invalid_papers)
if partial:
action = 'PID_UPDATE_PARTIAL'
else:
action = 'PID_UPDATE'
insert_user_log('daemon', '-1', action, 'bibsched', 'status', comment=log_comment, timestamp=starting_time)
def run_tortoise(from_scratch):
from bibauthorid_tortoise import tortoise, tortoise_from_scratch
if from_scratch:
tortoise_from_scratch()
else:
start_time = get_sql_time()
tortoise_db_name = 'tortoise'
last_run = get_user_log(userinfo=tortoise_db_name, only_most_recent=True)
if last_run:
modified = get_recently_modified_record_ids(last_run[0][2])
else:
modified = []
tortoise(modified)
insert_user_log(tortoise_db_name, '-1', '', '', '', timestamp=start_time)
def run_tortoise(from_scratch):
from bibauthorid_tortoise import tortoise, tortoise_from_scratch
if from_scratch:
tortoise_from_scratch()
else:
start_time = get_sql_time()
tortoise_db_name = 'tortoise'
last_run = get_user_log(userinfo=tortoise_db_name, only_most_recent=True)
if last_run:
modified = get_recently_modified_record_ids(last_run[0][2])
else:
modified = []
tortoise(modified)
insert_user_log(tortoise_db_name, '-1', '', '', '', timestamp=start_time)
def __init__(self, cluster_set, use_cache=False, save_cache=False):
'''
Constructs probability matrix. If use_cache is true, it will
try to load old computations from the database. If save cache
is true it will save the current results into the database.
@param cluster_set: A cluster set object, used to initialize
the matrix.
'''
def check_for_cleaning(cur_calc):
if cur_calc % 10000000 == 0:
clear_comparison_caches()
self._bib_matrix = bib_matrix(cluster_set)
old_matrix = bib_matrix()
ncl = sum(len(cl.bibs) for cl in cluster_set.clusters)
expected = ((ncl * (ncl - 1)) / 2)
if expected == 0:
expected = 1
if use_cache and old_matrix.load(cluster_set.last_name):
cached_bibs = set(filter_modified_record_ids(
old_matrix.get_keys(),
old_matrix.creation_time))
else:
cached_bibs = set()
if save_cache:
creation_time = get_sql_time()
cur_calc, opti = 0, 0
for cl1 in cluster_set.clusters:
update_status((float(opti) + cur_calc) / expected, "Prob matrix: calc %d, opti %d." % (cur_calc, opti))
for cl2 in cluster_set.clusters:
if id(cl1) < id(cl2) and not cl1.hates(cl2):
for bib1 in cl1.bibs:
for bib2 in cl2.bibs:
if bib1 in cached_bibs and bib2 in cached_bibs:
val = old_matrix[bib1, bib2]
if not val:
cur_calc += 1
check_for_cleaning(cur_calc)
val = compare_bibrefrecs(bib1, bib2)
else:
opti += 1
if bconfig.DEBUG_CHECKS:
assert _debug_is_eq_v(val, compare_bibrefrecs(bib1, bib2))
else:
cur_calc += 1
check_for_cleaning(cur_calc)
val = compare_bibrefrecs(bib1, bib2)
self._bib_matrix[bib1, bib2] = val
clear_comparison_caches()
if save_cache:
update_status(1., "saving...")
self._bib_matrix.store(cluster_set.last_name, creation_time)
update_status_final("Matrix done. %d calc, %d opt." % (cur_calc, opti))
