def rabbit(bibrecs, check_invalid_papers=False, personids_to_update_extids=None): ''' @param bibrecs: an iterable full of bibrecs @type bibrecs: an iterable of ints @return: none ''' if bconfig.RABBIT_USE_CACHED_PID: PID_NAMES_CACHE = get_name_string_to_pid_dictionary() def find_pids_by_exact_names_cache(name): try: return zip(PID_NAMES_CACHE[name]) except KeyError: return [] def add_signature_using_names_cache(sig, name, pid): try: PID_NAMES_CACHE[name].add(pid) except KeyError: PID_NAMES_CACHE[name] = set([pid]) _add_signature(sig, name, pid) def new_person_from_signature_using_names_cache(sig, name): pid = get_new_personid() add_signature_using_names_cache(sig, name, pid) return pid add_signature = add_signature_using_names_cache new_person_from_signature = new_person_from_signature_using_names_cache find_pids_by_exact_name = find_pids_by_exact_names_cache else: add_signature = _add_signature new_person_from_signature = _new_person_from_signature find_pids_by_exact_name = _find_pids_by_exact_name compare_names = cached_sym(lambda x: x)(comp_names) # fast assign threshold threshold = 0.80 if not bibrecs or check_invalid_papers: all_bibrecs = get_all_valid_bibrecs() if not bibrecs: bibrecs = all_bibrecs if check_invalid_papers: filter_bibrecs_outside(all_bibrecs) if (bconfig.RABBIT_USE_CACHED_GET_GROUPED_RECORDS and len(bibrecs) > bconfig.RABBIT_USE_CACHED_GET_GROUPED_RECORDS_THRESHOLD): populate_partial_marc_caches() SWAPPED_GET_GROUPED_RECORDS = True else: SWAPPED_GET_GROUPED_RECORDS = False updated_pids = set() deleted = frozenset(p[0] for p in get_deleted_papers()) for idx, rec in enumerate(bibrecs): task_sleep_now_if_required(True) update_status(float(idx) / len(bibrecs), "%d/%d current: %d" % (idx, len(bibrecs), rec)) if rec in deleted: delete_paper_from_personid(rec) continue markrefs = frozenset(chain(izip(cycle([100]), imap(itemgetter(0), get_authors_from_paper(rec))), izip(cycle([700]), imap(itemgetter(0), get_coauthors_from_paper(rec))))) personid_rows = [map(int, row[:3]) + [row[4]] for row in get_signatures_from_rec(rec)] personidrefs_names = dict(((row[1], row[2]), row[3]) for row in personid_rows) personidrefs = frozenset(personidrefs_names.keys()) new_signatures = list(markrefs - personidrefs) old_signatures = list(personidrefs - markrefs) new_signatures_names = dict((new, create_normalized_name(split_name_parts(get_name_by_bibrecref(new)))) for new in new_signatures) # matrix |new_signatures| X |old_signatures| matrix = [[compare_names(new_signatures_names[new], personidrefs_names[old]) for old in old_signatures] for new in new_signatures] # [(new_signatures, old_signatures)] best_match = [(new_signatures[new], old_signatures[old]) for new, old, score in maximized_mapping(matrix) if score > threshold] for new, old in best_match: modify_signature(old, rec, new, new_signatures_names[new]) remove_sigs(tuple(list(old) + [rec]) for old in old_signatures) not_matched = frozenset(new_signatures) - frozenset(map(itemgetter(0), best_match)) if not_matched: used_pids = set(r[0] for r in personid_rows) for sig in not_matched: name = new_signatures_names[sig] matched_pids = [] if USE_EXT_IDS: if USE_INSPIREID: inspire_id = get_inspire_id(sig + (rec,)) if inspire_id: matched_pids = list(get_person_with_extid(inspire_id[0])) if matched_pids: add_signature(list(sig) + [rec], name, matched_pids[0][0]) updated_pids.add(matched_pids[0][0]) continue matched_pids = find_pids_by_exact_name(name) matched_pids = [p for p in matched_pids if int(p[0]) not in used_pids] if not matched_pids: new_pid = new_person_from_signature(list(sig) + [rec], name) used_pids.add(new_pid) updated_pids.add(new_pid) else: add_signature(list(sig) + [rec], name, matched_pids[0][0]) used_pids.add(matched_pids[0][0]) updated_pids.add(matched_pids[0][0]) update_status_final() if personids_to_update_extids: updated_pids |= personids_to_update_extids if updated_pids: # an empty set will update all canonical_names update_personID_canonical_names(updated_pids) update_personID_external_ids(updated_pids, limit_to_claimed_papers=bconfig.LIMIT_EXTERNAL_IDS_COLLECTION_TO_CLAIMED_PAPERS) if SWAPPED_GET_GROUPED_RECORDS: destroy_partial_marc_caches()
def merge_dynamic(): ''' This function merges aidPERSONIDPAPERS with aidRESULTS. Use it after tortoise. This function is dynamic: it allows aid* tables to be changed while it is still running, hence the claiming faciity for example can stay online during the merge. This comfort however is paid off in term of speed. ''' last_names = frozenset(name[0].split('.')[0] for name in get_existing_result_clusters()) def get_free_pids(): while True: yield get_new_personid() free_pids = get_free_pids() def try_move_signature(sig, target_pid): """ """ paps = get_signature_info(sig) rejected = filter(lambda p: p[1] <= -2, paps) assigned = filter(lambda p:-2 < p[1] and p[1] < 2, paps) claimed = filter(lambda p: 2 <= p[1] and p[0] == target_pid, paps) if claimed or not assigned or assigned[0] == target_pid: return assert len(assigned) == 1 if rejected: move_signature(sig, free_pids.next()) else: conflicts = find_conflicts(sig, target_pid) if not conflicts: move_signature(sig, target_pid) else: assert len(conflicts) == 1 if conflicts[0][3] == 2: move_signature(sig, free_pids.next()) else: move_signature(conflicts[0][:3], free_pids.next()) move_signature(sig, target_pid) for idx, last in enumerate(last_names): update_status(float(idx) / len(last_names), "%d/%d current: %s" % (idx, len(last_names), last)) results = ((int(row[0].split(".")[1]), row[1:4]) for row in get_lastname_results(last)) # [(last name number, [bibrefrecs])] results = [(k, map(itemgetter(1), d)) for k, d in groupby(sorted(results, key=itemgetter(0)), key=itemgetter(0))] # List of dictionaries. # [{new_pid -> N}] matr = [] # Set of all old pids. old_pids = set() for k, ds in results: pids = [] claim = [] for d in ds: pid_flag = personid_from_signature(d) if pid_flag: pid, flag = pid_flag[0] pids.append(pid) old_pids.add(pid) if flag > 1: claim.append((d, pid)) matr.append(dict((k, len(list(d))) for k, d in groupby(sorted(pids)))) # We cast it to list in order to ensure the order persistence. old_pids = list(old_pids) best_match = maximized_mapping([[row.get(old, 0) for old in old_pids] for row in matr]) matched_clusters = [(results[new_idx][1], old_pids[old_idx]) for new_idx, old_idx, _ in best_match] not_matched_clusters = frozenset(xrange(len(results))) - frozenset(imap(itemgetter(0), best_match)) not_matched_clusters = izip((results[i][1] for i in not_matched_clusters), free_pids) for sigs, pid in chain(matched_clusters, not_matched_clusters): for sig in sigs: try_move_signature(sig, pid) update_status_final() delete_empty_persons() update_personID_canonical_names()
def merge_static_classy(): ''' This function merges aidPERSONIDPAPERS with aidRESULTS. Use it after tortoise. This function is static: if aid* tables are changed while it's running, probably everything will crash and a black hole will open, eating all your data. NOTE: this is more elegant that merge_static but much slower. Will have to be improved before it can replace it. ''' class Sig(object): def __init__(self, bibrefrec, pid_flag): self.rejected = dict(filter(lambda p: p[1] <= -2, pid_flag)) self.assigned = filter(lambda p:-2 < p[1] and p[1] < 2, pid_flag) self.claimed = filter(lambda p: 2 <= p[1], pid_flag) self.bibrefrec = bibrefrec assert self.invariant() def invariant(self): return len(self.assigned) + len(self.claimed) <= 1 def empty(self): return not self.isclaimed and not self.isassigned def isclaimed(self): return len(self.claimed) == 1 def get_claimed(self): return self.claimed[0][0] def get_assigned(self): return self.assigned[0][0] def isassigned(self): return len(self.assigned) == 1 def isrejected(self, pid): return pid in self.rejected def change_pid(self, pid): assert self.invariant() assert self.isassigned() self.assigned = [(pid, 0)] move_signature(self.bibrefrec, pid) class Cluster(object): def __init__(self, pid, sigs): self.pid = pid self.sigs = dict((sig.bibrefrec[2], sig) for sig in sigs if not sig.empty()) def send_sig(self, other, sig): paper = sig.bibrefrec[2] assert paper in self.sigs and paper not in other.sigs del self.sigs[paper] other.sigs[paper] = sig if sig.isassigned(): sig.change_pid(other.pid) last_names = frozenset(name[0].split('.')[0] for name in get_existing_result_clusters()) personid = get_bibrefrec_to_pid_flag_mapping() free_pids = backinterface_get_free_pids() for idx, last in enumerate(last_names): update_status(float(idx) / len(last_names), "Merging, %d/%d current: %s" % (idx, len(last_names), last)) results = ((int(row[0].split(".")[1]), row[1:4]) for row in get_lastname_results(last)) # [(last name number, [bibrefrecs])] results = [(k, map(itemgetter(1), d)) for k, d in groupby(sorted(results, key=itemgetter(0)), key=itemgetter(0))] # List of dictionaries. # [{new_pid -> N}] matr = [] # Set of all old pids. old_pids = set() for k, ds in results: pids = [] for d in ds: pid_flag = filter(lambda x: x[1] > -2, personid.get(d, [])) if pid_flag: assert len(pid_flag) == 1 pid = pid_flag[0][0] pids.append(pid) old_pids.add(pid) matr.append(dict((k, len(list(d))) for k, d in groupby(sorted(pids)))) old_pids = list(old_pids) best_match = maximized_mapping([[row.get(old, 0) for old in old_pids] for row in matr]) # [[bibrefrecs] -> pid] matched_clusters = [(results[new_idx][1], old_pids[old_idx]) for new_idx, old_idx, _ in best_match] not_matched_clusters = frozenset(xrange(len(results))) - frozenset(imap(itemgetter(0), best_match)) not_matched_clusters = izip((results[i][1] for i in not_matched_clusters), free_pids) # pid -> Cluster clusters = dict((pid, Cluster(pid, [Sig(bib, personid.get(bib, [])) for bib in sigs])) for sigs, pid in chain(matched_clusters, not_matched_clusters)) todo = clusters.items() for pid, clus in todo: assert clus.pid == pid for paper, sig in clus.sigs.items(): if sig.isclaimed(): if sig.get_claimed() != pid: target_clus = clusters[sig.get_claimed()] if paper in target_clus.sigs: new_clus = Cluster(free_pids.next(), []) target_clus.send_sig(new_clus, target_clus[paper]) todo.append(new_clus) clusters[new_clus.pid] = new_clus assert paper not in target_clus.sigs clus.send_sig(target_clus, sig) elif sig.get_assigned() != pid: if not sig.isrejected(pid): move_signature(sig.bibrefrec, pid) else: move_signature(sig.bibrefrec, free_pids.next()) else: assert not sig.isrejected(pid) update_status_final("Merging done.") update_status_final() delete_empty_persons() update_personID_canonical_names()
def merge_static(): ''' This function merges aidPERSONIDPAPERS with aidRESULTS. Use it after tortoise. This function is static: if aid* tables are changed while it's running, probably everything will crash and a black hole will open, eating all your data. ''' last_names = frozenset(name[0].split('.')[0] for name in get_existing_result_clusters()) def get_free_pids(): while True: yield get_new_personid() free_pids = get_free_pids() current_mapping = get_bibrefrec_to_pid_flag_mapping() def move_sig_and_update_mapping(sig, old_pid_flag, new_pid_flag): move_signature(sig, new_pid_flag[0]) current_mapping[sig].remove(old_pid_flag) current_mapping[sig].append(new_pid_flag) def try_move_signature(sig, target_pid): """ """ paps = current_mapping[sig] rejected = filter(lambda p: p[1] <= -2, paps) assigned = filter(lambda p:-2 < p[1] and p[1] < 2, paps) claimed = filter(lambda p: 2 <= p[1] and p[0] == target_pid, paps) if claimed or not assigned or assigned[0] == target_pid: return assert len(assigned) == 1 if rejected: newpid = free_pids.next() move_sig_and_update_mapping(sig, assigned[0], (newpid, assigned[0][1])) else: conflicts = find_conflicts(sig, target_pid) if not conflicts: move_sig_and_update_mapping(sig, assigned[0], (target_pid, assigned[0][1])) else: assert len(conflicts) == 1 if conflicts[0][3] == 2: newpid = free_pids.next() move_sig_and_update_mapping(sig, assigned[0], (newpid, assigned[0][1])) else: newpid = free_pids.next() csig = tuple(conflicts[0][:3]) move_sig_and_update_mapping(csig, (target_pid, conflicts[0][3]), (newpid, conflicts[0][3])) move_sig_and_update_mapping(sig, assigned[0], (target_pid, assigned[0][1])) for idx, last in enumerate(last_names): update_status(float(idx) / len(last_names), "%d/%d current: %s" % (idx, len(last_names), last)) results = ((int(row[0].split(".")[1]), row[1:4]) for row in get_lastname_results(last)) # [(last name number, [bibrefrecs])] results = [(k, map(itemgetter(1), d)) for k, d in groupby(sorted(results, key=itemgetter(0)), key=itemgetter(0))] # List of dictionaries. # [{new_pid -> N}] matr = [] # Set of all old pids. old_pids = set() for k, ds in results: pids = [] claim = [] for d in ds: pid_flag = current_mapping.get(d, []) if pid_flag: pid, flag = pid_flag[0] pids.append(pid) old_pids.add(pid) if flag > 1: claim.append((d, pid)) matr.append(dict((k, len(list(d))) for k, d in groupby(sorted(pids)))) # We cast it to list in order to ensure the order persistence. old_pids = list(old_pids) best_match = maximized_mapping([[row.get(old, 0) for old in old_pids] for row in matr]) matched_clusters = [(results[new_idx][1], old_pids[old_idx]) for new_idx, old_idx, _ in best_match] not_matched_clusters = frozenset(xrange(len(results))) - frozenset(imap(itemgetter(0), best_match)) not_matched_clusters = izip((results[i][1] for i in not_matched_clusters), free_pids) for sigs, pid in chain(matched_clusters, not_matched_clusters): for sig in sigs: if sig in current_mapping: if not pid in map(itemgetter(0), filter(lambda x: x[1] > -2, current_mapping[sig])): try_move_signature(sig, pid) update_status_final() delete_empty_persons() update_personID_canonical_names()
def merge_static_classy(): ''' This function merges aidPERSONIDPAPERS with aidRESULTS. Use it after tortoise. This function is static: if aid* tables are changed while it's running, probably everything will crash and a black hole will open, eating all your data. NOTE: this is more elegant that merge_static but much slower. Will have to be improved before it can replace it. ''' class Sig(object): def __init__(self, bibrefrec, pid_flag): self.rejected = dict(filter(lambda p: p[1] <= -2, pid_flag)) self.assigned = filter(lambda p: -2 < p[1] and p[1] < 2, pid_flag) self.claimed = filter(lambda p: 2 <= p[1], pid_flag) self.bibrefrec = bibrefrec assert self.invariant() def invariant(self): return len(self.assigned) + len(self.claimed) <= 1 def empty(self): return not self.isclaimed and not self.isassigned def isclaimed(self): return len(self.claimed) == 1 def get_claimed(self): return self.claimed[0][0] def get_assigned(self): return self.assigned[0][0] def isassigned(self): return len(self.assigned) == 1 def isrejected(self, pid): return pid in self.rejected def change_pid(self, pid): assert self.invariant() assert self.isassigned() self.assigned = [(pid, 0)] move_signature(self.bibrefrec, pid) class Cluster(object): def __init__(self, pid, sigs): self.pid = pid self.sigs = dict( (sig.bibrefrec[2], sig) for sig in sigs if not sig.empty()) def send_sig(self, other, sig): paper = sig.bibrefrec[2] assert paper in self.sigs and paper not in other.sigs del self.sigs[paper] other.sigs[paper] = sig if sig.isassigned(): sig.change_pid(other.pid) last_names = frozenset(name[0].split('.')[0] for name in get_existing_result_clusters()) personid = get_bibrefrec_to_pid_flag_mapping() free_pids = backinterface_get_free_pids() for idx, last in enumerate(last_names): update_status( float(idx) / len(last_names), "Merging, %d/%d current: %s" % (idx, len(last_names), last)) results = ((int(row[0].split(".")[1]), row[1:4]) for row in get_lastname_results(last)) # [(last name number, [bibrefrecs])] results = [(k, map(itemgetter(1), d)) for k, d in groupby(sorted(results, key=itemgetter(0)), key=itemgetter(0))] # List of dictionaries. # [{new_pid -> N}] matr = [] # Set of all old pids. old_pids = set() for k, ds in results: pids = [] for d in ds: pid_flag = filter(lambda x: x[1] > -2, personid.get(d, [])) if pid_flag: assert len(pid_flag) == 1 pid = pid_flag[0][0] pids.append(pid) old_pids.add(pid) matr.append( dict((k, len(list(d))) for k, d in groupby(sorted(pids)))) old_pids = list(old_pids) best_match = maximized_mapping([[row.get(old, 0) for old in old_pids] for row in matr]) # [[bibrefrecs] -> pid] matched_clusters = [(results[new_idx][1], old_pids[old_idx]) for new_idx, old_idx, _ in best_match] not_matched_clusters = frozenset(xrange(len(results))) - frozenset( imap(itemgetter(0), best_match)) not_matched_clusters = izip( (results[i][1] for i in not_matched_clusters), free_pids) # pid -> Cluster clusters = dict( (pid, Cluster(pid, [Sig(bib, personid.get(bib, [])) for bib in sigs])) for sigs, pid in chain(matched_clusters, not_matched_clusters)) todo = clusters.items() for pid, clus in todo: assert clus.pid == pid for paper, sig in clus.sigs.items(): if sig.isclaimed(): if sig.get_claimed() != pid: target_clus = clusters[sig.get_claimed()] if paper in target_clus.sigs: new_clus = Cluster(free_pids.next(), []) target_clus.send_sig(new_clus, target_clus[paper]) todo.append(new_clus) clusters[new_clus.pid] = new_clus assert paper not in target_clus.sigs clus.send_sig(target_clus, sig) elif sig.get_assigned() != pid: if not sig.isrejected(pid): move_signature(sig.bibrefrec, pid) else: move_signature(sig.bibrefrec, free_pids.next()) else: assert not sig.isrejected(pid) update_status_final("Merging done.") update_status_final() delete_empty_persons() update_personID_canonical_names()
def merge_dynamic(): ''' This function merges aidPERSONIDPAPERS with aidRESULTS. Use it after tortoise. This function is dynamic: it allows aid* tables to be changed while it is still running, hence the claiming faciity for example can stay online during the merge. This comfort however is paid off in term of speed. ''' last_names = frozenset(name[0].split('.')[0] for name in get_existing_result_clusters()) def get_free_pids(): while True: yield get_new_personid() free_pids = get_free_pids() def try_move_signature(sig, target_pid): """ """ paps = get_signature_info(sig) rejected = filter(lambda p: p[1] <= -2, paps) assigned = filter(lambda p: -2 < p[1] and p[1] < 2, paps) claimed = filter(lambda p: 2 <= p[1] and p[0] == target_pid, paps) if claimed or not assigned or assigned[0] == target_pid: return assert len(assigned) == 1 if rejected: move_signature(sig, free_pids.next()) else: conflicts = find_conflicts(sig, target_pid) if not conflicts: move_signature(sig, target_pid) else: assert len(conflicts) == 1 if conflicts[0][3] == 2: move_signature(sig, free_pids.next()) else: move_signature(conflicts[0][:3], free_pids.next()) move_signature(sig, target_pid) for idx, last in enumerate(last_names): update_status( float(idx) / len(last_names), "%d/%d current: %s" % (idx, len(last_names), last)) results = ((int(row[0].split(".")[1]), row[1:4]) for row in get_lastname_results(last)) # [(last name number, [bibrefrecs])] results = [(k, map(itemgetter(1), d)) for k, d in groupby(sorted(results, key=itemgetter(0)), key=itemgetter(0))] # List of dictionaries. # [{new_pid -> N}] matr = [] # Set of all old pids. old_pids = set() for k, ds in results: pids = [] claim = [] for d in ds: pid_flag = personid_from_signature(d) if pid_flag: pid, flag = pid_flag[0] pids.append(pid) old_pids.add(pid) if flag > 1: claim.append((d, pid)) matr.append( dict((k, len(list(d))) for k, d in groupby(sorted(pids)))) # We cast it to list in order to ensure the order persistence. old_pids = list(old_pids) best_match = maximized_mapping([[row.get(old, 0) for old in old_pids] for row in matr]) matched_clusters = [(results[new_idx][1], old_pids[old_idx]) for new_idx, old_idx, _ in best_match] not_matched_clusters = frozenset(xrange(len(results))) - frozenset( imap(itemgetter(0), best_match)) not_matched_clusters = izip( (results[i][1] for i in not_matched_clusters), free_pids) for sigs, pid in chain(matched_clusters, not_matched_clusters): for sig in sigs: try_move_signature(sig, pid) update_status_final() delete_empty_persons() update_personID_canonical_names()
def merge_static(): ''' This function merges aidPERSONIDPAPERS with aidRESULTS. Use it after tortoise. This function is static: if aid* tables are changed while it's running, probably everything will crash and a black hole will open, eating all your data. ''' last_names = frozenset(name[0].split('.')[0] for name in get_existing_result_clusters()) def get_free_pids(): while True: yield get_new_personid() free_pids = get_free_pids() current_mapping = get_bibrefrec_to_pid_flag_mapping() def move_sig_and_update_mapping(sig, old_pid_flag, new_pid_flag): move_signature(sig, new_pid_flag[0]) current_mapping[sig].remove(old_pid_flag) current_mapping[sig].append(new_pid_flag) def try_move_signature(sig, target_pid): """ """ paps = current_mapping[sig] rejected = filter(lambda p: p[1] <= -2, paps) assigned = filter(lambda p: -2 < p[1] and p[1] < 2, paps) claimed = filter(lambda p: 2 <= p[1] and p[0] == target_pid, paps) if claimed or not assigned or assigned[0] == target_pid: return assert len(assigned) == 1 if rejected: newpid = free_pids.next() move_sig_and_update_mapping(sig, assigned[0], (newpid, assigned[0][1])) else: conflicts = find_conflicts(sig, target_pid) if not conflicts: move_sig_and_update_mapping(sig, assigned[0], (target_pid, assigned[0][1])) else: assert len(conflicts) == 1 if conflicts[0][3] == 2: newpid = free_pids.next() move_sig_and_update_mapping(sig, assigned[0], (newpid, assigned[0][1])) else: newpid = free_pids.next() csig = tuple(conflicts[0][:3]) move_sig_and_update_mapping(csig, (target_pid, conflicts[0][3]), (newpid, conflicts[0][3])) move_sig_and_update_mapping(sig, assigned[0], (target_pid, assigned[0][1])) for idx, last in enumerate(last_names): update_status( float(idx) / len(last_names), "%d/%d current: %s" % (idx, len(last_names), last)) results = ((int(row[0].split(".")[1]), row[1:4]) for row in get_lastname_results(last)) # [(last name number, [bibrefrecs])] results = [(k, map(itemgetter(1), d)) for k, d in groupby(sorted(results, key=itemgetter(0)), key=itemgetter(0))] # List of dictionaries. # [{new_pid -> N}] matr = [] # Set of all old pids. old_pids = set() for k, ds in results: pids = [] claim = [] for d in ds: pid_flag = current_mapping.get(d, []) if pid_flag: pid, flag = pid_flag[0] pids.append(pid) old_pids.add(pid) if flag > 1: claim.append((d, pid)) matr.append( dict((k, len(list(d))) for k, d in groupby(sorted(pids)))) # We cast it to list in order to ensure the order persistence. old_pids = list(old_pids) best_match = maximized_mapping([[row.get(old, 0) for old in old_pids] for row in matr]) matched_clusters = [(results[new_idx][1], old_pids[old_idx]) for new_idx, old_idx, _ in best_match] not_matched_clusters = frozenset(xrange(len(results))) - frozenset( imap(itemgetter(0), best_match)) not_matched_clusters = izip( (results[i][1] for i in not_matched_clusters), free_pids) for sigs, pid in chain(matched_clusters, not_matched_clusters): for sig in sigs: if sig in current_mapping: if not pid in map( itemgetter(0), filter(lambda x: x[1] > -2, current_mapping[sig])): try_move_signature(sig, pid) update_status_final() delete_empty_persons() update_personID_canonical_names()
def rabbit(bibrecs, check_invalid_papers=False, personids_to_update_extids=None): ''' @param bibrecs: an iterable full of bibrecs @type bibrecs: an iterable of ints @return: none ''' if bconfig.RABBIT_USE_CACHED_PID: PID_NAMES_CACHE = get_name_string_to_pid_dictionary() def find_pids_by_exact_names_cache(name): try: return zip(PID_NAMES_CACHE[name]) except KeyError: return [] def add_signature_using_names_cache(sig, name, pid): try: PID_NAMES_CACHE[name].add(pid) except KeyError: PID_NAMES_CACHE[name] = set([pid]) _add_signature(sig, name, pid) def new_person_from_signature_using_names_cache(sig, name): pid = get_new_personid() add_signature_using_names_cache(sig, name, pid) return pid add_signature = add_signature_using_names_cache new_person_from_signature = new_person_from_signature_using_names_cache find_pids_by_exact_name = find_pids_by_exact_names_cache else: add_signature = _add_signature new_person_from_signature = _new_person_from_signature find_pids_by_exact_name = _find_pids_by_exact_name compare_names = cached_sym(lambda x: x)(comp_names) # fast assign threshold threshold = 0.80 if not bibrecs or check_invalid_papers: all_bibrecs = get_all_valid_bibrecs() if not bibrecs: bibrecs = all_bibrecs if check_invalid_papers: filter_bibrecs_outside(all_bibrecs) if (bconfig.RABBIT_USE_CACHED_GET_GROUPED_RECORDS and len(bibrecs) > bconfig.RABBIT_USE_CACHED_GET_GROUPED_RECORDS_THRESHOLD): populate_partial_marc_caches() SWAPPED_GET_GROUPED_RECORDS = True else: SWAPPED_GET_GROUPED_RECORDS = False updated_pids = set() deleted = frozenset(p[0] for p in get_deleted_papers()) for idx, rec in enumerate(bibrecs): task_sleep_now_if_required(True) update_status( float(idx) / len(bibrecs), "%d/%d current: %d" % (idx, len(bibrecs), rec)) if rec in deleted: delete_paper_from_personid(rec) continue markrefs = frozenset( chain( izip(cycle([100]), imap(itemgetter(0), get_authors_from_paper(rec))), izip(cycle([700]), imap(itemgetter(0), get_coauthors_from_paper(rec))))) personid_rows = [ map(int, row[:3]) + [row[4]] for row in get_signatures_from_rec(rec) ] personidrefs_names = dict( ((row[1], row[2]), row[3]) for row in personid_rows) personidrefs = frozenset(personidrefs_names.keys()) new_signatures = list(markrefs - personidrefs) old_signatures = list(personidrefs - markrefs) new_signatures_names = dict( (new, create_normalized_name( split_name_parts(get_name_by_bibrecref(new)))) for new in new_signatures) # matrix |new_signatures| X |old_signatures| matrix = [[ compare_names(new_signatures_names[new], personidrefs_names[old]) for old in old_signatures ] for new in new_signatures] # [(new_signatures, old_signatures)] best_match = [(new_signatures[new], old_signatures[old]) for new, old, score in maximized_mapping(matrix) if score > threshold] for new, old in best_match: modify_signature(old, rec, new, new_signatures_names[new]) remove_sigs(tuple(list(old) + [rec]) for old in old_signatures) not_matched = frozenset(new_signatures) - frozenset( map(itemgetter(0), best_match)) if not_matched: used_pids = set(r[0] for r in personid_rows) for sig in not_matched: name = new_signatures_names[sig] matched_pids = [] if USE_EXT_IDS: if USE_INSPIREID: inspire_id = get_inspire_id(sig + (rec, )) if inspire_id: matched_pids = list( get_person_with_extid(inspire_id[0])) if matched_pids: add_signature(list(sig) + [rec], name, matched_pids[0][0]) updated_pids.add(matched_pids[0][0]) continue matched_pids = find_pids_by_exact_name(name) matched_pids = [ p for p in matched_pids if int(p[0]) not in used_pids ] if not matched_pids: new_pid = new_person_from_signature(list(sig) + [rec], name) used_pids.add(new_pid) updated_pids.add(new_pid) else: add_signature(list(sig) + [rec], name, matched_pids[0][0]) used_pids.add(matched_pids[0][0]) updated_pids.add(matched_pids[0][0]) update_status_final() if personids_to_update_extids: updated_pids |= personids_to_update_extids if updated_pids: # an empty set will update all canonical_names update_personID_canonical_names(updated_pids) update_personID_external_ids( updated_pids, limit_to_claimed_papers=bconfig. LIMIT_EXTERNAL_IDS_COLLECTION_TO_CLAIMED_PAPERS) if SWAPPED_GET_GROUPED_RECORDS: destroy_partial_marc_caches()