def matchPub(self,
pubs,
extracted_map,
check_person=False,
debug_output=False):
'''Match pub with extracted
@return (pubs_matched, pubs_not_matched)
@params:
pubs - Publication read from database.
extracted_map - same with all_models {key_title:[ExtractedModel,...]}
check_person - if True, will check if authors is matched with authors in db.(will ignore ...).
default False. Search using author:xxx do not need author check, this work is done by google.
'''
if pubs is None or len(pubs) == 0: return [], pubs
if extracted_map is None or len(extracted_map) == 0: return [], pubs
if self.debug and False:
print 'match %s pubs in %s extracted items' % (len(pubs),
len(extracted_map))
# match
print_not_matched = False
pubs_matched = []
pubs_not_matched = []
for pub in pubs:
cleanned_tuple = GoogleDataCleaner.cleanGoogleTitle(pub.title)
key_title = cleanned_tuple[1]
has_dot = cleanned_tuple[2]
# find models list
models = []
if key_title in extracted_map: # title is full, no ignore
models = extracted_map[key_title]
else: # title in results has ..., ignored.
if has_dot:
for short_key, extracted_models in extracted_map.items():
if key_title.find(short_key) != -1:
models.extend(extracted_models)
# exact match
if models is not None and len(models) > 0:
max_citation_model = None
debug_all_author_string = []
for model in models:
debug_all_author_string.append(model.authors)
if model.authors is None or \
self.matchAuthors(model.authors, pub.authors, debug_output=False, debug_title=model.title): # if author matched.
if max_citation_model is None or int(
max_citation_model.ncitation) < int(
model.ncitation):
max_citation_model = model
# select max citation?
if max_citation_model is not None:
# allow 10% discount
if max_citation_model.ncitation >= pub.ncitation:
pub.ncitation = max_citation_model.ncitation
pub.increased = max_citation_model.ncitation - pub.ncitation
pubs_matched.append(pub)
else:
# citation model not found.
if debug_output:
print "[DEBUG] PubMatcher.matchPub: Author not match. ", \
"\n\tTitle:%s \n\tRequired:%s \n\tGot(last):%s" % \
(pub.title, pub.authors, "\n".join(debug_all_author_string))
# print not matched?
for pub in pubs:
title = pub.title
found = False
for matched in pubs_matched:
if title == matched.title:
found = True
break
if not found:
pubs_not_matched.append(pub)
if print_not_matched: print 'this pub not matched: ', pub
return (pubs_matched, pubs_not_matched)
def matchPub(self,
pubs,
extracted_map,
check_person=False,
debug_output=False):
'''Match pub with extracted
@return (pubs_matched, pubs_not_matched)
@params:
pubs - Publication read from database.
extracted_map - same with all_models {key_title:[ExtractedModel,...]}
check_person - if True, will check if authors is matched with authors in db.(will ignore ...).
default False. Search using author:xxx do not need author check, this work is done by google.
'''
if pubs is None or len(pubs) == 0:
return [], pubs
if extracted_map is None or len(extracted_map) == 0:
return [], pubs
if self.debug and False:
print 'match %s pubs in %s extracted items' % (len(pubs),
len(extracted_map))
# match
print_not_matched = False
pubs_matched = []
pubs_not_matched = []
for pub in pubs:
cleanned_tuple = GoogleDataCleaner.cleanGoogleTitle(pub.title)
key_title = cleanned_tuple[1]
has_dot = cleanned_tuple[2]
# First Match, select loose match. ExtractedPub List
models = []
# different with v1, a looser match.
# title allow 10% mistake.
# author loose match.
for short_key, extracted_models in extracted_map.items():
matched = False
if has_dot:
# ignore and contains
if key_title.find(short_key) != -1:
matched = True
_m = extracted_models
for m in _m: # Add Loose Value
m.looseValue += 1
models.extend(_m)
else:
# direct match
if key_title == short_key:
matched = True
_m = extracted_map[key_title]
for m in _m: # Add Loose Value
m.looseValue += 0
models.extend(_m)
# try loose match
if not matched:
ed = editdist.distance(short_key, key_title)
if ed < 10:
looseValue = float(len(key_title)) * (10 / float(100))
if looseValue > ed: # remove ed not match much
_m = extracted_models
for m in _m:
m.looseValue += ed
models.extend(_m)
# if True and ed < 10:
# print '-' * 100
# print 'title: %s ' % key_title
# print 'short: %s ' % short_key
# print 'ed is: %s ' % ed
# print 'loose: %s ' % looseValue
# Exact match, select who is the right one.
if models is not None and len(models) > 0:
max_citation_model = None
for model in models:
if model.authors is None or \
self.matchAuthors(model.authors, pub.authors, debug_output=False, debug_title=model.title): # if author matched.
if max_citation_model is None or int(
max_citation_model.ncitation) < int(
model.ncitation):
max_citation_model = model
# select max citation?
if max_citation_model is not None:
if max_citation_model.ncitation >= pub.ncitation:
pub.ncitation = max_citation_model.ncitation
pub.increased = max_citation_model.ncitation - pub.ncitation
pub.pdflink = max_citation_model.pdfLink
pub.web_url = max_citation_model.web_url
pubs_matched.append(pub)
if pub.pdflink is None:
file_object = open('web_url.txt', 'a')
web_url = pub.web_url
Id = str(pub.id)
Title = pub.title
Author = str(pub.authors)
file_object.write(" ".join(
[Id, Title, Author, web_url]))
file_object.write("\n")
file_object.close()
else:
file_object = open('paper_link.txt', 'a')
Id = str(pub.id)
Title = pub.title
Pdflink = str(pub.pdflink)
Author = str(pub.authors)
file_object.write(" ".join(
[Id, Title, Author, Pdflink]))
file_object.write("\n")
file_object.close()
for pub in pubs:
title = pub.title
found = False
for matched in pubs_matched:
if title == matched.title:
found = True
break
if not found:
pubs_not_matched.append(pub)
if print_not_matched:
print 'this pub not matched: ', pub
return (pubs_matched, pubs_not_matched)
def matchPub(self, pubs, extracted_map, check_person=False, debug_output=False):
'''Match pub with extracted
@return (pubs_matched, pubs_not_matched)
@params:
pubs - Publication read from database.
extracted_map - same with all_models {key_title:[ExtractedModel,...]}
check_person - if True, will check if authors is matched with authors in db.(will ignore ...).
default False. Search using author:xxx do not need author check, this work is done by google.
'''
if pubs is None or len(pubs) == 0: return [], pubs
if extracted_map is None or len(extracted_map) == 0: return [], pubs
if self.debug and False: print 'match %s pubs in %s extracted items' % (len(pubs), len(extracted_map))
# match
print_not_matched = False
pubs_matched = []
pubs_not_matched = []
for pub in pubs:
cleanned_tuple = GoogleDataCleaner.cleanGoogleTitle(pub.title)
key_title = cleanned_tuple[1]
has_dot = cleanned_tuple[2]
# find models list
models = []
if key_title in extracted_map: # title is full, no ignore
models = extracted_map[key_title]
else: # title in results has ..., ignored.
if has_dot:
for short_key, extracted_models in extracted_map.items():
if key_title.find(short_key) != -1:
models.extend(extracted_models)
# exact match
if models is not None and len(models) > 0:
max_citation_model = None
debug_all_author_string = []
for model in models:
debug_all_author_string.append(model.authors)
if model.authors is None or \
self.matchAuthors(model.authors, pub.authors, debug_output=False, debug_title=model.title): # if author matched.
if max_citation_model is None or int(max_citation_model.ncitation) < int(model.ncitation):
max_citation_model = model
# select max citation?
if max_citation_model is not None:
# allow 10% discount
if max_citation_model.ncitation >= pub.ncitation:
pub.ncitation = max_citation_model.ncitation
pub.increased = max_citation_model.ncitation - pub.ncitation
pubs_matched.append(pub)
else:
# citation model not found.
if debug_output:
print "[DEBUG] PubMatcher.matchPub: Author not match. ", \
"\n\tTitle:%s \n\tRequired:%s \n\tGot(last):%s" % \
(pub.title, pub.authors, "\n".join(debug_all_author_string))
# print not matched?
for pub in pubs:
title = pub.title
found = False;
for matched in pubs_matched:
if title == matched.title:
found = True
break
if not found:
pubs_not_matched.append(pub)
if print_not_matched: print 'this pub not matched: ', pub
return (pubs_matched, pubs_not_matched)
def matchPub(self, pubs, extracted_map, check_person=False, debug_output=False):
'''Match pub with extracted
@return (pubs_matched, pubs_not_matched)
@params:
pubs - Publication read from database.
extracted_map - same with all_models {key_title:[ExtractedModel,...]}
check_person - if True, will check if authors is matched with authors in db.(will ignore ...).
default False. Search using author:xxx do not need author check, this work is done by google.
'''
if pubs is None or len(pubs) == 0:
return [], pubs
if extracted_map is None or len(extracted_map) == 0:
return [], pubs
if self.debug and False:
print 'match %s pubs in %s extracted items' % (len(pubs), len(extracted_map))
# match
print_not_matched = False
pubs_matched = []
pubs_not_matched = []
for pub in pubs:
cleanned_tuple = GoogleDataCleaner.cleanGoogleTitle(pub.title)
key_title = cleanned_tuple[1]
has_dot = cleanned_tuple[2]
# First Match, select loose match. ExtractedPub List
models = []
# different with v1, a looser match.
# title allow 10% mistake.
# author loose match.
for short_key, extracted_models in extracted_map.items():
matched = False
if has_dot:
# ignore and contains
if key_title.find(short_key) != -1:
matched = True
_m = extracted_models
for m in _m: # Add Loose Value
m.looseValue += 1;
models.extend(_m)
else:
# direct match
if key_title == short_key:
matched = True
_m = extracted_map[key_title]
for m in _m: # Add Loose Value
m.looseValue += 0;
models.extend(_m);
# try loose match
if not matched:
ed = editdist.distance(short_key, key_title)
if ed < 10:
looseValue = float(len(key_title)) * (10 / float(100))
if looseValue > ed: # remove ed not match much
_m = extracted_models
for m in _m:
m.looseValue += ed;
models.extend(_m)
# if True and ed < 10:
# print '-' * 100
# print 'title: %s ' % key_title
# print 'short: %s ' % short_key
# print 'ed is: %s ' % ed
# print 'loose: %s ' % looseValue
# Exact match, select who is the right one.
if models is not None and len(models) > 0:
max_citation_model = None
for model in models:
if model.authors is None or \
self.matchAuthors(model.authors, pub.authors, debug_output=False, debug_title=model.title): # if author matched.
if max_citation_model is None or int(max_citation_model.ncitation) < int(model.ncitation):
max_citation_model = model
# select max citation?
if max_citation_model is not None:
if max_citation_model.ncitation >= pub.ncitation:
pub.ncitation = max_citation_model.ncitation
pub.increased = max_citation_model.ncitation - pub.ncitation
pub.pdflink = max_citation_model.pdfLink
pub.web_url = max_citation_model.web_url
pubs_matched.append(pub)
if pub.pdflink is None:
file_object = open('web_url.txt', 'a')
web_url= pub.web_url
Id = str(pub.id)
Title = pub.title
Author = str(pub.authors)
file_object.write(" ".join([Id, Title, Author, web_url]))
file_object.write("\n")
file_object.close()
else:
file_object = open('paper_link.txt', 'a')
Id = str(pub.id)
Title = pub.title
Pdflink = str(pub.pdflink)
Author = str(pub.authors)
file_object.write(" ".join([Id, Title, Author, Pdflink]))
file_object.write("\n")
file_object.close()
for pub in pubs:
title = pub.title
found = False;
for matched in pubs_matched:
if title == matched.title:
found = True
break
if not found:
pubs_not_matched.append(pub)
if print_not_matched:
print 'this pub not matched: ', pub
return (pubs_matched, pubs_not_matched)