def pages(self):
my_pages = []
if self.normalized_title:
if title_is_too_short(self.normalized_title):
# logger.info(u"title too short! don't match by title")
pass
elif title_is_too_common(self.normalized_title):
# logger.info(u"title too common! don't match by title.")
pass
else:
my_pages = self.page_matches_by_title_filtered
# do dois last, because the objects are actually the same, not copies, and then they get the doi reason
for my_page in self.page_matches_by_doi:
my_page.scrape_evidence = u"oa repository (via OAI-PMH doi match)"
my_pages.append(my_page)
# eventually only apply this filter to matches by title, once pages only includes
# the doi when it comes straight from the pmh record
if max_pages_from_one_repo(
[p.repo_id for p in self.page_matches_by_title_filtered]) >= 3:
my_pages = []
logger.info(
u"matched too many pages in one repo, not allowing matches")
return my_pages
def query_for_num_pub_matches(self):
from pmh_record import title_is_too_common
from pmh_record import title_is_too_short
from pub import Pub
# it takes too long to query for things like "tablecontents"
if title_is_too_common(self.normalized_title) or title_is_too_short(self.normalized_title):
logger.info(u"title is too common or too short, not scraping")
return -1
num_pubs_with_this_normalized_title = db.session.query(Pub.id).filter(Pub.normalized_title==self.normalized_title).count()
return num_pubs_with_this_normalized_title
def query_for_num_pub_matches(self):
from pmh_record import title_is_too_common
from pmh_record import title_is_too_short
from pub import Pub
# it takes too long to query for things like "tablecontents"
if title_is_too_common(self.normalized_title) or title_is_too_short(self.normalized_title):
logger.info(u"title is too common or too short, not scraping")
return -1
num_pubs_with_this_normalized_title = db.session.query(Pub.id).filter(Pub.normalized_title==self.normalized_title).count()
return num_pubs_with_this_normalized_title
