def get_pdf_url_status(pdf_url):
worker = current_process()
logger.info(u'{} checking pdf url: {}'.format(worker, pdf_url))
is_pdf = False
http_status = None
try:
response = http_get(url=pdf_url.url,
ask_slowly=True,
stream=True,
publisher=pdf_url.publisher,
session_id=get_session_id(),
verify=True)
except Exception as e:
logger.error(u"{} failed to get response: {}".format(
worker, e.message))
else:
with response:
try:
is_pdf = is_a_pdf_page(response, pdf_url.publisher)
http_status = response.status_code
except Exception as e:
logger.error(u"{} failed reading response: {}".format(
worker, e.message))
pdf_url.is_pdf = is_pdf
pdf_url.http_status = http_status
pdf_url.last_checked = datetime.utcnow()
logger.info(u'{} updated pdf url: {}'.format(worker, pdf_url))
return pdf_url
def set_r_for_pdf(self):
self.r = None
try:
self.r = http_get(url=self.scraped_pdf_url, stream=False, publisher=self.publisher, session_id=self.session_id, ask_slowly=self.ask_slowly)
except requests.exceptions.ConnectionError as e:
self.error += u"ERROR: connection error on {} in set_r_for_pdf: {}".format(self.scraped_pdf_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.Timeout as e:
self.error += u"ERROR: timeout error on {} in set_r_for_pdf: {}".format(self.scraped_pdf_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.exceptions.InvalidSchema as e:
self.error += u"ERROR: InvalidSchema error on {} in set_r_for_pdf: {}".format(self.scraped_pdf_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.exceptions.RequestException as e:
self.error += u"ERROR: RequestException in set_r_for_pdf"
logger.info(self.error)
except requests.exceptions.ChunkedEncodingError as e:
self.error += u"ERROR: ChunkedEncodingError error on {} in set_r_for_pdf: {}".format(self.scraped_pdf_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except NoDoiException as e:
self.error += u"ERROR: NoDoiException error on {} in set_r_for_pdf: {}".format(self.scraped_pdf_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except Exception as e:
self.error += u"ERROR: Exception error in set_r_for_pdf"
logger.exception(self.error)
def gets_a_word_doc(self, link, base_url):
if is_purchase_link(link):
return False
absolute_url = get_link_target(link.href, base_url)
if DEBUG_SCRAPING:
logger.info(
u"checking to see if {} is a word doc".format(absolute_url))
start = time()
try:
r = http_get(absolute_url,
stream=True,
publisher=self.publisher,
session_id=self.session_id,
ask_slowly=self.ask_slowly)
if r.status_code != 200:
return False
if is_a_word_doc(r):
return True
except Exception as e:
logger.exception(u'error in gets_a_word_doc: {}'.format(e))
return False
def set_info_for_pmc_page(self):
if not self.pmcid:
return
url_template = u"https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={}&resulttype=core&format=json&tool=oadoi"
url = url_template.format(self.pmcid)
# try:
r = http_get(url)
data = r.json()
result_list = data["resultList"]["result"]
if not result_list:
return
result = result_list[0]
has_pdf = result.get("hasPDF", None)
is_author_manuscript = result.get("authMan", None)
is_open_access = result.get("isOpenAccess", None)
raw_license = result.get("license", None)
self.scrape_metadata_url = u"http://europepmc.org/articles/{}".format(self.pmcid)
if has_pdf == u"Y":
self.scrape_pdf_url = u"http://europepmc.org/articles/{}?pdf=render".format(self.pmcid)
if is_author_manuscript == u"Y":
self.scrape_version = u"acceptedVersion"
else:
self.scrape_version = u"publishedVersion"
if raw_license:
self.scrape_license = find_normalized_license(raw_license)
elif is_open_access == "Y":
self.scrape_license = u"implied-oa"
def set_info_for_pmc_page(self):
if not self.pmcid:
return
url_template = u"https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={}&resulttype=core&format=json&tool=oadoi"
url = url_template.format(self.pmcid)
# try:
r = http_get(url)
data = r.json()
result_list = data["resultList"]["result"]
if not result_list:
return
result = result_list[0]
has_pdf = result.get("hasPDF", None)
is_author_manuscript = result.get("authMan", None)
is_open_access = result.get("isOpenAccess", None)
raw_license = result.get("license", None)
self.scrape_metadata_url = u"http://europepmc.org/articles/{}".format(
self.pmcid)
if has_pdf == u"Y":
self.scrape_pdf_url = u"http://europepmc.org/articles/{}?pdf=render".format(
self.pmcid)
if is_author_manuscript == u"Y":
self.scrape_version = u"acceptedVersion"
else:
self.scrape_version = u"publishedVersion"
if raw_license:
self.scrape_license = find_normalized_license(raw_license)
elif is_open_access == "Y":
self.scrape_license = u"implied-oa"
def set_r_for_pdf(self):
self.r = None
try:
self.r = http_get(url=self.scraped_pdf_url, stream=False, publisher=self.publisher, session_id=self.session_id, ask_slowly=self.ask_slowly)
except requests.exceptions.ConnectionError as e:
self.error += u"ERROR: connection error on {} in set_r_for_pdf: {}".format(self.scraped_pdf_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.Timeout as e:
self.error += u"ERROR: timeout error on {} in set_r_for_pdf: {}".format(self.scraped_pdf_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.exceptions.InvalidSchema as e:
self.error += u"ERROR: InvalidSchema error on {} in set_r_for_pdf: {}".format(self.scraped_pdf_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.exceptions.RequestException as e:
self.error += u"ERROR: RequestException in set_r_for_pdf"
logger.info(self.error)
except requests.exceptions.ChunkedEncodingError as e:
self.error += u"ERROR: ChunkedEncodingError error on {} in set_r_for_pdf: {}".format(self.scraped_pdf_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except NoDoiException as e:
self.error += u"ERROR: NoDoiException error on {} in set_r_for_pdf: {}".format(self.scraped_pdf_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except Exception as e:
self.error += u"ERROR: Exception error in set_r_for_pdf"
logger.exception(self.error)
def get_pdf_url_status(pdf_url):
worker = current_process()
logger.info(u'{} checking pdf url: {}'.format(worker, pdf_url))
is_pdf = False
http_status = None
try:
response = http_get(
url=pdf_url.url, ask_slowly=True, stream=True,
publisher=pdf_url.publisher, session_id=get_session_id()
)
except Exception as e:
logger.error(u"{} failed to get response: {}".format(worker, e.message))
else:
with response:
try:
is_pdf = is_a_pdf_page(response, pdf_url.publisher)
http_status = response.status_code
except Exception as e:
logger.error(u"{} failed reading response: {}".format(worker, e.message))
pdf_url.is_pdf = is_pdf
pdf_url.http_status = http_status
pdf_url.last_checked = datetime.utcnow()
logger.info(u'{} updated pdf url: {}'.format(worker, pdf_url))
return pdf_url
def query_pmc(query_text):
if not query_text:
return None
url_template = u"https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={}&resulttype=core&format=json&tool=oadoi"
url = url_template.format(query_text)
r = http_get(url)
data = r.json()
result_list = data["resultList"]["result"]
return result_list
def gets_a_pdf(link, base_url):
if is_purchase_link(link):
return False
absolute_url = get_link_target(link, base_url)
if DEBUG_SCRAPING:
print u"checking to see if {} is a pdf".format(absolute_url)
start = time()
try:
with closing(http_get(absolute_url, stream=True, timeout=10)) as r:
if resp_is_pdf(r):
if DEBUG_SCRAPING:
print u"http header says this is a PDF. took {}s [{}]".format(
elapsed(start), absolute_url)
return True
# some publishers send a pdf back wrapped in an HTML page using frames.
# this is where we detect that, using each publisher's idiosyncratic templates.
# we only check based on a whitelist of publishers, because downloading this whole
# page (r.content) is expensive to do for everyone.
if 'onlinelibrary.wiley.com' in absolute_url:
# = closed journal http://doi.org/10.1111/ele.12585
# = open journal http://doi.org/10.1111/ele.12587 cc-by
if '<iframe' in r.content:
if DEBUG_SCRAPING:
print u"this is a Wiley 'enhanced PDF' page. took {}s [{}]".format(
elapsed(start), absolute_url)
return True
elif 'ieeexplore' in absolute_url:
# (this is a good example of one dissem.in misses)
# = open journal http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=6740844
# = closed journal http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=6045214
if '<frame' in r.content:
if DEBUG_SCRAPING:
print u"this is a IEEE 'enhanced PDF' page. took {}s [{}]".format(
elapsed(start), absolute_url)
return True
if DEBUG_SCRAPING:
print u"we've decided this ain't a PDF. took {}s [{}]".format(
elapsed(start), absolute_url)
return False
except requests.exceptions.ConnectionError:
print u"ERROR: connection error in gets_a_pdf, skipping."
return False
except requests.Timeout:
print u"ERROR: timeout error in gets_a_pdf, skipping."
return False
def gets_a_pdf(self, link, base_url):
if is_purchase_link(link):
return False
absolute_url = get_link_target(link.href, base_url)
if DEBUG_SCRAPING:
logger.info(u"checking to see if {} is a pdf".format(absolute_url))
start = time()
try:
self.r = http_get(absolute_url, stream=True, publisher=self.publisher, session_id=self.session_id, ask_slowly=self.ask_slowly)
if self.r.status_code != 200:
if self.r.status_code in [401]:
# is unauthorized, so not open
pass
else:
self.error += u"ERROR: status_code={} on {} in gets_a_pdf".format(self.r.status_code, absolute_url)
return False
if self.is_a_pdf_page():
return True
except requests.exceptions.ConnectionError as e:
self.error += u"ERROR: connection error in gets_a_pdf for {}: {}".format(absolute_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.Timeout as e:
self.error += u"ERROR: timeout error in gets_a_pdf for {}: {}".format(absolute_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.exceptions.InvalidSchema as e:
self.error += u"ERROR: InvalidSchema error in gets_a_pdf for {}: {}".format(absolute_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.exceptions.RequestException as e:
self.error += u"ERROR: RequestException error in gets_a_pdf"
logger.info(self.error)
except requests.exceptions.ChunkedEncodingError as e:
self.error += u"ERROR: ChunkedEncodingError error in gets_a_pdf for {}: {}".format(absolute_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except NoDoiException as e:
self.error += u"ERROR: NoDoiException error in gets_a_pdf for {}: {}".format(absolute_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except Exception as e:
self.error += u"ERROR: Exception error in gets_a_pdf"
logger.exception(self.error)
if DEBUG_SCRAPING:
logger.info(u"we've decided this ain't a PDF. took {} seconds [{}]".format(
elapsed(start), absolute_url))
return False
def gets_a_pdf(self, link, base_url):
if is_purchase_link(link):
return False
absolute_url = get_link_target(link.href, base_url)
if DEBUG_SCRAPING:
logger.info(u"checking to see if {} is a pdf".format(absolute_url))
start = time()
try:
self.r = http_get(absolute_url, stream=True, publisher=self.publisher, session_id=self.session_id, ask_slowly=self.ask_slowly)
if self.r.status_code != 200:
if self.r.status_code in [401]:
# is unauthorized, so not open
pass
else:
self.error += u"ERROR: status_code={} on {} in gets_a_pdf".format(self.r.status_code, absolute_url)
return False
if self.is_a_pdf_page():
return True
except requests.exceptions.ConnectionError as e:
self.error += u"ERROR: connection error in gets_a_pdf for {}: {}".format(absolute_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.Timeout as e:
self.error += u"ERROR: timeout error in gets_a_pdf for {}: {}".format(absolute_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.exceptions.InvalidSchema as e:
self.error += u"ERROR: InvalidSchema error in gets_a_pdf for {}: {}".format(absolute_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except requests.exceptions.RequestException as e:
self.error += u"ERROR: RequestException error in gets_a_pdf"
logger.info(self.error)
except requests.exceptions.ChunkedEncodingError as e:
self.error += u"ERROR: ChunkedEncodingError error in gets_a_pdf for {}: {}".format(absolute_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except NoDoiException as e:
self.error += u"ERROR: NoDoiException error in gets_a_pdf for {}: {}".format(absolute_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
except Exception as e:
self.error += u"ERROR: Exception error in gets_a_pdf"
logger.exception(self.error)
if DEBUG_SCRAPING:
logger.info(u"we've decided this ain't a PDF. took {} seconds [{}]".format(
elapsed(start), absolute_url))
return False
def set_r_for_pdf(self):
self.r = http_get(url=self.scraped_pdf_url,
stream=True,
related_pub=self.related_pub,
ask_slowly=self.ask_slowly)
def scrape_for_fulltext_link(self, find_pdf_link=True):
landing_url = self.url
if DEBUG_SCRAPING:
logger.info(
u"checking to see if {} says it is open".format(landing_url))
start = time()
try:
self.r = http_get(landing_url,
stream=True,
publisher=self.publisher,
session_id=self.session_id,
ask_slowly=self.ask_slowly)
resolved_landing_url = self.r.url
if self.r.status_code != 200:
if self.r.status_code in [401]:
# is unauthorized, so not open
pass
else:
self.error += u"ERROR: status_code={} on {} in scrape_for_fulltext_link, skipping.".format(
self.r.status_code, self.r.url)
logger.info(u"DIDN'T GET THE PAGE: {}".format(self.error))
# logger.debug(self.r.request.headers)
return
# example 10.1007/978-3-642-01445-1
if u"crossref.org/_deleted-doi/" in resolved_landing_url:
logger.info(u"this is a deleted doi")
return
# if our landing_url redirects to a pdf, we're done.
# = open repo http://hdl.handle.net/2060/20140010374
if self.is_a_pdf_page():
if DEBUG_SCRAPING:
logger.info(
u"this is a PDF. success! [{}]".format(landing_url))
self.scraped_pdf_url = landing_url
self.open_version_source_string = "open (via free pdf)"
# don't bother looking for open access lingo because it is a PDF (or PDF wannabe)
return
else:
if DEBUG_SCRAPING:
logger.info(
u"landing page is not a PDF for {}. continuing more checks"
.format(landing_url))
# get the HTML tree
page = self.r.content_small()
# remove script tags
try:
soup = BeautifulSoup(page, 'html.parser')
[script.extract() for script in soup('script')]
page = str(soup)
except HTMLParseError as e:
logger.error(
u'error parsing html, skipped script removal: {}'.format(
e))
# Look for a pdf link. If we find one, look for a license.
pdf_download_link = self.find_pdf_link(
page) if find_pdf_link else None
if pdf_download_link is not None:
pdf_url = get_link_target(pdf_download_link.href, self.r.url)
if self.gets_a_pdf(pdf_download_link, self.r.url):
self.scraped_pdf_url = pdf_url
self.scraped_open_metadata_url = landing_url
self.open_version_source_string = "open (via free pdf)"
# set the license if we can find one
scraped_license = find_normalized_license(page)
if scraped_license:
self.scraped_license = scraped_license
# Look for patterns that indicate availability but not necessarily openness and make this a bronze location.
bronze_url_snippet_patterns = [
('sciencedirect.com/',
u'<div class="OpenAccessLabel">open archive</div>'),
]
for (url_snippet, pattern) in bronze_url_snippet_patterns:
if url_snippet in resolved_landing_url.lower() and re.findall(
pattern, page, re.IGNORECASE | re.DOTALL):
self.scraped_open_metadata_url = landing_url
self.open_version_source_string = "open (via free article)"
bronze_publisher_patterns = [
("New England Journal of Medicine (NEJM/MMS)",
u'<meta content="yes" name="evt-free"'),
("Massachusetts Medical Society",
u'<meta content="yes" name="evt-free"'),
]
for (publisher, pattern) in bronze_publisher_patterns:
if self.is_same_publisher(publisher) and re.findall(
pattern, page, re.IGNORECASE | re.DOTALL):
self.scraped_open_metadata_url = landing_url
self.open_version_source_string = "open (via free article)"
# Look for some license-like patterns that make this a hybrid location.
hybrid_url_snippet_patterns = [
('projecteuclid.org/',
u'<strong>Full-text: Open access</strong>'),
('sciencedirect.com/',
u'<div class="OpenAccessLabel">open access</div>'),
('journals.ametsoc.org/',
ur'src="/templates/jsp/_style2/_ams/images/access_free\.gif"'
),
('apsjournals.apsnet.org',
ur'src="/products/aps/releasedAssets/images/open-access-icon\.png"'
),
('psychiatriapolska.pl', u'is an Open Access journal:'),
('journals.lww.com', u'<span class="[^>]*ejp-indicator--free'),
]
for (url_snippet, pattern) in hybrid_url_snippet_patterns:
if url_snippet in resolved_landing_url.lower() and re.findall(
pattern, page, re.IGNORECASE | re.DOTALL):
self.scraped_open_metadata_url = landing_url
self.open_version_source_string = "open (via page says Open Access)"
self.scraped_license = "implied-oa"
hybrid_publisher_patterns = [
("Informa UK Limited", u"/accessOA.png"),
("Oxford University Press (OUP)",
u"<i class='icon-availability_open'"),
("Institute of Electrical and Electronics Engineers (IEEE)",
ur'"isOpenAccess":true'),
("Institute of Electrical and Electronics Engineers (IEEE)",
ur'"openAccessFlag":"yes"'),
("Informa UK Limited", u"/accessOA.png"),
("Royal Society of Chemistry (RSC)", u"/open_access_blue.png"),
("Cambridge University Press (CUP)",
u'<span class="icon access open-access cursorDefault">'),
]
for (publisher, pattern) in hybrid_publisher_patterns:
if self.is_same_publisher(publisher) and re.findall(
pattern, page, re.IGNORECASE | re.DOTALL):
self.scraped_open_metadata_url = landing_url
self.open_version_source_string = "open (via page says Open Access)"
self.scraped_license = "implied-oa"
# Look for more license-like patterns that make this a hybrid location.
# Extract the specific license if present.
license_patterns = [
ur"(creativecommons.org/licenses/[a-z\-]+)",
u"distributed under the terms (.*) which permits",
u"This is an open access article under the terms (.*) which permits",
u"This is an open access article published under (.*) which permits",
u'<div class="openAccess-articleHeaderContainer(.*?)</div>'
]
for pattern in license_patterns:
matches = re.findall(pattern, page, re.IGNORECASE)
if matches:
self.scraped_open_metadata_url = landing_url
normalized_license = find_normalized_license(matches[0])
self.scraped_license = normalized_license or 'implied-oa'
if normalized_license:
self.open_version_source_string = 'open (via page says license)'
else:
self.open_version_source_string = 'open (via page says Open Access)'
if self.is_open:
if DEBUG_SCRAPING:
logger.info(
u"we've decided this is open! took {} seconds [{}]".
format(elapsed(start), landing_url))
return True
else:
if DEBUG_SCRAPING:
logger.info(
u"we've decided this doesn't say open. took {} seconds [{}]"
.format(elapsed(start), landing_url))
return False
except requests.exceptions.ConnectionError as e:
self.error += u"ERROR: connection error in scrape_for_fulltext_link on {}: {}".format(
landing_url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except requests.Timeout as e:
self.error += u"ERROR: timeout error in scrape_for_fulltext_link on {}: {}".format(
landing_url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except requests.exceptions.InvalidSchema as e:
self.error += u"ERROR: InvalidSchema error in scrape_for_fulltext_link on {}: {}".format(
landing_url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except requests.exceptions.RequestException as e:
self.error += u"ERROR: RequestException error in scrape_for_fulltext_link"
logger.info(self.error)
return False
except requests.exceptions.ChunkedEncodingError as e:
self.error += u"ERROR: ChunkedEncodingError error in scrape_for_fulltext_link on {}: {}".format(
landing_url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except NoDoiException as e:
self.error += u"ERROR: NoDoiException error in scrape_for_fulltext_link on {}: {}".format(
landing_url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except Exception as e:
self.error += u"ERROR: Exception error in scrape_for_fulltext_link"
logger.exception(self.error)
return False
def scrape_for_fulltext_link(self):
url = self.url
dont_scrape_list = [
u"ncbi.nlm.nih.gov",
u"pubmed",
u"elar.rsvpu.ru", #these ones based on complaint in email
u"elib.uraic.ru",
u"elar.usfeu.ru",
u"elar.urfu.ru",
u"elar.uspu.ru"
]
for url_fragment in dont_scrape_list:
if url_fragment in url:
logger.info(
u"not scraping {} because is on our do not scrape list.".
format(url))
return
try:
with closing(
http_get(url,
stream=True,
related_pub=self.related_pub,
ask_slowly=self.ask_slowly)) as self.r:
if self.r.status_code != 200:
self.error += u"ERROR: status_code={} on {} in scrape_for_fulltext_link".format(
self.r.status_code, url)
return
# if our url redirects to a pdf, we're done.
# = open repo http://hdl.handle.net/2060/20140010374
if self.is_a_pdf_page():
if DEBUG_SCRAPING:
logger.info(
u"this is a PDF. success! [{}]".format(url))
self.scraped_pdf_url = url
return
else:
if DEBUG_SCRAPING:
logger.info(
u"is not a PDF for {}. continuing more checks".
format(url))
# now before reading the content, bail it too large
if is_response_too_large(self.r):
logger.info(u"landing page is too large, skipping")
return
# get the HTML tree
page = self.r.content
# set the license if we can find one
scraped_license = find_normalized_license(page)
if scraped_license:
self.scraped_license = scraped_license
# special exception for citeseer because we want the pdf link where
# the copy is on the third party repo, not the cached link, if we can get it
if u"citeseerx.ist.psu.edu/" in url:
matches = re.findall(
u'<h3>Download Links</h3>.*?href="(.*?)"', page,
re.DOTALL)
if matches:
self.scraped_pdf_url = unicode(matches[0], "utf-8")
self.scraped_open_metadata_url = url
return
pdf_download_link = self.find_pdf_link(page)
if pdf_download_link is not None:
if DEBUG_SCRAPING:
logger.info(
u"found a PDF download link: {} {} [{}]".format(
pdf_download_link.href,
pdf_download_link.anchor, url))
pdf_url = get_link_target(pdf_download_link.href,
self.r.url)
# if they are linking to a PDF, we need to follow the link to make sure it's legit
if DEBUG_SCRAPING:
logger.info(
u"checking to see the PDF link actually gets a PDF [{}]"
.format(url))
if self.gets_a_pdf(pdf_download_link, self.r.url):
self.scraped_pdf_url = pdf_url
self.scraped_open_metadata_url = url
return
# try this later because would rather get a pdfs
# if they are linking to a .docx or similar, this is open.
doc_link = find_doc_download_link(page)
if doc_link is not None:
if DEBUG_SCRAPING:
logger.info(
u"found a .doc download link {} [{}]".format(
get_link_target(doc_link.href, self.r.url),
url))
self.scraped_open_metadata_url = url
return
except requests.exceptions.ConnectionError as e:
self.error += u"ERROR: connection error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.Timeout as e:
self.error += u"ERROR: timeout error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.exceptions.InvalidSchema as e:
self.error += u"ERROR: InvalidSchema error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.exceptions.RequestException as e:
self.error += u"ERROR: RequestException error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.exceptions.ChunkedEncodingError as e:
self.error += u"ERROR: ChunkedEncodingError error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except NoDoiException as e:
self.error += u"ERROR: NoDoiException error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
if DEBUG_SCRAPING:
logger.info(
u"found no PDF download link. end of the line. [{}]".format(
url))
return self
def scrape_for_fulltext_link(self):
url = self.url
dont_scrape_list = [
u"ncbi.nlm.nih.gov",
u"europepmc.org",
u"/europepmc/",
u"pubmed",
u"elar.rsvpu.ru", #these ones based on complaint in email
u"elib.uraic.ru",
u"elar.usfeu.ru",
u"elar.urfu.ru",
u"elar.uspu.ru"
]
for url_fragment in dont_scrape_list:
if url_fragment in url:
logger.info(
u"not scraping {} because is on our do not scrape list.".
format(url))
return
try:
self.r = http_get(url,
stream=True,
publisher=self.publisher,
session_id=self.session_id,
ask_slowly=self.ask_slowly)
if self.r.status_code != 200:
if self.r.status_code in [401]:
# not authorized, so not open
pass
else:
self.error += u"ERROR: status_code={} on {} in scrape_for_fulltext_link".format(
self.r.status_code, url)
return
# if our url redirects to a pdf, we're done.
# = open repo http://hdl.handle.net/2060/20140010374
if self.is_a_pdf_page():
if DEBUG_SCRAPING:
logger.info(u"this is a PDF. success! [{}]".format(url))
self.scraped_pdf_url = url
return
else:
if DEBUG_SCRAPING:
logger.info(
u"is not a PDF for {}. continuing more checks".format(
url))
# now before reading the content, bail it too large
if is_response_too_large(self.r):
logger.info(u"landing page is too large, skipping")
return
# get the HTML tree
page = self.r.content_small()
# set the license if we can find one
scraped_license = find_normalized_license(page)
if scraped_license:
self.scraped_license = scraped_license
pdf_download_link = None
# osf doesn't have their download link in their pages
# so look at the page contents to see if it is osf-hosted
# if so, compute the url. example: http://osf.io/tyhqm
if page and u"osf-cookie" in unicode(page, "utf-8"):
pdf_download_link = DuckLink(u"{}/download".format(url),
"download")
# otherwise look for it the normal way
else:
pdf_download_link = self.find_pdf_link(page)
if pdf_download_link is not None:
if DEBUG_SCRAPING:
logger.info(
u"found a PDF download link: {} {} [{}]".format(
pdf_download_link.href, pdf_download_link.anchor,
url))
pdf_url = get_link_target(pdf_download_link.href, self.r.url)
# if they are linking to a PDF, we need to follow the link to make sure it's legit
if DEBUG_SCRAPING:
logger.info(
u"checking to see the PDF link actually gets a PDF [{}]"
.format(url))
if self.gets_a_pdf(pdf_download_link, self.r.url):
self.scraped_pdf_url = pdf_url
self.scraped_open_metadata_url = url
return
# try this later because would rather get a pdfs
# if they are linking to a .docx or similar, this is open.
doc_link = find_doc_download_link(page)
if doc_link is not None:
if DEBUG_SCRAPING:
logger.info(u"found a .doc download link {} [{}]".format(
get_link_target(doc_link.href, self.r.url), url))
self.scraped_open_metadata_url = url
return
except requests.exceptions.ConnectionError as e:
self.error += u"ERROR: connection error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.Timeout as e:
self.error += u"ERROR: timeout error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.exceptions.InvalidSchema as e:
self.error += u"ERROR: InvalidSchema error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.exceptions.RequestException as e:
self.error += u"ERROR: RequestException in scrape_for_fulltext_link"
logger.info(self.error)
return
except requests.exceptions.ChunkedEncodingError as e:
self.error += u"ERROR: ChunkedEncodingError error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except NoDoiException as e:
self.error += u"ERROR: NoDoiException error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except Exception as e:
self.error += u"ERROR: Exception error on in scrape_for_fulltext_link"
logger.exception(self.error)
return
if DEBUG_SCRAPING:
logger.info(
u"found no PDF download link. end of the line. [{}]".format(
url))
return self
def scrape_green(self):
# handle these special cases, where we compute the pdf rather than looking for it
if "oai:arXiv.org" in self.pmh_id:
self.scrape_metadata_url = self.url
self.scrape_pdf_url = self.url.replace("abs", "pdf")
if self.is_pmc:
self.set_info_for_pmc_page()
if not self.scrape_pdf_url or not self.scrape_version:
with PmhRepoWebpage(url=self.url,
scraped_pdf_url=self.scrape_pdf_url,
repo_id=self.repo_id) as my_webpage:
if not self.scrape_pdf_url:
my_webpage.scrape_for_fulltext_link()
self.error += my_webpage.error
if my_webpage.is_open:
logger.info(u"** found an open copy! {}".format(
my_webpage.fulltext_url))
self.scrape_updated = datetime.datetime.utcnow(
).isoformat()
self.scrape_metadata_url = self.url
if my_webpage.scraped_pdf_url:
self.scrape_pdf_url = my_webpage.scraped_pdf_url
if my_webpage.scraped_open_metadata_url:
self.scrape_metadata_url = my_webpage.scraped_open_metadata_url
if my_webpage.scraped_license:
self.scrape_license = my_webpage.scraped_license
if self.scrape_pdf_url and not self.scrape_version:
self.set_version_and_license(r=my_webpage.r)
if self.scrape_pdf_url and not self.scrape_version:
with PmhRepoWebpage(url=self.url,
scraped_pdf_url=self.scrape_pdf_url,
repo_id=self.repo_id) as my_webpage:
my_webpage.set_r_for_pdf()
self.set_version_and_license(r=my_webpage.r)
if self.is_open and not self.scrape_version:
self.scrape_version = "submittedVersion"
# associate certain landing page URLs with PDFs
# https://repository.uantwerpen.be
if self.endpoint and self.endpoint.id == 'mmv3envg3kaaztya9tmo':
if self.scrape_pdf_url and self.scrape_pdf_url == self.scrape_metadata_url and self.pmh_record:
logger.info(u'looking for landing page for {}'.format(
self.scrape_pdf_url))
landing_urls = [
u for u in self.pmh_record.urls if u'hdl.handle.net' in u
]
if len(landing_urls) == 1:
logger.info(u'trying landing page {}'.format(
landing_urls[0]))
try:
if http_get(landing_urls[0]).status_code == 200:
self.scrape_metadata_url = landing_urls[0]
except:
pass
if self.scrape_metadata_url:
logger.info(u'set landing page {}'.format(
self.scrape_metadata_url))
def scrape_for_fulltext_link(self):
landing_url = self.url
if DEBUG_SCRAPING:
logger.info(
u"checking to see if {} says it is open".format(landing_url))
start = time()
try:
self.r = http_get(landing_url,
stream=True,
publisher=self.publisher,
session_id=self.session_id,
ask_slowly=self.ask_slowly)
if self.r.status_code != 200:
if self.r.status_code in [401]:
# is unauthorized, so not open
pass
else:
self.error += u"ERROR: status_code={} on {} in scrape_for_fulltext_link, skipping.".format(
self.r.status_code, self.r.url)
logger.info(u"DIDN'T GET THE PAGE: {}".format(self.error))
# logger.debug(self.r.request.headers)
return
# example 10.1007/978-3-642-01445-1
if u"crossref.org/_deleted-doi/" in self.r.url:
logger.info(u"this is a deleted doi")
return
# if our landing_url redirects to a pdf, we're done.
# = open repo http://hdl.handle.net/2060/20140010374
if self.is_a_pdf_page():
if DEBUG_SCRAPING:
logger.info(
u"this is a PDF. success! [{}]".format(landing_url))
self.scraped_pdf_url = landing_url
self.open_version_source_string = "open (via free pdf)"
# don't bother looking for open access lingo because it is a PDF (or PDF wannabe)
return
else:
if DEBUG_SCRAPING:
logger.info(
u"landing page is not a PDF for {}. continuing more checks"
.format(landing_url))
# get the HTML tree
page = self.r.content_small()
# set the license if we can find one
scraped_license = find_normalized_license(page)
if scraped_license:
self.scraped_license = scraped_license
pdf_download_link = self.find_pdf_link(page)
if pdf_download_link is not None:
pdf_url = get_link_target(pdf_download_link.href, self.r.url)
if self.gets_a_pdf(pdf_download_link, self.r.url):
self.scraped_pdf_url = pdf_url
self.scraped_open_metadata_url = self.url
self.open_version_source_string = "open (via free pdf)"
# now look and see if it is not just free, but open!
license_patterns = [
u"(creativecommons.org\/licenses\/[a-z\-]+)",
u"distributed under the terms (.*) which permits",
u"This is an open access article under the terms (.*) which permits",
u"This is an open access article published under (.*) which permits",
u'<div class="openAccess-articleHeaderContainer(.*?)</div>'
]
for pattern in license_patterns:
matches = re.findall(pattern, page, re.IGNORECASE)
if matches:
self.scraped_license = find_normalized_license(matches[0])
self.scraped_open_metadata_url = self.url
self.open_version_source_string = "open (via page says license)"
says_open_url_snippet_patterns = [
("projecteuclid.org/",
u'<strong>Full-text: Open access</strong>'),
]
for (url_snippet, pattern) in says_open_url_snippet_patterns:
matches = re.findall(pattern, self.r.content_small(),
re.IGNORECASE)
if url_snippet in self.r.request.url.lower() and matches:
self.scraped_open_metadata_url = self.r.request.url
self.open_version_source_string = "open (via page says Open Access)"
self.scraped_license = "implied-oa"
says_open_access_patterns = [
("Informa UK Limited", u"/accessOA.png"),
("Oxford University Press (OUP)",
u"<i class='icon-availability_open'"),
("Institute of Electrical and Electronics Engineers (IEEE)",
ur'"isOpenAccess":true'),
("Institute of Electrical and Electronics Engineers (IEEE)",
ur'"openAccessFlag":"yes"'),
("Informa UK Limited", u"/accessOA.png"),
("Royal Society of Chemistry (RSC)", u"/open_access_blue.png"),
("Cambridge University Press (CUP)",
u'<span class="icon access open-access cursorDefault">'),
]
for (publisher, pattern) in says_open_access_patterns:
matches = re.findall(pattern, page, re.IGNORECASE | re.DOTALL)
if self.is_same_publisher(publisher) and matches:
self.scraped_license = "implied-oa"
self.scraped_open_metadata_url = landing_url
self.open_version_source_string = "open (via page says Open Access)"
if self.is_open:
if DEBUG_SCRAPING:
logger.info(
u"we've decided this is open! took {} seconds [{}]".
format(elapsed(start), landing_url))
return True
else:
if DEBUG_SCRAPING:
logger.info(
u"we've decided this doesn't say open. took {} seconds [{}]"
.format(elapsed(start), landing_url))
return False
except requests.exceptions.ConnectionError as e:
self.error += u"ERROR: connection error in scrape_for_fulltext_link on {}: {}".format(
landing_url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except requests.Timeout as e:
self.error += u"ERROR: timeout error in scrape_for_fulltext_link on {}: {}".format(
landing_url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except requests.exceptions.InvalidSchema as e:
self.error += u"ERROR: InvalidSchema error in scrape_for_fulltext_link on {}: {}".format(
landing_url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except requests.exceptions.RequestException as e:
self.error += u"ERROR: RequestException error in scrape_for_fulltext_link"
logger.info(self.error)
return False
except requests.exceptions.ChunkedEncodingError as e:
self.error += u"ERROR: ChunkedEncodingError error in scrape_for_fulltext_link on {}: {}".format(
landing_url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except NoDoiException as e:
self.error += u"ERROR: NoDoiException error in scrape_for_fulltext_link on {}: {}".format(
landing_url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except Exception as e:
self.error += u"ERROR: Exception error in scrape_for_fulltext_link"
logger.exception(self.error)
return False
def scrape_for_fulltext_link(url):
if DEBUG_SCRAPING:
print u"getting URL: {}".format(url)
license = "unknown"
is_journal = is_doi_url(url) or (u"/doi/" in url)
if u"ncbi.nlm.nih.gov" in url:
print u"not scraping {} because is on our do not scrape list.".format(
url)
if "ncbi.nlm.nih.gov/pmc/articles/PMC" in url:
# pmc has fulltext
return (url, license)
else:
# is an nlm page but not a pmc page, so is not full text
return (None, license)
if DEBUG_SCRAPING:
print u"in scrape_for_fulltext_link"
with closing(http_get(url, stream=True, timeout=10)) as r:
# if our url redirects to a pdf, we're done.
# = open repo http://hdl.handle.net/2060/20140010374
if resp_is_pdf(r):
if DEBUG_SCRAPING:
print u"the head says this is a PDF. success! [{}]".format(url)
return (url, license)
else:
if DEBUG_SCRAPING:
print u"head says not a PDF. continuing more checks"
# get the HTML tree
page = r.content
license = find_normalized_license(page)
# if they are linking to a .docx or similar, this is open.
# this only works for repos... a ".doc" in a journal is not the article. example:
# = closed journal http://doi.org/10.1007/s10822-012-9571-0
if not is_journal:
doc_link = find_doc_download_link(page)
if doc_link is not None:
if DEBUG_SCRAPING:
print u"found a .doc download link {} [{}]".format(
get_link_target(doc_link, r.url), url)
return (url, license)
pdf_download_link = find_pdf_link(page, url)
if pdf_download_link is not None:
if DEBUG_SCRAPING:
print u"found a PDF download link: {} {} [{}]".format(
pdf_download_link.href, pdf_download_link.anchor, url)
pdf_url = get_link_target(pdf_download_link, r.url)
if is_journal:
# if they are linking to a PDF, we need to follow the link to make sure it's legit
if DEBUG_SCRAPING:
print u"this is a journal. checking to see the PDF link actually gets a PDF [{}]".format(
url)
if gets_a_pdf(pdf_download_link, r.url):
return (pdf_url, license)
else:
return (pdf_url, license)
if license != "unknown":
# = open 10.1136/bmj.i2716 cc-by
# = open 10.1136/bmj.i1209 cc-by-nc
# print "FOUND A LICENSE!", license, url
return (None, license)
if DEBUG_SCRAPING:
print u"found no PDF download link [{}]".format(url)
return (None, license)
def scrape_green(self):
# handle these special cases, where we compute the pdf rather than looking for it
if "oai:arXiv.org" in self.pmh_id:
self.scrape_metadata_url = self.url
self.scrape_pdf_url = self.url.replace("abs", "pdf")
if self.is_pmc:
self.set_info_for_pmc_page()
return
# https://ink.library.smu.edu.sg/do/oai/
if self.endpoint and self.endpoint.id == 'ys9xnlw27yogrfsecedx' and u'ink.library.smu.edu.sg' in self.url:
if u'viewcontent.cgi?' in self.url:
return
if self.pmh_record and find_normalized_license(self.pmh_record.license):
self.scrape_metadata_url = self.url
self.set_version_and_license()
return
if not self.scrape_pdf_url or not self.scrape_version:
with PmhRepoWebpage(url=self.url, scraped_pdf_url=self.scrape_pdf_url, repo_id=self.repo_id) as my_webpage:
if not self.scrape_pdf_url:
my_webpage.scrape_for_fulltext_link()
self.error += my_webpage.error
if my_webpage.is_open:
logger.info(u"** found an open copy! {}".format(my_webpage.fulltext_url))
self.scrape_updated = datetime.datetime.utcnow().isoformat()
self.scrape_metadata_url = self.url
if my_webpage.scraped_pdf_url:
self.scrape_pdf_url = my_webpage.scraped_pdf_url
if my_webpage.scraped_open_metadata_url:
self.scrape_metadata_url = my_webpage.scraped_open_metadata_url
if my_webpage.scraped_license:
self.scrape_license = my_webpage.scraped_license
if my_webpage.scraped_version:
self.scrape_version = my_webpage.scraped_version
if self.scrape_pdf_url and not self.scrape_version:
self.set_version_and_license(r=my_webpage.r)
if self.scrape_pdf_url and not self.scrape_version:
with PmhRepoWebpage(url=self.url, scraped_pdf_url=self.scrape_pdf_url, repo_id=self.repo_id) as my_webpage:
my_webpage.set_r_for_pdf()
self.set_version_and_license(r=my_webpage.r)
if self.is_open and not self.scrape_version:
self.scrape_version = self.default_version()
# associate certain landing page URLs with PDFs
# https://repository.uantwerpen.be
if self.endpoint and self.endpoint.id == 'mmv3envg3kaaztya9tmo':
if self.scrape_pdf_url and self.scrape_pdf_url == self.scrape_metadata_url and self.pmh_record:
logger.info(u'looking for landing page for {}'.format(self.scrape_pdf_url))
landing_urls = [u for u in self.pmh_record.urls if u'hdl.handle.net' in u]
if len(landing_urls) == 1:
logger.info(u'trying landing page {}'.format(landing_urls[0]))
try:
if http_get(landing_urls[0]).status_code == 200:
self.scrape_metadata_url = landing_urls[0]
except:
pass
if self.scrape_metadata_url:
logger.info(u'set landing page {}'.format(self.scrape_metadata_url))
# https://lirias.kuleuven.be
if (self.endpoint
and self.endpoint.id == 'ycf3gzxeiyuw3jqwjmx3'
and self.scrape_pdf_url == self.scrape_metadata_url
and self.scrape_pdf_url and 'lirias.kuleuven.be' in self.scrape_pdf_url
):
if self.pmh_record and self.pmh_record.bare_pmh_id and 'oai:lirias2repo.kuleuven.be:' in self.pmh_record.bare_pmh_id:
self.scrape_metadata_url = 'https://lirias.kuleuven.be/handle/{}'.format(
self.pmh_record.bare_pmh_id.replace('oai:lirias2repo.kuleuven.be:', '')
)
def scrape_for_fulltext_link(self):
landing_url = self.url
if DEBUG_SCRAPING:
logger.info(u"checking to see if {} says it is open".format(landing_url))
start = time()
try:
self.r = http_get(landing_url, stream=True, publisher=self.publisher, session_id=self.session_id, ask_slowly=self.ask_slowly)
resolved_landing_url = self.r.url
if self.r.status_code != 200:
if self.r.status_code in [401]:
# is unauthorized, so not open
pass
else:
self.error += u"ERROR: status_code={} on {} in scrape_for_fulltext_link, skipping.".format(self.r.status_code, self.r.url)
logger.info(u"DIDN'T GET THE PAGE: {}".format(self.error))
# logger.debug(self.r.request.headers)
return
# example 10.1007/978-3-642-01445-1
if u"crossref.org/_deleted-doi/" in self.r.url:
logger.info(u"this is a deleted doi")
return
# if our landing_url redirects to a pdf, we're done.
# = open repo http://hdl.handle.net/2060/20140010374
if self.is_a_pdf_page():
if DEBUG_SCRAPING:
logger.info(u"this is a PDF. success! [{}]".format(landing_url))
self.scraped_pdf_url = landing_url
self.open_version_source_string = "open (via free pdf)"
# don't bother looking for open access lingo because it is a PDF (or PDF wannabe)
return
else:
if DEBUG_SCRAPING:
logger.info(u"landing page is not a PDF for {}. continuing more checks".format(landing_url))
# get the HTML tree
page = self.r.content_small()
# remove script tags
try:
soup = BeautifulSoup(page, 'html.parser')
[script.extract() for script in soup('script')]
page = str(soup)
except HTMLParseError as e:
logger.error(u'error parsing html, skipped script removal: {}'.format(e))
# set the license if we can find one
scraped_license = find_normalized_license(page)
if scraped_license:
self.scraped_license = scraped_license
pdf_download_link = self.find_pdf_link(page)
if pdf_download_link is not None:
pdf_url = get_link_target(pdf_download_link.href, self.r.url)
if self.gets_a_pdf(pdf_download_link, self.r.url):
self.scraped_pdf_url = pdf_url
self.scraped_open_metadata_url = self.url
self.open_version_source_string = "open (via free pdf)"
# now look and see if it is not just free, but open!
says_open_url_snippet_patterns = [
('projecteuclid.org/', u'<strong>Full-text: Open access</strong>'),
('sciencedirect.com/', u'<div class="OpenAccessLabel">open access</div>'),
('sciencedirect.com/', u'<div class="OpenAccessLabel">open archive</div>'),
]
for (url_snippet, pattern) in says_open_url_snippet_patterns:
matches = re.findall(pattern, page, re.IGNORECASE)
if url_snippet in resolved_landing_url.lower() and matches:
self.scraped_open_metadata_url = landing_url
self.open_version_source_string = "open (via page says Open Access)"
self.scraped_license = "implied-oa"
says_open_access_patterns = [
("Informa UK Limited", u"/accessOA.png"),
("Oxford University Press (OUP)", u"<i class='icon-availability_open'"),
("Institute of Electrical and Electronics Engineers (IEEE)", ur'"isOpenAccess":true'),
("Institute of Electrical and Electronics Engineers (IEEE)", ur'"openAccessFlag":"yes"'),
("Informa UK Limited", u"/accessOA.png"),
("Royal Society of Chemistry (RSC)", u"/open_access_blue.png"),
("Cambridge University Press (CUP)", u'<span class="icon access open-access cursorDefault">'),
]
for (publisher, pattern) in says_open_access_patterns:
matches = re.findall(pattern, page, re.IGNORECASE | re.DOTALL)
if self.is_same_publisher(publisher) and matches:
self.scraped_license = "implied-oa"
self.scraped_open_metadata_url = landing_url
self.open_version_source_string = "open (via page says Open Access)"
license_patterns = [
ur"(creativecommons.org/licenses/[a-z\-]+)",
u"distributed under the terms (.*) which permits",
u"This is an open access article under the terms (.*) which permits",
u"This is an open access article published under (.*) which permits",
u'<div class="openAccess-articleHeaderContainer(.*?)</div>'
]
for pattern in license_patterns:
matches = re.findall(pattern, page, re.IGNORECASE)
if matches:
self.scraped_license = find_normalized_license(matches[0])
self.scraped_open_metadata_url = self.url
self.open_version_source_string = "open (via page says license)"
if self.is_open:
if DEBUG_SCRAPING:
logger.info(u"we've decided this is open! took {} seconds [{}]".format(
elapsed(start), landing_url))
return True
else:
if DEBUG_SCRAPING:
logger.info(u"we've decided this doesn't say open. took {} seconds [{}]".format(
elapsed(start), landing_url))
return False
except requests.exceptions.ConnectionError as e:
self.error += u"ERROR: connection error in scrape_for_fulltext_link on {}: {}".format(landing_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except requests.Timeout as e:
self.error += u"ERROR: timeout error in scrape_for_fulltext_link on {}: {}".format(landing_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except requests.exceptions.InvalidSchema as e:
self.error += u"ERROR: InvalidSchema error in scrape_for_fulltext_link on {}: {}".format(landing_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except requests.exceptions.RequestException as e:
self.error += u"ERROR: RequestException error in scrape_for_fulltext_link"
logger.info(self.error)
return False
except requests.exceptions.ChunkedEncodingError as e:
self.error += u"ERROR: ChunkedEncodingError error in scrape_for_fulltext_link on {}: {}".format(landing_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except NoDoiException as e:
self.error += u"ERROR: NoDoiException error in scrape_for_fulltext_link on {}: {}".format(landing_url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
return False
except Exception as e:
self.error += u"ERROR: Exception error in scrape_for_fulltext_link"
logger.exception(self.error)
return False
def scrape_for_fulltext_link(self, find_pdf_link=True):
url = self.url
dont_scrape_list = [
u"ncbi.nlm.nih.gov",
u"europepmc.org",
u"/europepmc/",
u"pubmed",
u"elar.rsvpu.ru", #these ones based on complaint in email
u"elib.uraic.ru",
u"elar.usfeu.ru",
u"elar.urfu.ru",
u"elar.uspu.ru"
]
for url_fragment in dont_scrape_list:
if url_fragment in url:
logger.info(
u"not scraping {} because is on our do not scrape list.".
format(url))
return
try:
self.r = http_get(url,
stream=True,
publisher=self.publisher,
session_id=self.session_id,
ask_slowly=self.ask_slowly)
resolved_url = self.r.url
if self.r.status_code != 200:
if self.r.status_code in [401]:
# not authorized, so not open
pass
else:
self.error += u"ERROR: status_code={} on {} in scrape_for_fulltext_link".format(
self.r.status_code, url)
return
# if our url redirects to a pdf, we're done.
# = open repo http://hdl.handle.net/2060/20140010374
if self.is_a_pdf_page():
if DEBUG_SCRAPING:
logger.info(u"this is a PDF. success! [{}]".format(url))
self.scraped_pdf_url = url
return
else:
if DEBUG_SCRAPING:
logger.info(
u"is not a PDF for {}. continuing more checks".format(
url))
if is_a_word_doc(self.r):
if DEBUG_SCRAPING:
logger.info(
u"this is a word doc. success! [{}]".format(url))
self.scraped_open_metadata_url = url
return
# now before reading the content, bail it too large
if is_response_too_large(self.r):
logger.info(u"landing page is too large, skipping")
return
# get the HTML tree
page = self.r.content_small()
# remove script tags
try:
soup = BeautifulSoup(page, 'html.parser')
[script.extract() for script in soup('script')]
page = str(soup)
except HTMLParseError as e:
logger.error(
u'error parsing html, skipped script removal: {}'.format(
e))
# set the license if we can find one
scraped_license = find_normalized_license(page)
if scraped_license:
self.scraped_license = scraped_license
pdf_download_link = None
# special exception for citeseer because we want the pdf link where
# the copy is on the third party repo, not the cached link, if we can get it
if url and u"citeseerx.ist.psu.edu/" in url:
matches = re.findall(u'<h3>Download Links</h3>.*?href="(.*?)"',
page, re.DOTALL)
if matches:
pdf_download_link = DuckLink(unicode(matches[0], "utf-8"),
"download")
# osf doesn't have their download link in their pages
# so look at the page contents to see if it is osf-hosted
# if so, compute the url. example: http://osf.io/tyhqm
elif page and u"osf-cookie" in unicode(
page, "utf-8", errors='replace'):
pdf_download_link = DuckLink(u"{}/download".format(url),
"download")
# otherwise look for it the normal way
else:
pdf_download_link = self.find_pdf_link(page)
if pdf_download_link is not None:
if DEBUG_SCRAPING:
logger.info(
u"found a PDF download link: {} {} [{}]".format(
pdf_download_link.href, pdf_download_link.anchor,
url))
pdf_url = get_link_target(pdf_download_link.href, self.r.url)
# if they are linking to a PDF, we need to follow the link to make sure it's legit
if DEBUG_SCRAPING:
logger.info(
u"checking to see the PDF link actually gets a PDF [{}]"
.format(url))
if self.gets_a_pdf(pdf_download_link, self.r.url):
self.scraped_pdf_url = pdf_url
self.scraped_open_metadata_url = url
return
# try this later because would rather get a pdfs
# if they are linking to a .docx or similar, this is open.
doc_link = find_doc_download_link(page)
if doc_link is not None:
absolute_doc_url = get_link_target(doc_link.href, resolved_url)
if DEBUG_SCRAPING:
logger.info(
u"found a possible .doc download link [{}]".format(
absolute_doc_url))
if self.gets_a_word_doc(doc_link, self.r.url):
if DEBUG_SCRAPING:
logger.info(
u"we've decided this is a word doc. [{}]".format(
absolute_doc_url))
self.scraped_open_metadata_url = url
return
else:
if DEBUG_SCRAPING:
logger.info(
u"we've decided this ain't a word doc. [{}]".
format(absolute_doc_url))
bhl_link = find_bhl_view_link(resolved_url, page)
if bhl_link is not None:
logger.info('found a BHL document link: {}'.format(
get_link_target(bhl_link.href, resolved_url)))
self.scraped_open_metadata_url = url
return
if _trust_repo_license(resolved_url) and self.scraped_license:
logger.info(u'trusting license {}'.format(
self.scraped_license))
self.scraped_open_metadata_url = self.url
except requests.exceptions.ConnectionError as e:
self.error += u"ERROR: connection error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.Timeout as e:
self.error += u"ERROR: timeout error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.exceptions.InvalidSchema as e:
self.error += u"ERROR: InvalidSchema error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.exceptions.RequestException as e:
self.error += u"ERROR: RequestException in scrape_for_fulltext_link"
logger.info(self.error)
return
except requests.exceptions.ChunkedEncodingError as e:
self.error += u"ERROR: ChunkedEncodingError error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except NoDoiException as e:
self.error += u"ERROR: NoDoiException error on {} in scrape_for_fulltext_link: {}".format(
url,
unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except Exception as e:
self.error += u"ERROR: Exception error on in scrape_for_fulltext_link"
logger.exception(self.error)
return
if DEBUG_SCRAPING:
logger.info(
u"found no PDF download link. end of the line. [{}]".format(
url))
return self
def scrape_for_fulltext_link(self):
url = self.url
dont_scrape_list = [
u"ncbi.nlm.nih.gov",
u"europepmc.org",
u"/europepmc/",
u"pubmed",
u"elar.rsvpu.ru", #these ones based on complaint in email
u"elib.uraic.ru",
u"elar.usfeu.ru",
u"elar.urfu.ru",
u"elar.uspu.ru"]
for url_fragment in dont_scrape_list:
if url_fragment in url:
logger.info(u"not scraping {} because is on our do not scrape list.".format(url))
return
try:
self.r = http_get(url, stream=True, publisher=self.publisher, session_id=self.session_id, ask_slowly=self.ask_slowly)
if self.r.status_code != 200:
if self.r.status_code in [401]:
# not authorized, so not open
pass
else:
self.error += u"ERROR: status_code={} on {} in scrape_for_fulltext_link".format(self.r.status_code, url)
return
# if our url redirects to a pdf, we're done.
# = open repo http://hdl.handle.net/2060/20140010374
if self.is_a_pdf_page():
if DEBUG_SCRAPING:
logger.info(u"this is a PDF. success! [{}]".format(url))
self.scraped_pdf_url = url
return
else:
if DEBUG_SCRAPING:
logger.info(u"is not a PDF for {}. continuing more checks".format(url))
# now before reading the content, bail it too large
if is_response_too_large(self.r):
logger.info(u"landing page is too large, skipping")
return
# get the HTML tree
page = self.r.content_small()
# set the license if we can find one
scraped_license = find_normalized_license(page)
if scraped_license:
self.scraped_license = scraped_license
pdf_download_link = None
# special exception for citeseer because we want the pdf link where
# the copy is on the third party repo, not the cached link, if we can get it
if url and u"citeseerx.ist.psu.edu/" in url:
matches = re.findall(u'<h3>Download Links</h3>.*?href="(.*?)"', page, re.DOTALL)
if matches:
pdf_download_link = DuckLink(unicode(matches[0], "utf-8"), "download")
# osf doesn't have their download link in their pages
# so look at the page contents to see if it is osf-hosted
# if so, compute the url. example: http://osf.io/tyhqm
elif page and u"osf-cookie" in unicode(page, "utf-8", errors='replace'):
pdf_download_link = DuckLink(u"{}/download".format(url), "download")
# otherwise look for it the normal way
else:
pdf_download_link = self.find_pdf_link(page)
if pdf_download_link is not None:
if DEBUG_SCRAPING:
logger.info(u"found a PDF download link: {} {} [{}]".format(
pdf_download_link.href, pdf_download_link.anchor, url))
pdf_url = get_link_target(pdf_download_link.href, self.r.url)
# if they are linking to a PDF, we need to follow the link to make sure it's legit
if DEBUG_SCRAPING:
logger.info(u"checking to see the PDF link actually gets a PDF [{}]".format(url))
if self.gets_a_pdf(pdf_download_link, self.r.url):
self.scraped_pdf_url = pdf_url
self.scraped_open_metadata_url = url
return
# try this later because would rather get a pdfs
# if they are linking to a .docx or similar, this is open.
doc_link = find_doc_download_link(page)
if doc_link is not None:
if DEBUG_SCRAPING:
logger.info(u"found a .doc download link {} [{}]".format(
get_link_target(doc_link.href, self.r.url), url))
self.scraped_open_metadata_url = url
return
except requests.exceptions.ConnectionError as e:
self.error += u"ERROR: connection error on {} in scrape_for_fulltext_link: {}".format(url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.Timeout as e:
self.error += u"ERROR: timeout error on {} in scrape_for_fulltext_link: {}".format(url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.exceptions.InvalidSchema as e:
self.error += u"ERROR: InvalidSchema error on {} in scrape_for_fulltext_link: {}".format(url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except requests.exceptions.RequestException as e:
self.error += u"ERROR: RequestException in scrape_for_fulltext_link"
logger.info(self.error)
return
except requests.exceptions.ChunkedEncodingError as e:
self.error += u"ERROR: ChunkedEncodingError error on {} in scrape_for_fulltext_link: {}".format(url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except NoDoiException as e:
self.error += u"ERROR: NoDoiException error on {} in scrape_for_fulltext_link: {}".format(url, unicode(e.message).encode("utf-8"))
logger.info(self.error)
return
except Exception as e:
self.error += u"ERROR: Exception error on in scrape_for_fulltext_link"
logger.exception(self.error)
return
if DEBUG_SCRAPING:
logger.info(u"found no PDF download link. end of the line. [{}]".format(url))
return self
def scrape_for_fulltext_link(self):
url = self.url
is_journal = u"/doi/" in url or u"10." in url
if DEBUG_SCRAPING:
print u"in scrape_for_fulltext_link, getting URL: {}".format(url)
if u"ncbi.nlm.nih.gov" in url:
print u"not scraping {} because is on our do not scrape list.".format(
url)
if "ncbi.nlm.nih.gov/pmc/articles/PMC" in url:
# pmc has fulltext
self.scraped_open_metadata_url = url
pmcid_matches = re.findall(".*(PMC\d+).*", url)
if pmcid_matches:
pmcid = pmcid_matches[0]
self.scraped_pdf_url = u"https://www.ncbi.nlm.nih.gov/pmc/articles/{}/pdf".format(
pmcid)
else:
# is an nlm page but not a pmc page, so is not full text
return
try:
with closing(
http_get(url, stream=True, read_timeout=10,
doi=self.doi)) as r:
if is_response_too_large(r):
print "landing page is too large, skipping"
return
# if our url redirects to a pdf, we're done.
# = open repo http://hdl.handle.net/2060/20140010374
if resp_is_pdf_from_header(r):
if DEBUG_SCRAPING:
print u"the head says this is a PDF. success! [{}]".format(
url)
self.scraped_pdf_url = url
return
else:
if DEBUG_SCRAPING:
print u"head says not a PDF for {}. continuing more checks".format(
url)
# get the HTML tree
page = r.content
# set the license if we can find one
scraped_license = find_normalized_license(page)
if scraped_license:
self.scraped_license = scraped_license
pdf_download_link = find_pdf_link(page, url)
if pdf_download_link is not None:
if DEBUG_SCRAPING:
print u"found a PDF download link: {} {} [{}]".format(
pdf_download_link.href, pdf_download_link.anchor,
url)
pdf_url = get_link_target(pdf_download_link, r.url)
if is_journal:
# if they are linking to a PDF, we need to follow the link to make sure it's legit
if DEBUG_SCRAPING:
print u"this is a journal. checking to see the PDF link actually gets a PDF [{}]".format(
url)
if gets_a_pdf(pdf_download_link, r.url, self.doi):
self.scraped_pdf_url = pdf_url
self.scraped_open_metadata_url = url
return
else:
self.scraped_pdf_url = pdf_url
self.scraped_open_metadata_url = url
return
# try this later because would rather get a pdfs
# if they are linking to a .docx or similar, this is open.
# this only works for repos... a ".doc" in a journal is not the article. example:
# = closed journal http://doi.org/10.1007/s10822-012-9571-0
if not is_journal:
doc_link = find_doc_download_link(page)
if doc_link is not None:
if DEBUG_SCRAPING:
print u"found a .doc download link {} [{}]".format(
get_link_target(doc_link, r.url), url)
self.scraped_open_metadata_url = url
return
except requests.exceptions.ConnectionError:
print u"ERROR: connection error on {} in scrape_for_fulltext_link, skipping.".format(
url)
return
except requests.Timeout:
print u"ERROR: timeout error on {} in scrape_for_fulltext_link, skipping.".format(
url)
return
except requests.exceptions.InvalidSchema:
print u"ERROR: InvalidSchema error on {} in scrape_for_fulltext_link, skipping.".format(
url)
return
except requests.exceptions.RequestException as e:
print u"ERROR: RequestException error on {} in scrape_for_fulltext_link, skipping.".format(
url)
return
if DEBUG_SCRAPING:
print u"found no PDF download link. end of the line. [{}]".format(
url)
return self
