def scrape_unit(self, unit: NewsdeskUnit):
article_element, article = unit
try:
article_element.find_element_by_css_selector(".extract_full_link_button").click()
except NoSuchElementException:
pass
else:
self.wait(".extract_full_link_button", on=article_element)
try:
inner = self.wait(".extract_inner", on=article_element)
except NotVisible:
if article.get_property("wordcount_int") <= 1:
article.text = "[NO TEXT]"
else:
raise
else:
article_html = inner.get_attribute("innerHTML")
article.text = html2text(article_html)
# Cut off data urls at #3; no article actually has that many. All instances so far led to the same
# data (even though the url differed).
data_urls = list(get_data_urls(inner))[:3]
for i, data_url in enumerate(data_urls):
article.set_property("data{}_url".format(i), data_url)
# Be gentle with servers
time.sleep(random.uniform(0.1, 0.3))
return article
def scrape_unit(self, url):
reader_url = "about:reader?url={}".format(url)
doc = self.get_html(reader_url, wait_for="div.content p")
for tag in REMOVE_TAGS:
for element in doc.cssselect(tag):
element.getparent().remove(element)
article = doc.cssselect("div.content")[0]
article_html = lxml.html.tostring(article).decode()
title = doc.cssselect("h1.reader-title")[0].text_content().strip()
text = html2text(article_html)
if self.__class__.get_date is not GenericScraper.get_date:
# Get contents of un-firefox-read-ed article
self.wait(".reader-toolbar .close-button").click()
time.sleep(0.3)
doc_html = self.wait("html").get_attribute("outerHTML")
doc = lxml.html.fromstring(doc_html, base_url=url)
try:
date = self.get_date(doc)
except NotImplementedError:
date = self.now
except Exception as e:
log.warning("get_date() failed for {} with: {}".format(url, e))
date = self.now
else:
date = self.now
article = Article(date=date, title=title, text=text, url=url)
return article
def scrape_unit(self, date_and_article_url):
date, article_url = date_and_article_url
log.info("Fetching {}".format(article_url))
article_doc = self.session.get_html(article_url)
article_el = article_doc.cssselect("#content > article")
if not article_el:
log.error("Could not find article on {article_url}".format(**locals()))
return None
title = article_el[0].cssselect("h1")[0].text
text = html2text(article_el[0].cssselect("p"))
text = text.strip() or "."
try:
footer = article_el[0].cssselect("footer")[0]
except IndexError as e:
# Contains <embed> tag which is not closed gracefully :-(
log.exception(e)
return None
author = footer.text.rsplit("|", 1)[0].strip()
timestamp = parse_date(article_el[0].cssselect("footer > time")[0].get("datetime"))
if not title:
return None
children = self._get_comments(title, article_url, article_doc)
article = Article(date=timestamp, title=title, text=text)
article.set_property("author", author)
article.set_property("url", article_url)
article.set_property("medium", "GeenStijl")
return ArticleTree(article, [ArticleTree(c, []) for c in children])
def scrape_unit(self, url):
reader_url = "about:reader?url={}".format(url)
doc = self.get_html(reader_url, wait_for="div.content p")
for tag in REMOVE_TAGS:
for element in doc.cssselect(tag):
element.getparent().remove(element)
article = doc.cssselect("div.content")[0]
article_html = lxml.html.tostring(article).decode()
title = doc.cssselect("h1.reader-title")[0].text_content().strip()
text = html2text(article_html)
if self.__class__.get_date is not GenericScraper.get_date:
# Get contents of un-firefox-read-ed article
self.wait(".reader-toolbar .close-button").click()
time.sleep(0.3)
doc_html = self.wait("html").get_attribute("outerHTML")
doc = lxml.html.fromstring(doc_html, base_url=url)
try:
date = self.get_date(doc)
except NotImplementedError:
date = self.now
except Exception as e:
log.warning("get_date() failed for {} with: {}".format(url, e))
date = self.now
else:
date = self.now
article = Article(date=date, title=title, text=text, url=url)
return article
def _scrape_unit(self, article):
doc = self.session.get_html(article['url'])
article['headline'] = doc.cssselect("div.intro h1")[0].text
article['text'] = html2text(
doc.cssselect("div.intro h2") + doc.cssselect("div.text"))
if doc.cssselect("span.author"):
article['author'] = doc.cssselect("span.author"
)[0].text.split("Door:")[1]
def _scrape_unit(self, url):
doc = self.session.get_html(url)
article = {
'url' : doc.base_url,
'text' : tools.html2text(doc.cssselect("#article-content p")),
'headline' : doc.cssselect("#article h1")[0].text_content().strip(),
'date' : tools.read_date(doc.cssselect("abbr.page-last-modified")[0].text),
'externalid' : int(url.split("/")[-1].split("-")[0])}
return article
def get_article_section_text(self, url):
article_doc = self.session.get_html(url)
print(url)
yield article_doc.cssselect("#main")[0].text_content().strip()
#yield article_doc.cssselect(".breadcrumb li.active")[0].text_content().strip()
#text = html2text(article_doc.cssselect(".article.body")[0]).strip()
text = html2text(
article_doc.cssselect("div.block.article-body")[0]).strip()
print(f"{text}")
# Bylines are sometimes included in the main body text. We won't put in
# any effort to pluck out those cases though..
byline = article_doc.cssselect(".article.header .item-excerpt")
if byline:
text += "\n\n"
text += html2text(byline[0])
yield text
def scrape_unit(self, article_info: ArticleTuple):
date, page_num, url = article_info
try:
text_url = strip_query(self.session.get_redirected_url(url))
except RedirectError as e:
if e.status_code == 404:
return None
raise
text_doc = self.session.get_html(text_url)
for image in text_doc.cssselect(".image"):
image.getparent().remove(image)
date = datetime.datetime(date.year, date.month, date.day)
try:
title = text_doc.cssselect("article > h1")[0].text.strip()
except:
return None
text = html2text(text_doc.cssselect("main > article > .body"))
if not text.strip():
return None
article = Article(title=title, date=date, text=text, url=url)
if text_doc.cssselect("article > header.themed"):
# New headers style
author = text_doc.cssselect("article > header .author")[0].text
section = text_doc.cssselect("article > header .title")[0].text
article.set_property("author", author)
else:
# Old header style
section = text_doc.cssselect("article > header > .title")
section = section[0].text if section else "NOSECTION"
author_a = text_doc.cssselect("article .author a")
if author_a:
author = author_a[0].text.strip()
article.set_property("author", author)
if author == section:
section = "Opinie"
download = text_doc.cssselect('form[name="download"]')
if download:
pdf_url = download[0].get("action")
article.set_property("pdf_url", pdf_url)
article.set_property("text_url", text_url)
article.set_property("image_url", text_url + "?view=img")
if section:
article.set_property("section", section.strip())
return article
def _parse_comment(self, comment, base_title, base_url):
text = html2text(comment.cssselect("p"))
article_id = comment.get("id")
title = "{base_title}#{article_id}".format(**locals())
url = "{base_url}#{article_id}".format(**locals())
author, timestamp = _parse_comment_footer(comment.cssselect("footer")[0].text_content())
article = Article(date=timestamp, title=title, text=text.strip() or ".", url=url)
article.set_property("author", author.strip())
article.set_property("medium", "GeenStijl Comments")
return article
def _get_deduplicate_units(self, date, edition=None):
self.browser.get("https://digitalpublishing.telegraaf.nl/static/krant/")
found = False
for day_container in self.browser.find_elements_by_css_selector(".Day__date-container"):
paper_date_string = " ".join(day_container.text.split()[1:3] + ["2018"])
paper_date = dutch_strptime(paper_date_string, "%d %B %Y").date()
print(paper_date_string, paper_date, date, paper_date==date)
if date == paper_date:
self.wait(".Day__button", on=day_container).click()
found = True
break
if found:
self.wait("#next-page-button")
while self.next_button().is_displayed():
for article in self.browser.find_elements_by_css_selector(".pages-swiper-slide-active .article-layer"):
self.click(article)
time.sleep(1.5)
try:
self.browser.switch_to_frame(self.wait("iframe.article-contents", timeout=10))
except NotVisible:
print("Warning: article skipped because frame was not visible")
continue
article_html = self.wait("body").get_property("outerHTML")
text = html2text(article_html)
url = self.browser.current_url
(scheme, netloc, path, params, query, fragment) = parse.urlparse(url)
query += "&hash=" + hashlib.sha256(article_html.encode()).hexdigest()[:20]
url = parse.urlunparse((scheme, netloc, path, params, query, fragment))
page_range = fragment.split("/")[-1]
try:
title = self.wait("body > .head", timeout=2).text.strip()
except:
continue
else:
yield TelegraafUnit(url, date, title, text, page_range)
finally:
self.browser.switch_to_default_content()
# Close modal
self.wait(".article-modal-default-button").click()
time.sleep(0.5)
self.next_button().click()
time.sleep(0.5)
def _scrape_unit(self, url):
doc = self.session.get_html(url)
date_txt = doc.cssselect("div.dateplace-data")[0].text
date = read_date(date_txt)
if not date.date() in self.dates:
return
hours, minutes = map(int,date_txt.split()[-1].split(":"))
date += timedelta(hours = hours, minutes = minutes)
content = doc.cssselect("#leadarticle")[0]
article = {
'text' : html2text(content.cssselect("div.content")),
'headline' : content.cssselect("h1")[0].text,
'section' : url.split("/")[3],
'date' : date,
'url' : url,
}
if content.cssselect("span.smallprint"):
article['author'] = content.cssselect("span.smallprint"
)[-1].text.split(":")[1].strip()
return article
def scrape_unit(self, article_info: ArticleTuple):
date, page_num, url = article_info
try:
text_url = strip_query(self.session.get_redirected_url(url))
except RedirectError as e:
if e.status_code == 404:
return None
raise
try:
text_doc = self.session.get_html(text_url)
except HTTPError as e:
if e.response.status_code == 404:
logging.warning(f"{url} returned 404 skipping")
return None
else:
raise
for image in text_doc.cssselect(".image"):
image.getparent().remove(image)
date = datetime.datetime(date.year, date.month, date.day)
try:
title = text_doc.cssselect("article > h1")[0].text.strip()
except:
return None
text = html2text(text_doc.cssselect("main > article > .body"))
if not text.strip():
return None
article = Article(title=title, date=date, text=text, url=url)
if text_doc.cssselect("article > header.themed"):
# New headers style
author = text_doc.cssselect("article > header .author")[0].text
section = text_doc.cssselect("article > header .title")[0].text
article.set_property("author", author)
else:
# Old header style
section = text_doc.cssselect("article > header > .title")
section = section[0].text if section else "NOSECTION"
author_a = text_doc.cssselect("article .author a")
if author_a:
author = author_a[0].text.strip()
article.set_property("author", author)
if author == section:
section = "Opinie"
download = text_doc.cssselect('form[name="download"]')
if download:
pdf_url = download[0].get("action")
article.set_property("pdf_url", pdf_url)
article.set_property("text_url", text_url)
article.set_property("image_url", text_url + "?view=img")
if section:
article.set_property("section", section.strip())
return article
def _get_deduplicate_units(self, date: datetime.datetime, edition=None):
# Select edition
logging.info(f"Selecting date {date}")
self.accept_cookie(timeout=1)
self.browser.get(self.login_url)
if edition is not None:
self.click(
self.wait('//div[text() = "{}"]'.format(edition), by=By.XPATH))
# accept cookies
self.accept_cookie2()
# Go to archive and select paper of this date
self.wait("paper-button.showMoreButton").click()
# make sure right header is not hidden
header = self.wait('#rightHeader', visible=False)
self.browser.execute_script('arguments[0].removeAttribute("hidden");',
header)
# click "Archief" button
self.wait('archive-calendar-button').click()
# find correct year
while True:
picked_year = int(
self.wait("#yearSelection > paper-slider").get_attribute(
"value"))
#picked_year = int(self.wait('input.style-scope.vl-date-picker').text)
if picked_year == date.year:
break
year_buttons = self.browser.find_elements_by_css_selector(
"#yearSelection iron-icon")
if picked_year > date.year:
year_buttons[0].click()
elif len(year_buttons) > 1:
year_buttons[1].click()
else:
raise Exception(
f"Only one year button, but {picked_year} < {date.year}")
# find correct month
self.wait("#monthSelection").find_element_by_xpath(
f"//paper-button[@data-month={date.month-1}]").click()
# find correct day -- wait 3 seconds to give date picker time to load
time.sleep(3)
day_button = self.wait("#daySelection").find_element_by_xpath(
f"//paper-button[@data-current and @data-day={date.day}]")
logging.info(
f'{date}: aria-disabled: {day_button.get_attribute("aria-disabled")}, active: {day_button.get_attribute("active")}'
)
if day_button.get_attribute("aria-disabled") == "true":
logging.warning(f"No newspaper for {date}, Sunday?")
return
if day_button.get_attribute("active") == "false":
# Don't click if correct day was already selected
day_button.click()
self.wait("#selectButton").click()
for archive_issue in self.browser.find_elements_by_css_selector(
"archive-issue"):
try:
archive_date = archive_issue.find_element_by_css_selector(
".issueDate").text.strip()
except NoSuchElementException:
continue
if not archive_date:
continue
if dutch_strptime(archive_date, "%d %B %Y").date() == date:
archive_issue.click()
break
else:
logging.warning(f"Could not find date {date}")
return
# Scrape unit
self.browser.switch_to_frame(self.wait("iframe#issue"))
seconds_forgone = 0
start = datetime.datetime.now()
while seconds_forgone < 30:
seconds_forgone = (datetime.datetime.now() - start).total_seconds()
try:
self.wait("#articleMenuItem", timeout=10).click()
except ElementClickInterceptedException:
pass
else:
break
article_list_buttons = self.browser.find_elements_by_css_selector(
"#articleListSectionsButtons > button")
article_list_buttons = list(article_list_buttons) or [lambda: None]
for article_list_button in article_list_buttons:
if not "selected" in article_list_button.get_attribute("class"):
article_list_button.click()
articles = list(
self.browser.find_elements_by_css_selector(".articleListItem"))
for article in articles:
page = int(article.get_attribute("data-page"))
refid = article.get_attribute("data-refid")
url = urljoin(self.browser.current_url + "/", refid)
def collect_headers(els):
for el in els:
el_text = el.get_property("textContent").strip()
if el_text:
yield (el, el_text)
h1s = list(
collect_headers(
article.find_elements_by_css_selector(
".articleListItem > h1")))
h2s = list(
collect_headers(
article.find_elements_by_css_selector(
".articleListItem > h2")))
h3s = list(
collect_headers(
article.find_elements_by_css_selector(
".articleListItem > h3")))
if h1s:
_, title = h1s.pop(0)
elif h2s:
_, title = h2s.pop(0)
else:
_, title = h3s.pop(0)
try:
content = article.find_element_by_css_selector(
"div.content").get_property("outerHTML")
except NoSuchElementException:
continue
subtitles = [
element.get_property("outerHTML")
for element, _ in h1s + h2s + h3s
]
article_html = "".join(subtitles) + content
text = html2text(article_html)
#try:
# author = article.find_element_by_css_selector(".byline").get_property("textContent").strip()
#except NoSuchElementException:
# pass
#else:
# print(author)
# Screenshot code:
# article.click()
# self.browser.switch_to_frame(self.wait("#articleViewContent > iframe"))
# screenshot = self.wait("#page article").screenshot_as_base64
# self.browser.switch_to_default_content()
# self.browser.switch_to_frame(self.wait("#issue"))
# self.wait("#articleNavigationBack").click()
# time.sleep(0.5)
screenshot = None
yield EPagesUnit(url, date, title, page, screenshot, text)
def _get_deduplicate_units(self, date, edition=None):
self.browser.get(
"https://digitalpublishing.telegraaf.nl/static/krant/")
found = False
for day_container in self.browser.find_elements_by_css_selector(
".Day__date-container"):
paper_date_string = " ".join(day_container.text.split()[1:3] +
[str(date.year)])
paper_date = dutch_strptime(paper_date_string, "%d %B %Y").date()
print(
f"datum krant {paper_date_string}, {paper_date}, {date}, {paper_date==date}"
)
if date == paper_date:
self.wait(".Day__button", on=day_container).click()
found = True
break
if found:
self.wait("#next-page-button")
while self.next_button().is_displayed():
try:
articles = self.wait_multiple(
".pages-swiper-slide-active .article-layer")
except NoSuchElementException:
logging.warning(f"Could not find article layer in {url}")
articles = []
for article in articles:
self.click(article)
time.sleep(1.5)
try:
self.browser.switch_to_frame(
self.wait("iframe.article-contents", timeout=10))
except NotVisible:
print(
"Warning: article skipped because frame was not visible"
)
continue
article_html = self.wait("body").get_property("outerHTML")
text = html2text(article_html)
url = self.browser.current_url
(scheme, netloc, path, params, query,
fragment) = parse.urlparse(url)
query += "&hash=" + hashlib.sha256(
article_html.encode()).hexdigest()[:20]
url = parse.urlunparse(
(scheme, netloc, path, params, query, fragment))
page_range = fragment.split("/")[-1]
try:
title = self.wait("body > .head",
timeout=2).text.strip()
except NoSuchElementException:
try:
title = self.wait("body > .head1",
timeout=2).text.strip()
except NoSuchElementException:
logging.warning(f"No title found: {url}")
title = "-"
if not title:
logging.warning(f"Empty title for {url}")
title = "-"
yield TelegraafUnit(url, date, title, text, page_range)
self.browser.switch_to_default_content()
# Close modal
self.wait(".article-modal-default-button").click()
time.sleep(0.5)
self.next_button().click()
time.sleep(0.5)
def _get_deduplicate_units(self, date, edition=None):
# Select edition
self.browser.get(self.login_url)
if edition is not None:
self.click(self.wait('//div[text() = "{}"]'.format(edition), by=By.XPATH))
# Go to archive and select paper of this date
self.wait("paper-button.showMoreButton").click()
for archive_issue in self.browser.find_elements_by_css_selector("archive-issue"):
try:
archive_date = archive_issue.find_element_by_css_selector(".issueDate").text.strip()
except NoSuchElementException:
continue
if not archive_date:
continue
if dutch_strptime(archive_date, "%d %B %Y").date() == date:
archive_issue.click()
break
else:
return
# Scrape unit
self.browser.switch_to_frame(self.wait("#issue"))
seconds_forgone = 0
start = datetime.datetime.now()
while seconds_forgone < 30:
seconds_forgone = (datetime.datetime.now() - start).total_seconds()
try:
self.wait("#articleMenuItem", timeout=60).click()
except ElementClickInterceptedException:
pass
else:
break
article_list_buttons = self.browser.find_elements_by_css_selector("#articleListSectionsButtons > button")
article_list_buttons = list(article_list_buttons) or [lambda: None]
time.sleep(2)
for article_list_button in article_list_buttons:
article_list_button.click()
time.sleep(2)
articles = list(self.browser.find_elements_by_css_selector(".articleListItem"))
for article in articles:
page = int(article.get_attribute("data-page"))
refid = article.get_attribute("data-refid")
url = urljoin(self.browser.current_url + "/", refid)
def collect_headers(els):
for el in els:
el_text = el.get_property("textContent").strip()
if el_text:
yield (el, el_text)
h1s = list(collect_headers(article.find_elements_by_css_selector(".articleListItem > h1")))
h2s = list(collect_headers(article.find_elements_by_css_selector(".articleListItem > h2")))
h3s = list(collect_headers(article.find_elements_by_css_selector(".articleListItem > h3")))
if h1s:
_, title = h1s.pop(0)
elif h2s:
_, title = h2s.pop(0)
else:
_, title = h3s.pop(0)
try:
content = article.find_element_by_css_selector("div.content").get_property("outerHTML")
except NoSuchElementException:
continue
subtitles = [element.get_property("outerHTML") for element, _ in h1s + h2s + h3s]
article_html = "".join(subtitles) + content
text = html2text(article_html)
#try:
# author = article.find_element_by_css_selector(".byline").get_property("textContent").strip()
#except NoSuchElementException:
# pass
#else:
# print(author)
# Screenshot code:
# article.click()
# self.browser.switch_to_frame(self.wait("#articleViewContent > iframe"))
# screenshot = self.wait("#page article").screenshot_as_base64
# self.browser.switch_to_default_content()
# self.browser.switch_to_frame(self.wait("#issue"))
# self.wait("#articleNavigationBack").click()
# time.sleep(0.5)
screenshot = None
yield EPagesUnit(url, date, title, page, screenshot, text)