def __call__(self, list_manager):
soup = load_soup(url_opener(_main_page_url))
master_list = list_manager.get_or_create_managed_list_by_name("Daily Info")
for ical_link in soup.xpath("//@href[contains(., 'generate_ics')]"):
ical_uri = absolute_url(_main_page_url, ical_link)
for event in load_ical(url_opener(ical_uri),
master_list=master_list,
url_for_logging=_main_page_url):
yield event
def __call__(self, list_manager):
with ThreadPoolExecutor(max_workers=8) as executor:
waiting_futures = []
master_list = list_manager.get_or_create_managed_list_by_name("Oxford University What's On")
soup = load_soup(url_opener(url))
for event_element in soup.xpath("//div[@id='primary-content']//ul/li//div[@class='content']"):
event_page_path, = event_element.xpath(".//@href[contains(., 'go.rm')]")
series, = event_element.xpath("p[1]")
series = list_manager.get_or_create_managed_list_by_name(render_html_text(series))
lists = [master_list, series]
event_page_url = absolute_url(url, event_page_path)
waiting_futures.append(executor.submit(load_events, event_page_url, master_list, lists))
for finished_future in as_completed(waiting_futures):
for talk in finished_future.result():
yield talk
def load_events(event_page_url, master_list, lists):
try:
event_page_soup = load_soup(url_opener(event_page_url))
description_nodes = event_page_soup.xpath("//div[@class='description']")
if len(description_nodes) > 0:
description_node = description_nodes[0]
description = render_html_text(description_node)
else:
description = ""
description = "%s\nLoaded from \"%s\":%s" % (description, event_page_url, event_page_url)
ical_path, = event_page_soup.xpath("//a/@href[contains(., 'add_event_to_calendar')]")
ical_link = absolute_url(event_page_url, ical_path)
results = []
for event in load_ical(url_opener(ical_link), raw_hacks=[munge_character],
master_list=master_list, lists=lists):
results.append(event.clone_with_modifications(description=description,
start=munge_dst(event.start),
end=munge_dst(event.end)))
return results
except Exception:
log_exception_via(logger.warning, "Failed loading event from %s" % event_page_url)
return []
