def __call__(self, list_manager):
soup = load_soup(url_opener(_main_page_url))
master_list = list_manager.get_or_create_managed_list_by_name("Daily Info")
for ical_link in soup.xpath("//@href[contains(., 'generate_ics')]"):
ical_uri = absolute_url(_main_page_url, ical_link)
for event in load_ical(url_opener(ical_uri),
master_list=master_list,
url_for_logging=_main_page_url):
yield event
def __call__(self, list_manager):
oxitems_list = list_manager.get_or_create_managed_list_by_name("OxItems")
soup = load_soup(url_opener(OxItems.url))
all_events = []
for oxitems_feed_option in soup.xpath("//option"):
try:
oxitems_feed_id, = oxitems_feed_option.xpath("@value")
if oxitems_feed_id != "_UNSET_":
oxitems_feed_longname, = oxitems_feed_option.xpath("text()")
oxitems_feed_longname = oxitems_feed_longname[len(oxitems_feed_id) + 2:]
master_list = list_manager.get_or_create_managed_list_by_name(oxitems_feed_longname)
ical_url = OxItems.feed_url % oxitems_feed_id
new_events = list(load_ical(url_opener(ical_url), url_for_logging=ical_url,
lists=[oxitems_list, master_list],
master_list=list_manager.get_or_create_managed_list_by_name(oxitems_feed_longname)))
logger.debug("OxItems: loaded %i talks from %s" % (len(new_events), ical_url))
all_events.extend(new_events)
except Exception:
log_exception_via(logger.warning, "OxItems failed on %s (%s)" % (oxitems_feed_id, ical_url))
return all_events
def load_events(event_page_url, master_list, lists):
try:
event_page_soup = load_soup(url_opener(event_page_url))
description_nodes = event_page_soup.xpath("//div[@class='description']")
if len(description_nodes) > 0:
description_node = description_nodes[0]
description = render_html_text(description_node)
else:
description = ""
description = "%s\nLoaded from \"%s\":%s" % (description, event_page_url, event_page_url)
ical_path, = event_page_soup.xpath("//a/@href[contains(., 'add_event_to_calendar')]")
ical_link = absolute_url(event_page_url, ical_path)
results = []
for event in load_ical(url_opener(ical_link), raw_hacks=[munge_character],
master_list=master_list, lists=lists):
results.append(event.clone_with_modifications(description=description,
start=munge_dst(event.start),
end=munge_dst(event.end)))
return results
except Exception:
log_exception_via(logger.warning, "Failed loading event from %s" % event_page_url)
return []
def __call__(self, list_manager):
with ThreadPoolExecutor(max_workers=8) as executor:
waiting_futures = []
master_list = list_manager.get_or_create_managed_list_by_name("Oxford University What's On")
soup = load_soup(url_opener(url))
for event_element in soup.xpath("//div[@id='primary-content']//ul/li//div[@class='content']"):
event_page_path, = event_element.xpath(".//@href[contains(., 'go.rm')]")
series, = event_element.xpath("p[1]")
series = list_manager.get_or_create_managed_list_by_name(render_html_text(series))
lists = [master_list, series]
event_page_url = absolute_url(url, event_page_path)
waiting_futures.append(executor.submit(load_events, event_page_url, master_list, lists))
for finished_future in as_completed(waiting_futures):
for talk in finished_future.result():
yield talk
def create(cls, url=None, name=None, raw_hacks=[], description=None):
return ICalEventSource(url_opener(url), url, name, name, description,
[find_thing(raw_hack_name, sys.modules[__name__]) for raw_hack_name in raw_hacks])