def pull_events(options, settings):
aggregator_summary_logger.info("Starting pull_events")
talks_api = OxTalksAPI(settings.oxtalks_hostname, settings.oxtalks_username,
settings.oxtalks_password)
talks, list_manager = talks_api.load_talks()
sources = load_sources(settings.sources_filename)
single_trawler_to_run = getattr(options, "trawler", None)
if single_trawler_to_run is not None:
sources = filter(lambda source: source.name == single_trawler_to_run, sources)
if len(sources) == 0:
raise ValueError("Could not find trawler named %s, use --list_trawlers to see all those in the system" % single_trawler_to_run)
all_instructions = []
failed_trawler = None
# Make use of threadpoolexecutor to run all sources on different threads.
with ThreadPoolExecutor(max_workers=10) as sources_executor:
source_results = []
for source in sources:
future = sources_executor.submit(_load_talks_from_source, source, list_manager)
source_results.append((source, future))
for source, future in source_results:
new_instructions, succeeded = future.result()
all_instructions.extend(new_instructions)
if not succeeded:
failed_trawler = source
# Remove dull events
all_instructions = filter(lambda instruction: not isinstance(instruction, AddEvent) or is_worthy_talk(instruction.event),
all_instructions)
if single_trawler_to_run is None:
if failed_trawler is None:
all_instructions.append(DeleteOutstanding())
else:
logger.error("Since %s failed, we will not flush out stale "
"events, as we can't be sure it's not a temporary "
"failure" % failed_trawler.name)
if options.dry_run:
talks_api.print_dry_run_output(all_instructions)
else:
try:
talks_api.upload(all_instructions)
except:
log_exception(logger, "Failed to upload events to OxTalks")
def list_trawlers(options, settings):
for source in load_sources(settings.sources_filename):
print source.name
description = getattr(source, "description", None)
print "\t%s" % (description if description is not None else "No description available")
print
