def main(): pubsub = get_pubsub(pageS_PING) for notification in pubsub.listen(): domains = pop_n_items(DOMAINS_MOST_RECENT, DOMAINS_MAX_PER_CHANNEL) print domains # TODO: split domains among threads/processes & launch pages_worker(domains) ping(ACCELERANDS_PING)
def domains_worker(): domains = get_available_domains() write_items_to_datastore(DOMAINS_MOST_RECENT, domains) ping(pageS_PING)