def map_each_attendee(db_event):
fbl = fb_mapreduce.get_fblookup()
fbl.request(fb_api.LookupEventAttending, db_event.fb_event_id)
fbl.batch_fetch()
fb_event_attending = fbl.fetched_data(fb_api.LookupEventAttending, db_event.fb_event_id)
for attendee in fb_event_attending['attending']['data']:
yield (db_event.city_name, attendee['id'])
def scrape_sources_for_events(sources):
fbl = fb_mapreduce.get_fblookup()
fbl.allow_cache = False
discovered_list = thing_scraper.discover_events_from_sources(fbl, sources)
for x in discovered_list:
state = (x.event_id, x.source_id, x.source_field, x.extra_source_id)
mr.increment('found-event-to-check')
yield (_shard_for(x.event_id), json.dumps(state))
def map_each_attendee(db_event):
fbl = fb_mapreduce.get_fblookup()
fbl.request(fb_api.LookupEventAttending, db_event.fb_event_id)
fbl.batch_fetch()
fb_event_attending = fbl.fetched_data(fb_api.LookupEventAttending,
db_event.fb_event_id)
for attendee in fb_event_attending['attending']['data']:
yield (db_event.city_name, attendee['id'])
def process_events(event_id, via_sources):
fbl = fb_mapreduce.get_fblookup()
discovered_list = []
for data in via_sources:
source_id, source_field, extra_source_id = json.loads(data)
discovered = potential_events.DiscoveredEvent(event_id, None, source_field, extra_source_id)
discovered.source = None # TODO: This will come back to bite us I'm sure :(
discovered.source_id = source_id
discovered_list.append(discovered)
event_pipeline.process_discovered_events(fbl, discovered_list)
def map_events_needing_access_tokens(all_db_events):
fbl = fb_mapreduce.get_fblookup()
db_events = [x for x in all_db_events if x.is_fb_event and not x.visible_to_fb_uids]
try:
fb_events = fbl.get_multi(fb_api.LookupEvent, [x.fb_event_id for x in db_events])
except fb_api.ExpiredOAuthToken:
# Not longer a valid source for access_tokens
logging.exception("ExpiredOAuthToken")
for db_event, fb_event in zip(db_events, fb_events):
if fb_event['empty'] == fb_api.EMPTY_CAUSE_INSUFFICIENT_PERMISSIONS:
yield '%s\n' % db_event.fb_event_id
def map_events_needing_access_tokens(all_db_events):
fbl = fb_mapreduce.get_fblookup()
db_events = [x for x in all_db_events if x.is_fb_event and not x.visible_to_fb_uids]
try:
fb_events = fbl.get_multi(fb_api.LookupEvent, [x.fb_event_id for x in db_events])
except fb_api.ExpiredOAuthToken:
# Not longer a valid source for access_tokens
logging.exception("ExpiredOAuthToken")
for db_event, fb_event in zip(db_events, fb_events):
if fb_event['empty'] == fb_api.EMPTY_CAUSE_INSUFFICIENT_PERMISSIONS:
yield '%s\n' % db_event.fb_event_id
def process_events(event_id, via_sources):
fbl = fb_mapreduce.get_fblookup()
discovered_list = []
for data in via_sources:
source_id, source_field, extra_source_id = json.loads(data)
discovered = potential_events.DiscoveredEvent(event_id, None,
source_field,
extra_source_id)
discovered.source = None # TODO: This will come back to bite us I'm sure :(
discovered.source_id = source_id
discovered_list.append(discovered)
event_pipeline.process_discovered_events(fbl, discovered_list)
def process_events(shard_id, via_sources):
fbl = fb_mapreduce.get_fblookup()
fbl.allow_cache = True
discovered_list = []
logging.info('Running process_events with %s event-sources',
len(via_sources))
for data in via_sources:
event_id, source_id, source_field, extra_source_id = json.loads(data)
discovered = potential_events.DiscoveredEvent(event_id, None,
source_field,
extra_source_id)
discovered.source = None # TODO: This will come back to bite us I'm sure :(
discovered.source_id = source_id
discovered_list.append(discovered)
# Some of these are newly-discovered events, some of these are already-cached and classified.
# TODO: Filter out the already-classified ones, so we don't waste time re-classifying on cached on data.
event_pipeline.process_discovered_events(fbl, discovered_list)
def map_load_potential_events(user):
fbl = fb_mapreduce.get_fblookup(user)
if user.expired_oauth_token:
return
try:
user_events = fbl.get(fb_api.LookupUserEvents, user.fb_uid)
except fb_api.ExpiredOAuthToken as e:
logging.warning("Auth token now expired, skip for now until user-load fixes this: %s", e)
# or do we want to fix-and-put here
#user.expired_oauth_token_reason = e.args[0]
#user.expired_oauth_token = True
#user.put()
#return
else:
# Since we've loaded the latest events from the user, allow future event lookups to come from cache
fbl.allow_cache = True
scrape_user_potential_events.get_potential_dance_events(fbl, user.fb_uid, user_events)
rsvped_events.setup_reminders(user.fb_uid, user_events)
def map_each_attendee(db_events):
db_events = [x for x in db_events if x.is_fb_event]
fbl = fb_mapreduce.get_fblookup()
fbl.request_multi(fb_api.LookupEventAttending,
[x.fb_event_id for x in db_events])
fbl.batch_fetch()
for db_event in db_events:
try:
fb_event_attending = fbl.fetched_data(fb_api.LookupEventAttending,
db_event.id)
except fb_api.NoFetchedDataException:
logging.warning('No attending found for %s', db_event.id)
continue
if fb_event_attending['empty']:
continue
for attendee in fb_event_attending['attending']['data']:
yield ('City: %s' % db_event.city_name, attendee['id'])
yield ('Country: %s' % db_event.country, attendee['id'])
def scrape_sources_for_events(sources):
fbl = fb_mapreduce.get_fblookup()
discovered_list = thing_scraper.discover_events_from_sources(fbl, sources)
for x in discovered_list:
state = (x.source_id, x.source_field, x.extra_source_id)
yield (x.event_id, json.dumps(state))
def scrape_sources_for_events(sources):
fbl = fb_mapreduce.get_fblookup()
discovered_list = thing_scraper.discover_events_from_sources(fbl, sources)
for x in discovered_list:
state = (x.source_id, x.source_field, x.extra_source_id)
yield (x.event_id, json.dumps(state))