def process_event_source_ids(discovered_list, fbl):
# TODO(lambert): maybe trim any ids from posts with dates "past" the last time we scraped? tricky to get correct though
logging.info("Loading processing %s discovered events",
len(discovered_list))
event_pipeline.process_discovered_events(fbl, discovered_list)
# TODO: Should only run this code on events that we actually decide are worth adding
if False:
potential_new_source_ids = set(
[x.extra_source_id for x in discovered_list if x.extra_source_id])
existing_source_ids = set([
x.graph_id
for x in thing_db.Source.get_by_key_name(potential_new_source_ids)
if x
])
new_source_ids = set([
x for x in potential_new_source_ids if x not in existing_source_ids
])
for source_id in new_source_ids:
#TODO(lambert): we know it doesn't exist, why does create_source_from_id check datastore?
s = thing_db.Source(key_name=source_id)
s.put()
logging.info("Found %s new sources", len(new_source_ids))
# initiate an out-of-band-scrape for our new sources we found
if new_source_ids:
deferred.defer(scrape_events_from_source_ids, fbl, new_source_ids)
def function_migrate_thing_to_new_id(fbapi_obj, old_source_id, new_source_id):
old_source = thing_db.Source.get_by_key_name(old_source_id)
# Maybe we got two of these and it already ran in parallel, so ignore this one
if not old_source:
return
fbl = fb_api.FBLookup(None, fbapi_obj.access_token_list)
fbl.fb.raise_on_page_redirect = True
try:
results = fbl.get(fb_api.LookupThingCommon, new_source_id)
except fb_api.PageRedirectException as e:
# If our forwarding address in turn has its own forwarding address,
# repoint the old thing further down the chain
deferred.defer(function_migrate_thing_to_new_id, fbl.fb, old_source_id, e.to_id)
return
new_source = thing_db.create_source_from_id(fbl, new_source_id)
new_source.creating_fb_uid = new_source.creating_fb_uid or old_source.creating_fb_uid
new_source.creation_time = new_source.creation_time or old_source.creation_time
new_source.last_scrape_time = new_source.last_scrape_time or old_source.last_scrape_time
new_source.verticals = new_source.verticals or old_source.verticals
new_source.num_all_events = (new_source.num_all_events or 0) + (old_source.num_all_events or 0)
new_source.num_potential_events = (new_source.num_potential_events or 0) + (old_source.num_potential_events or 0)
new_source.num_real_events = (new_source.num_real_events or 0) + (old_source.num_real_events or 0)
new_source.num_false_negatives = (new_source.num_false_negatives or 0) + (old_source.num_false_negatives or 0)
# Who has pointers to sources??
migrate_potential_events(old_source_id, new_source_id)
new_source.put()
old_source.delete()
def rebuild_from_query(cls, force=False):
logging.info("Loading Index")
if cls._is_ndb():
db_query = cls.obj_type.query(
*cls._get_query_params_for_indexing())
else:
db_query = cls.obj_type.all()
object_keys = db_query.fetch(MAX_OBJECTS, keys_only=True)
object_ids = set(cls._get_id(x) for x in object_keys)
logging.info("Loaded %s objects for indexing", len(object_ids))
if len(object_ids) >= MAX_OBJECTS:
logging.critical(
'Found %s objects. Increase the MAX_OBJECTS limit to search more events.',
MAX_OBJECTS)
doc_index = cls.real_index()
docs_per_group = search.MAXIMUM_DOCUMENTS_PER_PUT_REQUEST
logging.info("Deleting Expired docs")
start_id = '0'
doc_ids_to_delete = set()
while True:
doc_ids = [
x.doc_id
for x in doc_index.get_range(ids_only=True,
start_id=start_id,
include_start_object=False)
]
if not doc_ids:
break
new_ids_to_delete = set(doc_ids).difference(object_ids)
doc_ids_to_delete.update(new_ids_to_delete)
logging.info(
"Looking at %s doc_id candidates for deletion, will delete %s entries.",
len(doc_ids), len(new_ids_to_delete))
start_id = doc_ids[-1]
if not force and len(doc_ids_to_delete) and len(
doc_ids_to_delete) > len(object_ids) * cls.delete_threshold:
logging.critical(
"Deleting %s docs, more than %d%% of total %s docs",
len(doc_ids_to_delete), cls.delete_threshold * 100,
len(object_ids))
return
logging.info("Deleting %s docs", len(doc_ids_to_delete))
cls.delete_ids(list(doc_ids_to_delete))
# Add all events
logging.info("Loading %s docs, in groups of %s", len(object_ids),
docs_per_group)
object_ids_list = sorted(object_ids)
for x in object_ids_list:
logging.info('index: %s', x)
for i in range(0, len(object_ids_list), docs_per_group):
group_object_ids = object_ids_list[i:i + docs_per_group]
deferred.defer(cls._save_ids, group_object_ids)
def add_update_web_event(json_body, allow_posting=True):
event_id = eventdata.DBEvent.generate_id(json_body['namespace'],
json_body['namespaced_id'])
e = eventdata.DBEvent.get_or_insert(event_id)
newly_created = (e.creating_method is None)
e.creating_method = eventdata.CM_WEB_SCRAPE
event_updates.update_and_save_web_events([(e, json_body)])
post_pubsub = newly_created and allow_posting
send_email = newly_created
deferred.defer(after_add_event, e.id, None, send_email, post_pubsub)
def post(self):
event_id = self.request.get('event_id')
remapped_address = self.request.get('remapped_address')
override_address = self.request.get('override_address')
verticals = [x for x in self.request.get('verticals').split(',') if x]
if self.request.get('delete'):
e = eventdata.DBEvent.get_by_id(event_id)
# This e will be None if the user submits a deletion-form twice
if e:
event_updates.delete_event(e)
self.user.add_message("Event deleted!")
return self.redirect('/events/admin_edit?event_id=%s' % event_id)
# We could be looking at a potential event for something that is inaccessable to our admin.
# So we want to grab the cached value here if possible, which should exist given the admin-edit flow.
fb_event = get_fb_event(self.fbl, event_id)
logging.info("Fetched fb_event %s", fb_event)
if not fb_events.is_public_ish(fb_event):
self.add_error('Cannot add secret/closed events to dancedeets!')
self.errors_are_fatal()
if self.request.get('background'):
deferred.defer(add_entities.add_update_fb_event,
fb_event,
self.fbl,
creating_uid=self.user.fb_uid,
remapped_address=remapped_address,
override_address=override_address,
creating_method=eventdata.CM_ADMIN,
verticals=verticals)
self.response.out.write("<title>Added!</title>Added!")
else:
try:
add_entities.add_update_fb_event(
fb_event,
self.fbl,
creating_uid=self.user.fb_uid,
remapped_address=remapped_address,
override_address=override_address,
creating_method=eventdata.CM_ADMIN,
verticals=verticals)
except Exception as e:
logging.exception('Error adding event')
self.add_error(str(e))
self.errors_are_fatal()
self.user.add_message("Changes saved!")
return self.redirect('/events/admin_edit?event_id=%s' % event_id)
def process_event_source_ids(discovered_list, fbl):
# TODO(lambert): maybe trim any ids from posts with dates "past" the last time we scraped? tricky to get correct though
logging.info("Loading processing %s discovered events", len(discovered_list))
event_pipeline.process_discovered_events(fbl, discovered_list)
# TODO: Should only run this code on events that we actually decide are worth adding
if False:
potential_new_source_ids = set([x.extra_source_id for x in discovered_list if x.extra_source_id])
existing_source_ids = set([x.graph_id for x in thing_db.Source.get_by_key_name(potential_new_source_ids) if x])
new_source_ids = set([x for x in potential_new_source_ids if x not in existing_source_ids])
for source_id in new_source_ids:
#TODO(lambert): we know it doesn't exist, why does create_source_from_id check datastore?
s = thing_db.Source(key_name=source_id)
s.put()
logging.info("Found %s new sources", len(new_source_ids))
# initiate an out-of-band-scrape for our new sources we found
if new_source_ids:
deferred.defer(scrape_events_from_source_ids, fbl, new_source_ids)
def migrate_potential_events(old_source_id, new_source_id):
#STR_ID_MIGRATE
potential_event_list = potential_events.PotentialEvent.gql("WHERE source_ids = %s" % long(old_source_id)).fetch(100)
for pe in potential_event_list:
logging.info("old pe %s has ids: %s", pe.fb_event_id, [x.id for x in pe.sources()])
source_infos = set()
for source in pe.sources():
# remap ids
if source.id == old_source_id:
#STR_ID_MIGRATE
source = source.copy()
source.id = new_source_id
source_infos.add(source)
pe.set_sources(source_infos)
logging.info("new pe %s has ids: %s", pe.fb_event_id, [x.id for x in pe.sources()])
pe.put()
if len(potential_event_list):
# Tail recursion via task queues!
deferred.defer(migrate_potential_events, old_source_id, new_source_id)
def setup_login_state(self, request):
#TODO(lambert): change fb api to not request access token, and instead pull it from the user
# only request the access token from FB when it's been longer than a day, and do it out-of-band to fetch-and-update-db-and-memcache
self.fb_uid = None
self.user = None
self.access_token = None
if len(request.get_all('nt')) > 1:
logging.error('Have too many nt= parameters, something is Very Wrong!')
for k, v in request.cookies.iteritems():
logging.info("DEBUG: cookie %r = %r", k, v)
fb_cookie_uid = login_logic.get_uid_from_fb_cookie(request.cookies)
our_cookie_uid, set_by_access_token_param = login_logic.get_uid_from_user_login_cookie(request.cookies)
# Normally, our trusted source of login id is the FB cookie,
# though we may override it if access_token_md5 is set from the app
if set_by_access_token_param:
trusted_cookie_uid = our_cookie_uid
logging.info("Validated cookie, logging in as %s", our_cookie_uid)
else:
trusted_cookie_uid = fb_cookie_uid
# If the user has changed facebook users, let's automatically re-login at dancedeets
if fb_cookie_uid and fb_cookie_uid != our_cookie_uid:
self.set_login_cookie(fb_cookie_uid)
our_cookie_uid = fb_cookie_uid
if self.request.cookies.get('user_login', ''):
logging.info("Deleting old-style user_login cookie")
self.response.set_cookie('user_login', '', max_age=0, path='/', domain=self._get_cookie_domain())
# Don't force-logout the user if there is a our_cookie_uid but not a trusted_cookie_uid
# The fb cookie probably expired after a couple hours, and we'd prefer to keep our users logged-in
# Logged-out view, just return without setting anything up
if not our_cookie_uid:
return
self.fb_uid = our_cookie_uid
self.user = users.User.get_by_id(self.fb_uid)
# If we have a user, grab the access token
if self.user:
if trusted_cookie_uid:
# Long-lived tokens should last "around" 60 days, so let's refresh-renew if there's only 40 days left
if self.user.fb_access_token_expires:
token_expires_soon = (self.user.fb_access_token_expires - datetime.datetime.now()) < datetime.timedelta(days=40)
else:
# These are either infinite-access tokens (which won't expire soon)
# or they are ancient tokens (in which case, our User reload mapreduce has already set user.expired_oauth_token)
token_expires_soon = False
# Update the access token if necessary
if self.user.expired_oauth_token or token_expires_soon or self.request.get('update_fb_access_token'):
try:
access_token, access_token_expires = self.get_long_lived_token_and_expires(request)
except TypeError:
logging.info("Could not access cookie ")
except facebook.AlreadyHasLongLivedToken:
logging.info("Already have long-lived token, FB wouldn't give us a new one, so no need to refresh anything.")
else:
logging.info("New access token from cookie: %s, expires %s", access_token, access_token_expires)
if access_token:
self.user = users.User.get_by_id(self.fb_uid)
self.user.fb_access_token = access_token
self.user.fb_access_token_expires = access_token_expires
self.user.expired_oauth_token = False
self.user.expired_oauth_token_reason = None
# this also sets to memcache
self.user.put()
logging.info("Stored the new access_token to the User db")
else:
logging.error("Got a cookie, but no access_token. Using the one from the existing user. Strange!")
if 'web' not in self.user.clients:
self.user = users.User.get_by_id(self.fb_uid)
self.user.clients.append('web')
self.user.put()
logging.info("Added the web client to the User db")
self.access_token = self.user.fb_access_token
else:
self.access_token = self.user.fb_access_token
logging.info("Have dd login cookie but no fb login cookie")
if self.user.expired_oauth_token:
self.fb_uid = None
self.user = None
self.access_token = None
return
elif trusted_cookie_uid:
# if we don't have a user but do have a token, the user has granted us permissions, so let's construct the user now
try:
access_token, access_token_expires = self.get_long_lived_token_and_expires(request)
except facebook.AlreadyHasLongLivedToken:
logging.warning(
"Don't have user, just trusted_cookie_uid. And unable to get long lived token for the incoming request. Giving up and doing logged-out"
)
self.fb_uid = None
self.access_token = None
self.user = None
return
self.access_token = access_token
# Fix this ugly import hack:
fbl = fb_api.FBLookup(self.fb_uid, self.access_token)
fbl.debug = 'fbl' in self.debug_list
fb_user = fbl.get(fb_api.LookupUser, self.fb_uid)
referer = self.get_cookie('User-Referer')
city = self.request.get('city') or self.get_location_from_headers() or get_location(fb_user)
logging.info("User passed in a city of %r, facebook city is %s", self.request.get('city'), get_location(fb_user))
ip = ips.get_remote_ip(self.request)
user_creation.create_user_with_fbuser(
self.fb_uid, fb_user, self.access_token, access_token_expires, city, ip, send_email=True, referer=referer, client='web'
)
# TODO(lambert): handle this MUUUCH better
logging.info("Not a /login request and there is no user object, constructed one realllly-quick, and continuing on.")
self.user = users.User.get_by_id(self.fb_uid)
# Should not happen:
if not self.user:
logging.error("We still don't have a user!")
self.fb_uid = None
self.access_token = None
self.user = None
return
else:
# no user, no trusted_cookie_uid, but we have fb_uid from the user_login cookie
logging.error("We have a user_login cookie, but no user, and no trusted_cookie_uid. Acting as logged-out")
self.fb_uid = None
self.access_token = None
self.user = None
return
logging.info("Logged in uid %s with name %s and token %s", self.fb_uid, self.user.full_name, self.access_token)
# Track last-logged-in state
hour_ago = datetime.datetime.now() - datetime.timedelta(hours=1)
if not getattr(self.user, 'last_login_time', None) or self.user.last_login_time < hour_ago:
# Do this in a separate request so we don't increase latency on this call
deferred.defer(update_last_login_time, self.user.fb_uid, datetime.datetime.now())
backgrounder.load_users([self.fb_uid], allow_cache=False)
def add_update_fb_event(
fb_event,
fbl,
creating_uid=None,
visible_to_fb_uids=None,
remapped_address=None,
override_address=None,
creating_method=None,
allow_posting=True,
verticals=None,
):
if not fb_events.is_public_ish(fb_event):
raise AddEventException(
'Cannot add secret/closed events to dancedeets!')
if remapped_address is not None:
event_locations.update_remapped_address(fb_event, remapped_address)
event_id = fb_event['info']['id']
e = eventdata.DBEvent.get_or_insert(event_id)
newly_created = (e.creating_method is None)
if override_address is not None:
e.address = override_address
if e.creating_method is None:
e.creating_method = creating_method or eventdata.CM_UNKNOWN
# Allow an override if we get a user or admin taking a human action
if creating_method in eventdata.ALL_CM_HUMAN_CREATED:
e.creating_method = creating_method or e.creating_method
# Don't override the original creating_fb_uid
if not e.creating_fb_uid:
#STR_ID_MIGRATE
e.creating_fb_uid = long(creating_uid) if creating_uid else None
if e.creating_fb_uid:
user = fbl.get(fb_api.LookupUser, creating_uid)
e.creating_name = user['profile']['name']
if visible_to_fb_uids is None:
if creating_uid is not None:
visible_to_fb_uids = [creating_uid]
else:
visible_to_fb_uids = []
e.visible_to_fb_uids = visible_to_fb_uids
e.verticals = verticals or []
try:
fb_event_attending_maybe = fbl.get(fb_api.LookupEventAttendingMaybe,
event_id)
except fb_api.NoFetchedDataException as error:
logging.warning(
'Error loading fb-event-attending-maybe in add_update_fb_event: %s',
error)
fb_event_attending_maybe = None
# Updates and saves the event
event_updates.update_and_save_fb_events([(e, fb_event,
fb_event_attending_maybe)])
post_pubsub = newly_created and allow_posting and event_types.should_show(
e)
# Trigger lookup requests (these will be refreshed later, but we want them populated here at least)
fbl.request(fb_api.LookupEventWall, event_id)
fbl.request(fb_api.LookupEventAttending, event_id)
fbl.batch_fetch()
fbl.clear_local_cache()
send_email = newly_created and event_types.should_show(e)
deferred.defer(after_add_event, e.id, fbl, send_email, post_pubsub)
return e
def yield_load_fb_event(fbl, all_events):
ctx = context.get()
if ctx:
params = ctx.mapreduce_spec.mapper.params
disable_updates = params['disable_updates']
only_if_updated = params['only_if_updated']
else:
disable_updates = []
only_if_updated = True
for event in all_events:
# Temporary hack to fix our double-street events
verticals = sorted(list(set(event.verticals)))
if verticals != sorted(event.verticals):
event.verticals = verticals
event.put()
for vertical in event.verticals:
mr.increment('event-vertical-%s' % vertical)
# Process web_events
web_events = [x for x in all_events if not x.is_fb_event]
web_events_to_update = []
for web_event in web_events:
if event_updates.need_forced_update(web_event):
web_events_to_update.append(web_event)
event_updates.reload_web_events(web_events_to_update, disable_updates=disable_updates)
# Now process fb_events
db_events = [x for x in all_events if x.is_fb_event]
logging.info("loading db events %s", [db_event.fb_event_id for db_event in db_events])
fbl.request_multi(fb_api.LookupEvent, [x.fb_event_id for x in db_events])
fbl.request_multi(fb_api.LookupEventWall, [x.fb_event_id for x in db_events])
fbl.request_multi(fb_api.LookupEventAttending, [x.fb_event_id for x in db_events])
# We load these too, just in case we want to check up on our auto-attendee criteria for events
fbl.request_multi(fb_api.LookupEventAttendingMaybe, [x.fb_event_id for x in db_events])
# fbl.request_multi(fb_api.LookupEventPageComments, [x.fb_event_id for x in db_events])
fbl.batch_fetch()
events_to_update = []
empty_fb_event_ids = []
for db_event in db_events:
try:
real_fb_event = fbl.fetched_data(fb_api.LookupEvent, db_event.fb_event_id)
# If it's an empty fb_event with our main access token, and we have other tokens we'd like to try...
# If there are no visible_to_fb_uids and we don't have permissions, then we don't do this...
#
# TODO: This would happen on event deletion?
#
# TODO: Also, who sets visible_to_fb_uids? Why didn't this event have any?
# TODO: Who re-sets visible_to_fb_uids after it goes empty? Can we ensure that keeps going?
#
# TODO: And what happens if we have a deleted event, with visible_to_fb_uids, that we attempt to run and query, and nothing happens?
# Should we distinguish between deleted (and inaccessible) and permissions-lost-to-token (and inaccessible)?
#
# TODO: Why doesn't this update the event? Because add_event_tuple_if_updating seems to do nothing, probably because no fb_event is returned
if real_fb_event['empty'] == fb_api.EMPTY_CAUSE_INSUFFICIENT_PERMISSIONS and db_event.visible_to_fb_uids:
empty_fb_event_ids.append(db_event.fb_event_id)
else:
# Otherwise if it's visible to our main token, or there are no other tokens to try, deal with it here.
add_event_tuple_if_updating(events_to_update, fbl, db_event, only_if_updated)
except fb_api.NoFetchedDataException as e:
logging.info("No data fetched for event id %s: %s", db_event.fb_event_id, e)
# Now trigger off a background reloading of empty fb_events
if empty_fb_event_ids:
logging.info("Couldn't fetch, using backup tokens for events: %s", empty_fb_event_ids)
deferred.defer(
load_fb_events_using_backup_tokens,
empty_fb_event_ids,
allow_cache=fbl.allow_cache,
only_if_updated=only_if_updated,
disable_updates=disable_updates
)
logging.info("Updating events: %s", [x[0].id for x in events_to_update])
# And then re-save all the events in here
if events_to_update:
event_updates.update_and_save_fb_events(events_to_update, disable_updates=disable_updates)
def _fetch_object_keys(self, object_keys_to_lookup):
logging.info("BatchLookup: Fetching IDs from FB: %s", object_keys_to_lookup)
# initiate RPCs
object_keys_to_rpcs = {}
for object_key in object_keys_to_lookup:
cls, oid = break_key(object_key)
cls.track_lookup()
parts_to_urls = cls.get_lookups(oid)
batch_list = [
dict(method='GET', name=part_key, relative_url=url, omit_response_on_success=False) for (part_key, url) in parts_to_urls
]
rpc, token = self._create_rpc_for_batch(batch_list, cls.use_access_token)
object_keys_to_rpcs[object_key] = rpc, token
# fetch RPCs
fetched_objects = {}
for object_key, (object_rpc, object_token) in object_keys_to_rpcs.iteritems():
cls, oid = break_key(object_key)
parts_to_urls = cls.get_lookups(oid)
mini_batch_list = [dict(name=part_key, relative_url=url) for (part_key, url) in parts_to_urls]
this_object = {}
this_object['empty'] = None
object_is_bad = False
rpc_results = self._map_rpc_to_data(object_rpc)
if isinstance(rpc_results, list):
named_results = zip(mini_batch_list, rpc_results)
elif rpc_results is None:
logging.warning("BatchLookup: Has empty rpc_results, perhaps due to URL fetch timeout")
object_is_bad = True
named_results = []
else:
error_code = rpc_results.get('error', {}).get('code')
error_type = rpc_results.get('error', {}).get('type')
error_message = rpc_results.get('error', {}).get('message')
# expired/invalidated OAuth token for User objects. We use one OAuth token per BatchLookup, so no use continuing...
# we don't trigger on UserEvents objects since those are often optional and we don't want to break on those, or set invalid bits on those (get it from the User failures instead)
if error_code == 190 and error_type == 'OAuthException':
logging.warning("Error with expired token: %s", object_token)
raise ExpiredOAuthToken(error_message)
logging.error("BatchLookup: Error occurred on response, rpc_results is %s", rpc_results)
object_is_bad = True
named_results = []
for batch_item, result in named_results:
object_rpc_name = batch_item['name']
if result is None:
logging.warning("BatchLookup: Got timeout when requesting %s", batch_item)
if object_rpc_name not in cls.optional_keys:
object_is_bad = True
continue
object_result_code = result['code']
try:
object_json = json.loads(result['body'])
except:
logging.error('Error parsing result body for %r: %r', batch_item, result)
raise
if object_result_code in [200, 400] and object_json is not None:
error_code = None
if type(object_json) == dict and ('error_code' in object_json or 'error' in object_json):
error_code = object_json.get('error_code', object_json.get('error', {}).get('code', None))
if error_code == 100:
# This means the event exists, but the current access_token is insufficient to query it
this_object['empty'] = EMPTY_CAUSE_INSUFFICIENT_PERMISSIONS
elif error_code == 21:
message = object_json['error']['message']
# Facebook gave us a huge hack when they decided to rename/merge page ids,
# and so we are forced to deal with remapping by parsing strings at this lowest level.
# "Page ID 289919164441106 was migrated to page ID 175608368718. Please update your API calls to the new ID"
# But only do it once per object, so rely on object_is_bad to tell us whether we've been through this before
if not object_is_bad and re.search('Page ID \d+ was migrated to page ID \d+.', message):
from_id, to_id = re.findall(r'ID (\d+)', message)
if self.raise_on_page_redirect:
raise PageRedirectException(from_id, to_id)
else:
from dancedeets.event_scraper import thing_db_fixer
from dancedeets.util import deferred
logging.warning(message)
logging.warning("Executing deferred call to migrate to new ID, returning None here.")
deferred.defer(thing_db_fixer.function_migrate_thing_to_new_id, self, from_id, to_id)
object_is_bad = True
elif error_code in [
2, # Temporary API error: An unexpected error has occurred. Please retry your request later.
2500, # Dependent-lookup on non-existing field: Cannot specify an empty identifier.
]:
# Handle errors as documented here: https://developers.facebook.com/docs/graph-api/using-graph-api/v2.0#errors
logging.warning("BatchLookup: Error code from FB server for %s: %s: %s", object_rpc_name, error_code, object_json)
mr.increment('fb-lookups-errors-%s-%s' % (object_rpc_name, error_code))
if object_rpc_name not in cls.optional_keys:
object_is_bad = True
elif error_code:
logging.error("BatchLookup: Error code from FB server for %s: %s: %s", object_rpc_name, error_code, object_json)
mr.increment('fb-lookups-errors-%s-%s' % (object_rpc_name, error_code))
if object_rpc_name not in cls.optional_keys:
object_is_bad = True
elif object_json == False:
this_object['empty'] = EMPTY_CAUSE_DELETED
else:
this_object[object_rpc_name] = object_json
else:
logging.warning("BatchLookup: Got code %s when requesting %s: %s", object_result_code, batch_item, result)
if object_rpc_name not in cls.optional_keys:
object_is_bad = True
if object_is_bad:
logging.warning("BatchLookup: Failed to complete object: %s, only have keys %s", object_key, this_object.keys())
else:
fetched_objects[object_key] = this_object
return fetched_objects
def yield_load_fb_event(fbl, all_events):
ctx = context.get()
if ctx:
params = ctx.mapreduce_spec.mapper.params
disable_updates = params['disable_updates']
only_if_updated = params['only_if_updated']
else:
disable_updates = []
only_if_updated = True
for event in all_events:
# Temporary hack to fix our double-street events
verticals = sorted(list(set(event.verticals)))
if verticals != sorted(event.verticals):
event.verticals = verticals
event.put()
for vertical in event.verticals:
mr.increment('event-vertical-%s' % vertical)
# Process web_events
web_events = [x for x in all_events if not x.is_fb_event]
web_events_to_update = []
for web_event in web_events:
if event_updates.need_forced_update(web_event):
web_events_to_update.append(web_event)
event_updates.reload_web_events(web_events_to_update,
disable_updates=disable_updates)
# Now process fb_events
db_events = [x for x in all_events if x.is_fb_event]
logging.info("loading db events %s",
[db_event.fb_event_id for db_event in db_events])
fbl.request_multi(fb_api.LookupEvent, [x.fb_event_id for x in db_events])
fbl.request_multi(fb_api.LookupEventWall,
[x.fb_event_id for x in db_events])
fbl.request_multi(fb_api.LookupEventAttending,
[x.fb_event_id for x in db_events])
# We load these too, just in case we want to check up on our auto-attendee criteria for events
fbl.request_multi(fb_api.LookupEventAttendingMaybe,
[x.fb_event_id for x in db_events])
# fbl.request_multi(fb_api.LookupEventPageComments, [x.fb_event_id for x in db_events])
fbl.batch_fetch()
events_to_update = []
empty_fb_event_ids = []
for db_event in db_events:
try:
real_fb_event = fbl.fetched_data(fb_api.LookupEvent,
db_event.fb_event_id)
# If it's an empty fb_event with our main access token, and we have other tokens we'd like to try...
# If there are no visible_to_fb_uids and we don't have permissions, then we don't do this...
#
# TODO: This would happen on event deletion?
#
# TODO: Also, who sets visible_to_fb_uids? Why didn't this event have any?
# TODO: Who re-sets visible_to_fb_uids after it goes empty? Can we ensure that keeps going?
#
# TODO: And what happens if we have a deleted event, with visible_to_fb_uids, that we attempt to run and query, and nothing happens?
# Should we distinguish between deleted (and inaccessible) and permissions-lost-to-token (and inaccessible)?
#
# TODO: Why doesn't this update the event? Because add_event_tuple_if_updating seems to do nothing, probably because no fb_event is returned
if real_fb_event[
'empty'] == fb_api.EMPTY_CAUSE_INSUFFICIENT_PERMISSIONS and db_event.visible_to_fb_uids:
empty_fb_event_ids.append(db_event.fb_event_id)
else:
# Otherwise if it's visible to our main token, or there are no other tokens to try, deal with it here.
add_event_tuple_if_updating(events_to_update, fbl, db_event,
only_if_updated)
except fb_api.NoFetchedDataException as e:
logging.info("No data fetched for event id %s: %s",
db_event.fb_event_id, e)
# Now trigger off a background reloading of empty fb_events
if empty_fb_event_ids:
logging.info("Couldn't fetch, using backup tokens for events: %s",
empty_fb_event_ids)
deferred.defer(load_fb_events_using_backup_tokens,
empty_fb_event_ids,
allow_cache=fbl.allow_cache,
only_if_updated=only_if_updated,
disable_updates=disable_updates)
logging.info("Updating events: %s", [x[0].id for x in events_to_update])
# And then re-save all the events in here
if events_to_update:
event_updates.update_and_save_fb_events(
events_to_update, disable_updates=disable_updates)