def runTest(self, Twitter, Client_request, keys_get):
keys_get.return_value = 'dummy_key'
twitter_instance = Twitter.return_value
twitter_instance.help.configuration.return_value = {
'short_url_length': 22,
'characters_reserved_per_media': 24,
}
# No-op works with with no tokens
pubsub.pull_and_publish_event()
# No-op works with a freshly created token
Client_request.return_value = {'status': '200'}, 'oauth_token=token&oauth_token_secret=secret'
auth_setup.twitter_oauth1('user_id', 'token_nickname', None)
auth_setup.twitter_oauth2('token', 'verifier')
pubsub.pull_and_publish_event()
# Now when we add a token to the pull queue, it will get run
event_id = '383948038362054'
fbl = fb_api.FBLookup(None, None)
fb_event = fbl.get(fb_api.LookupEvent, event_id)
db_event = eventdata.DBEvent.get_or_insert(event_id)
event_updates.update_and_save_fb_events([(db_event, fb_event, None)], disable_updates=['photo'])
db_event.put()
pubsub.eventually_publish_event(event_id)
pubsub.pull_and_publish_event()
# Check that Twitter().statuses.update(...) was called
self.assertTrue(Twitter.return_value.statuses.update.called)
def get(self):
from dancedeets.nlp.soulline.tests import classifier_test
tb = classifier_test.TestSoulLine()
tb.fbl = fb_api.FBLookup("dummyid",
unittest.get_local_access_token_for_testing())
event_runs = []
good_ids = set(classifier_test.GOOD_IDS)
all_ids = classifier_test.GOOD_IDS + classifier_test.BAD_IDS
for event_id in all_ids:
fb_event = tb.get_event(event_id)
classified_event = event_classifier.get_classified_event(fb_event)
data = classifier_test.FUNC(classified_event)
event_runs.append({
'id': event_id,
'event': fb_event,
'desired_result': event_id in good_ids,
'result': bool(data[0]),
'result_string': data[0],
'reasons': data[1],
})
self.display['false_negatives'] = len(
[x for x in event_runs if not x['result'] and x['desired_result']])
self.display['false_positives'] = len(
[x for x in event_runs if x['result'] and not x['desired_result']])
self.display['vertical'] = 'soul line'
self.display['event_runs'] = event_runs
self.render_template('test_nlp_results')
def handle_alternate_login(self, request):
# If the mobile app sent the user to a /....?uid=XX&access_token_md5=YY URL,
# then let's verify the parameters, and log the user in as that user
if request.get('uid'):
if request.get('access_token'):
fbl = fb_api.FBLookup(request.get('uid'), request.get('access_token'))
fb_user = fbl.get(fb_api.LookupUser, 'me')
logging.info("Requested /me with given access_token, got %s", fb_user)
if fb_user['profile']['id'] == request.get('uid'):
user = users.User.get_by_id(request.get('uid'))
access_token_md5 = hashlib.md5(user.fb_access_token).hexdigest()
self.set_login_cookie(request.get('uid'), access_token_md5=access_token_md5)
if request.get('access_token_md5'):
user = users.User.get_by_id(request.get('uid'))
if user and request.get('access_token_md5') == hashlib.md5(user.fb_access_token).hexdigest():
# Authenticated! Now save cookie so subsequent requests can trust that this user is authenticated.
# The subsequent request will see a valid user_login param (though without an fb_cookie_uid)
self.set_login_cookie(request.get('uid'), access_token_md5=self.request.get('access_token_md5'))
# But regardless of whether the token was correct, let's redirect and get rid of these url params.
current_url_args = {}
for arg in sorted(self.request.GET):
if arg in ['uid', 'access_token', 'access_token_md5']:
continue
current_url_args[arg] = self.request.GET.getall(arg)
final_url = self.request.path + '?' + urls.urlencode(current_url_args, doseq=True)
# Make sure we immediately stop running the initialize() code if we return a URL here
return final_url
else:
return False
def setup_fbl(self):
self.allow_cache = bool(int(self.request.get('allow_cache', 1)))
self.fbl = fb_api.FBLookup(self.fb_uid, self.access_token)
self.fbl.allow_cache = self.allow_cache
# Refresh our potential event cache every N days (since they may have updated with better keywords, as often happens)
expiry_days = int(self.request.get('expiry_days', 0)) or None
if expiry_days:
expiry_days += random.uniform(-0.5, 0.5)
def create_user_with_fbuser(fb_uid,
fb_user,
access_token,
access_token_expires,
location,
ip,
send_email=True,
referer=None,
client=None,
send_new_user_email=True):
user = users.User(id=fb_uid)
user.ip = ip
user.fb_access_token = access_token
user.fb_access_token_expires = access_token_expires
user.expired_oauth_token = False
user.expired_oauth_token_reason = None
user.location = location
# grab the cookie to figure out who referred this user
logging.info("Referer was: %s", referer)
if referer:
#STR_ID_MIGRATE
user.inviting_fb_uid = long(referer)
user.clients = [client]
user.send_email = send_email
user.distance = '50'
user.distance_units = 'miles'
user.min_attendees = 0
user.creation_time = datetime.datetime.now()
user.login_count = 1
user.last_login_time = user.creation_time
user.compute_derived_properties(fb_user)
logging.info("Saving user with name %s", user.full_name)
user.put()
logging.info("Requesting background load of user's friends")
# Must occur after User is put with fb_access_token
taskqueue.add(method='GET',
url='/tasks/track_newuser_friends?' +
urls.urlencode({'user_id': fb_uid}),
queue_name='slow-queue')
# Now load their potential events, to make "add event page" faster (and let us process/scrape their events)
backgrounder.load_potential_events_for_users([fb_uid])
fbl = fb_api.FBLookup(fb_uid, user.fb_access_token)
if send_new_user_email:
try:
new_user_email.email_for_user(user, fbl, should_send=True)
except new_user_email.NoEmailException as e:
logging.info('Not sending new-user email due to: %s', e)
return user
def runTest(self):
import pickle
fbl = fb_api.FBLookup('uid', 'access_token')
dumped = pickle.dumps(fbl)
self.assertTrue(len(dumped) < 1000)
fbl._fetched_objects['a'] = 'x' * 10000
dumped = pickle.dumps(fbl)
self.assertTrue(len(dumped) < 1000)
fbl2 = pickle.loads(dumped)
self.assertEqual(fbl2._fetched_objects, {})
def get_dancedeets_fbl():
page_id = '110312662362915'
#page_id = '1375421172766829' # DD-Manager-Test
tokens = db.OAuthToken.query(
db.OAuthToken.user_id == '701004',
db.OAuthToken.token_nickname == page_id,
db.OAuthToken.application == db.APP_FACEBOOK).fetch(1)
if tokens:
return fb_api.FBLookup(None, tokens[0].oauth_token)
else:
return None
def main(style_name):
filename = os.path.join(TEST_IDS_PATH, '%s.txt' % style_name)
if os.path.exists(filename):
raise Exception('File already exists: %s' % filename)
fbl = fb_api.FBLookup("dummyid", unittest.get_local_access_token_for_testing())
fbl.make_passthrough()
all_ids = load_for(style_name, fbl)
write_ids(filename, style_name, all_ids)
def runTest(self):
fbl = fb_api.FBLookup('uid', fb_api_stub.EXPIRED_ACCESS_TOKEN)
fbl.allow_cache = False
self.assertRaises(fb_api.ExpiredOAuthToken, fbl.get, fb_api.LookupUser,
'uid')
try:
fbl.get(fb_api.LookupUser, 'eid')
except fb_api.ExpiredOAuthToken as e:
self.assertTrue('Error validating access token' in e.args[0])
fbl.clear_local_cache()
def init(self):
try:
self.json_data
except AttributeError:
self.json_data = {}
# We shouldn't need any tokens to access pages
if self.graph_id:
fbl = fb_api.FBLookup(None, None)
fb_source = fbl.get(LookupTopicPage, self.graph_id)
self.fb_data = None if fb_source['empty'] else fb_source
else:
self.fb_data = None
def runTest(self):
fbl = fb_api.FBLookup('uid', 'access_token')
# Set up our facebook backend
fb_api.FBAPI.results = {
'/v2.9/uid': (200, {}),
}
# And fetching it then populates our memcache and db
result = fbl.get(fb_api.LookupProfile, 'uid')
self.assertEqual(result, {
'profile': {},
'empty': None,
})
def _initialize(self, request):
self.response.headers.add_header('Access-Control-Allow-Headers',
'Content-Type')
# We use _initialize instead of webapp2's initialize, so that exceptions can be caught easily
self.fbl = fb_api.FBLookup(None, None)
if self.request.body:
logging.info("Request body: %r", self.request.body)
escaped_body = urllib.unquote_plus(self.request.body.strip('='))
self.json_body = json.loads(escaped_body)
logging.info("json_request: %r", self.json_body)
else:
self.json_body = None
if self.requires_auth or self.supports_auth:
if self.json_body.get('access_token'):
access_token = self.json_body.get('access_token')
self.fb_uid = get_user_id_for_token(access_token)
self.fbl = fb_api.FBLookup(self.fb_uid, access_token)
logging.info("Access token for user ID %s", self.fb_uid)
elif self.requires_auth:
self.add_error("Needs access_token parameter")
def get_fblookup(user=None):
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
if params.get('fbl_access_tokens'):
tokens = params.get('fbl_access_tokens')
parent_token = random.choice(tokens)
else:
parent_token = params['fbl_access_token']
access_token = (user and user.fb_access_token or parent_token)
fbl = fb_api.FBLookup(user and user.fb_uid or params['fbl_fb_uid'],
access_token)
fbl.allow_cache = params['fbl_allow_cache']
if params.get('fbl_oldest_allowed') is not None:
fbl.db.oldest_allowed = params['fbl_oldest_allowed']
return fbl
def create_user(user_id='701004',
access_token='Access Token',
access_token_expires=None,
location='NYC'):
fields_str = '%2C'.join(fb_api.OBJ_USER_FIELDS)
base_url = '/v2.9/%s' % user_id
url = '%s?fields=%s' % (base_url, fields_str)
fb_api.FBAPI.results.update({
url: (200, {
'id': user_id,
'name': 'Test User',
'email': '*****@*****.**',
'timezone': -8,
}),
'%s/events?since=yesterday&fields=id,rsvp_status&limit=3000' % base_url:
(200, {
"data": {},
}),
'%s/friends' % base_url: (200, {}),
'%s/permissions' % base_url: (200, {}),
})
existing_user = users.User.get_by_id(user_id)
assert not existing_user, "Found user: %s" % existing_user
if not access_token_expires:
access_token_expires = datetime.datetime.now() + datetime.timedelta(
days=60)
client = 'test'
fbl = fb_api.FBLookup(None, None)
fb_user = fbl.get(fb_api.LookupUser, user_id)
ip = '127.0.0.1'
user = user_creation.create_user_with_fbuser(
user_id,
fb_user,
access_token,
access_token_expires,
location,
ip,
send_email=True,
client=client,
send_new_user_email=False,
)
return user
def get(self):
self.response.headers['Content-Type'] = 'text/plain'
access_token = None
real_key = self.request.get('key')
if not real_key:
fb_uid = self.request.get('fb_uid')
fbl = fb_api.FBLookup(fb_uid, access_token)
fbtype_lookup = {
'OBJ_PROFILE': fb_api.LookupProfile,
'OBJ_USER': fb_api.LookupUser,
'OBJ_USER_EVENTS': fb_api.LookupUserEvents,
'OBJ_EVENT': fb_api.LookupEvent,
'OBJ_EVENT_MEMBERS': fb_api.LookupEventMembers,
'OBJ_THING_FEED': fb_api.LookupThingCommon,
'OBJ_THING_USER': fb_api.LookupThingUser,
'OBJ_THING_GROUP': fb_api.LookupThingGroup,
'OBJ_THING_PAGE': fb_api.LookupThingPage,
}
req_type = self.request.get('type')
if req_type in fbtype_lookup:
fbtype = fbtype_lookup[req_type]
else:
self.response.out.write('type %s must be one of %s' %
(req_type, fbtype_lookup.keys()))
return
key = fb_api.generate_key(fbtype, self.request.get('arg'))
real_key = fbl.key_to_cache_key(key)
memcache_result = memcache.get(real_key)
db_result = fb_api.FacebookCachedObject.get_by_key_name(real_key)
if not db_result:
self.response.set_status(404)
return
self.response.out.write('Database Date:\n%s\n\n' %
db_result.date_cached)
self.response.out.write('Memcache:\n%s\n\n' %
pprint.pformat(memcache_result, width=200))
self.response.out.write('Database:\n%s\n\n' % pprint.pformat(
db_result and db_result.decode_data() or None, width=200))
self.response.out.write('MemcacheJSON:\n%s\n\n' %
json.dumps(memcache_result))
self.response.out.write(
'DatabaseJSON:\n%s\n\n' %
json.dumps(db_result and db_result.decode_data() or None))
def _facebook_weekly_post(db_auth_token, city_data):
city_key = city_data['city']
cities = cities_db.lookup_city_from_geoname_ids([city_key])
city = cities[0]
page_id = db_auth_token.token_nickname
fbl = fb_api.FBLookup(None, db_auth_token.oauth_token)
d = datetime.date.today()
week_start = d - datetime.timedelta(
days=d.weekday()) # round down to last monday
search_results = _generate_results_for(city, week_start)
if len(search_results) < 2:
return {}
# Generate image on GCS
image_url = weekly_images.build_and_cache_image(city, week_start,
search_results)
# Generate the weekly text post
message = _generate_post_for(city, week_start, search_results)
# Can't upload image to FB Page...as the uploaded photo then becomes un-animated
# So instead lets link to the image from our FB post
# Now post to FB Feed about it
post_values = {
'message': message,
'link': image_url,
}
feed_targeting = get_city_targeting_data(fbl, city)
if feed_targeting:
# Ideally we'd do this as 'feed_targeting', but Facebook appears to return errors with that due to:
# {u'error': {u'message': u'Invalid parameter', u'code': 100, u'is_transient': False,
# u'error_user_title': u'Invalid Connection', u'error_subcode': 1487124, u'type': u'FacebookApiException',
# u'error_user_msg': u'You can only specify connections to objects you are an administrator or developer of.',
# u'error_data': {u'blame_field': u'targeting'}}}
post_values['targeting'] = json.dumps(feed_targeting)
logging.info("FB Feed Post Values: %s", post_values)
endpoint = 'v2.9/%s/feed' % page_id
result = fbl.fb.post(endpoint, None, post_values)
logging.info('Post Result for %s: %s', city.display_name(), result)
return result
def create_event(event_id='1000001', start_time=None, location='NYC'):
if not start_time:
start_time = datetime.datetime.now()
fields_str = '%2C'.join(fb_api.OBJ_EVENT_FIELDS)
base_url = '/v2.9/%s' % event_id
url = '%s?fields=%s' % (base_url, fields_str)
picture_url = '%s/picture?redirect=false&type=large' % base_url
wall_fields_str = '%2C'.join(fb_api.OBJ_EVENT_WALL_FIELDS)
wall_url = '%s/feed?fields=%s&limit=1000' % (base_url, wall_fields_str)
fb_api.FBAPI.results.update({
url: (200, {
"name": "Event Title",
"start_time": start_time.strftime("%Y-%m-%dT%H:%M:%S-0400"),
"id": event_id,
}),
'/v2.9/?fields=images%2Cwidth%2Cheight&ids=%7Bresult%3Dinfo%3A%24.cover.id%7D':
(400, {
'error': {
'message': 'Cannot specify an empty identifier',
'code': 2500,
'type': 'OAuthException'
}
}),
picture_url: (200, {
"data": {
"url": "test: image url",
}
}),
wall_url: (200, {
'data': []
})
})
fbl = fb_api.FBLookup(None, None)
fb_event = fbl.get(fb_api.LookupEvent, event_id)
event = add_entities.add_update_fb_event(fb_event,
fbl,
override_address=location)
return event
def post(self):
if self.json_body['scrapinghub_key'] != keys.get('scrapinghub_key'):
self.response.status = 403
return
fb_uid = '701004'
user = users.User.get_by_id(fb_uid)
fbl = fb_api.FBLookup(fb_uid, user.fb_access_token)
for event_url in self.json_body['events']:
logging.info('Adding %s', event_url)
event_id = urls.get_event_id_from_url(event_url)
if not event_id:
logging.warning('Not a valid fb event for adding: %s',
event_url)
continue
fb_event = fbl.get(fb_api.LookupEvent, event_id, allow_cache=False)
try:
add_entities.add_update_fb_event(
fb_event, fbl, creating_method=eventdata.CM_AUTO_WEB)
except add_entities.AddEventException:
logging.exception('Error adding event %s', event_id)
def load_potential_events_for_user(user):
fbl = fb_api.FBLookup(user.fb_uid, user.fb_access_token)
fbl.allow_cache = False
load_potential_events_for_user_ids(fbl, [user.fb_uid])
def initialize(self, request, response):
super(BaseRequestHandler, self).initialize(request, response)
self.run_handler = True
if abuse.is_abuse(self.request):
self.run_handler = False
self.response.out.write(
'You are destroying our server with your request rate. Please implement rate-limiting, respect robots.txt, and/or email [email protected]'
)
return
url = urlparse.urlsplit(self.request.url)
# Always turn https on!
# This only 'takes effect' when it is returned on an https domain,
# so we still need to make sure to add an https redirect.
https_redirect_duration = 60 * 60 * 24 * 365
if 'dev.dancedeets.com' not in url.netloc:
self.response.headers.add_header('Strict-Transport-Security', 'max-age=%s' % https_redirect_duration)
# This is how we detect if the incoming url is on https in GAE Flex (we cannot trust request.url)
http_only_host = 'dev.dancedeets.com' in url.netloc or 'localhost' in url.netloc
if request.method == 'GET' and request.headers.get('x-forwarded-proto', 'http') == 'http' and not http_only_host:
new_url = urlparse.urlunsplit([
'https',
url.netloc,
url.path,
url.query,
url.fragment,
])
self.run_handler = False
self.redirect(new_url, permanent=True, abort=True)
login_url = self.get_login_url()
redirect_url = self.handle_alternate_login(request)
if redirect_url:
self.run_handler = False
# We need to run with abort=False here, or otherwise our set_cookie calls don't work. :(
# Reported in https://github.com/GoogleCloudPlatform/webapp2/issues/111
self.redirect(redirect_url, abort=False)
return
self.setup_login_state(request)
self.display['attempt_autologin'] = 1
# If they've expired, and not already on the login page, then be sure we redirect them to there...
redirect_for_new_oauth_token = (self.user and self.user.expired_oauth_token)
if redirect_for_new_oauth_token:
logging.error("We have a logged in user, but an expired access token. How?!?!")
# TODO(lambert): delete redirect_for_new_oauth_token codepaths
# TODO(lambert): delete codepaths that handle user-id but no self.user. assume this entire thing relates to no-user.
if redirect_for_new_oauth_token or (self.requires_login() and (not self.fb_uid or not self.user)):
# If we're getting a referer id and not signed up, save off a cookie until they sign up
if not self.fb_uid:
logging.info("No facebook cookie.")
if not self.user:
logging.info("No database user object.")
if self.user and self.user.expired_oauth_token:
logging.info("User's OAuth token expired")
#self.set_cookie('fbsr_' + FACEBOOK_CONFIG['app_id'], '', 'Thu, 01 Jan 1970 00:00:01 GMT')
#logging.info("clearing cookie %s", 'fbsr_' + FACEBOOK_CONFIG['app_id'])
self.set_cookie('User-Message', "You changed your facebook password, so will need to click login again.")
if self.request.get('referer'):
self.set_cookie('User-Referer', self.request.get('referer'))
if not self.is_login_page():
logging.info("Login required, redirecting to login page: %s", login_url)
self.run_handler = False
return self.redirect(login_url)
else:
self.display['attempt_autologin'] = 0 # do not attempt auto-login. wait for them to re-login
self.fb_uid = None
self.access_token = None
self.user = None
# If they have a fb_uid, let's do lookups on that behalf (does not require a user)
if self.fb_uid:
self.setup_fbl()
# Always look up the user's information for every page view...?
self.fbl.request(fb_api.LookupUser, self.fb_uid)
else:
self.fbl = fb_api.FBLookup(None, None)
self.fbl.debug = 'fbl' in self.debug_list
if self.user:
self.jinja_env.filters['date_only_human_format'] = self.user.date_only_human_format
self.jinja_env.filters['date_human_format'] = self.user.date_human_format
self.jinja_env.filters['time_human_format'] = self.user.time_human_format
self.jinja_env.globals['duration_human_format'] = self.user.duration_human_format
self.display['messages'] = self.user.get_and_purge_messages()
else:
self.jinja_env.filters['date_only_human_format'] = dates.date_only_human_format
self.jinja_env.filters['date_human_format'] = dates.date_human_format
self.jinja_env.filters['time_human_format'] = dates.time_human_format
self.jinja_env.globals['duration_human_format'] = dates.duration_human_format
self.display['login_url'] = login_url
self.jinja_env.filters['datetime_format'] = dates.datetime_format
self.jinja_env.globals['dd_event_url'] = urls.dd_event_url
self.jinja_env.globals['raw_fb_event_url'] = urls.raw_fb_event_url
self.jinja_env.globals['dd_admin_event_url'] = urls.dd_admin_event_url
self.jinja_env.globals['dd_admin_source_url'] = urls.dd_admin_source_url
self.jinja_env.globals['event_image_url'] = urls.event_image_url
locales = self.request.headers.get('Accept-Language', '').split(',')
self.locales = [x.split(';')[0] for x in locales]
if self.request.get('hl'):
self.locales = self.request.get('hl').split(',')
logging.info('Accept-Language is %s, final locales are %s', self.request.headers.get('Accept-Language', ''), self.locales)
self.display['request'] = request
self.display['app_id'] = facebook.FACEBOOK_CONFIG['app_id']
self.display['prod_mode'] = self.request.app.prod_mode
self.display['base_hostname'] = 'www.dancedeets.com' if self.request.app.prod_mode else 'dev.dancedeets.com'
self.display['full_hostname'] = self._get_full_hostname()
self.display['email_suffix'] = ''
self.display['keyword_tokens'] = [{'value': x.public_name} for x in event_types.STYLES]
fb_permissions = 'rsvp_event,email,user_events'
if self.request.get('all_access'):
fb_permissions += ',read_friendlists,manage_pages'
self.display['fb_permissions'] = fb_permissions
already_used_mobile = self.user and (
'react-android' in self.user.clients or 'react-ios' in self.user.clients or 'android' in self.user.clients or
'ios' in self.user.clients or False
)
mobile_platform = mobile.get_mobile_platform(self.request.user_agent)
show_mobile_promo = not mobile_platform and not already_used_mobile
self.display['show_mobile_promo'] = show_mobile_promo
self.display['mobile_platform'] = mobile_platform
if mobile_platform == mobile.MOBILE_ANDROID:
self.display['mobile_app_url'] = mobile.ANDROID_URL
elif mobile_platform == mobile.MOBILE_IOS:
self.display['mobile_app_url'] = mobile.IOS_URL
self.display['mobile'] = mobile
self.display['mobile_show_smartbanner'] = True
start = time.time()
self.display['ip_location'] = self.get_location_from_headers()
timelog.log_time_since('Getting City from IP', start)
self.display['styles'] = event_types.STYLES
self.display['cities'] = [(
'North America', [
'Albuquerque',
'Austin',
'Baltimore',
'Boston',
'Chicago',
'Detroit',
'Houston',
'Las Vegas',
'Los Angeles',
'Miami',
'New York City',
'Orlando',
'Philadelphia',
'Portland',
'San Francisco',
'San Jose',
'San Diego',
'Seattle',
'Washington DC',
'',
'Calgary',
'Edmonton',
'Montreal',
'Ottawa',
'Toronto',
'Vancouver',
''
'Mexico: Mexico City',
]
), (
'Latin/South America', [
'Argentina: Buenos Aires',
'Argentina: Neuquen',
'Brazil: Belo Horizonte',
'Brazil: Brasilia',
'Brazil: Cruitiba',
'Brazil: Porto Alegre',
'Brazil: Rio de Janeiro',
'Brazil: Sao Paulo',
'Colombia',
'Chile: Santiago',
'Peru: Lima',
]
), (
'Europe', [
'Austria: Vienna',
'Belgium: Brussels',
'Czech: Prague Republic',
'Denmark: Copenhagen',
'Estonia: Tallinn',
'Finland: Helsinki',
'France: Nantes',
'France: Paris',
'France: Perpignan',
'Germany: Berlin',
'Germany: Hamburg',
u'Germany: Köln/Cologne',
'Germany: Leipzig',
u'Germany: München/Munich',
'Italy: Milan',
'Italy: Rome',
'Netherlands: Amsterdam',
'Norway: Oslo',
'Poland: Warsaw',
'Poland: Wroclaw',
'Russia: Moscow',
'Slovakia: Bratislava',
'Spain: Barcelona',
'Sweden: Malmoe',
'Sweden: Stockholm',
'Switzerland: Basel',
'Switzerland: Geneve',
'Switzerland: Zurich',
'United Kingdom: Leeds',
'United Kingdom: London',
]
), (
'Asia', [
'Hong Kong',
'India',
u'Japan: Tokyo (日本東京)',
u'Japan: Osaka (日本大阪)',
'Korea',
u'Taiwan: Kaohsiung (台灣高雄市)',
u'Taiwan: Taipei (台灣台北市)',
u'Taiwan: Taichung (台灣臺中市)',
'Philippines',
'Singapore',
'Australia: Melbourne',
'Australia: Perth',
'Australia: Sydney',
]
)]
self.display['deb'] = self.request.get('deb')
self.display['debug_list'] = self.debug_list
self.display['user'] = self.user
webview = bool(request.get('webview'))
self.display['webview'] = webview
if webview:
self.display['class_base_template'] = '_new_base_webview.html'
else:
self.display['class_base_template'] = '_new_base.html'
totals = rankings.retrieve_summary()
totals['total_events'] = humanize.intcomma(totals['total_events'])
totals['total_users'] = humanize.intcomma(totals['total_users'])
self.display.update(totals)
self.setup_inlined_css()
def facebook_post(auth_token, db_event):
link = common.campaign_url(db_event, 'fb_feed')
datetime_string = db_event.start_time.strftime(
'%s @ %s' % (common.DATE_FORMAT, common.TIME_FORMAT))
page_id = auth_token.token_nickname
endpoint = 'v2.9/%s/feed' % page_id
fbl = fb_api.FBLookup(None, auth_token.oauth_token)
post_values = {}
# post_values['message'] = db_event.name
post_values['link'] = link
post_values['name'] = db_event.name
post_values['caption'] = datetime_string
human_date = db_event.start_time.strftime('%B %-d')
# TODO: Sometimes we definitely know the City (from a lat/long), but FB doesn't give us a city.
# Hopefully in the Great Event Location Cleanup, can take care of this...
if db_event.city:
location = db_event.city
else:
location = ''
host = ''
admins = db_event.admins
if admins:
admin_ids = [x['id'] for x in admins]
page_admin_ids = fb_api_util.filter_by_type(fbl, admin_ids, 'page')
# TODO: Can I @mention the people here too, like I do as a human? Or does it only work with pages?
host = text.human_list('@[%s]' % x for x in page_admin_ids)
# Tag it if we can
if db_event.venue_id:
venue = '@[%s]' % db_event.venue_id
else:
venue = db_event.location_name
if not location:
# Don't want to post events globally...too noisy
return {}
intro = random.choice([
'Coming up soon! ',
'Are you ready? ',
'Just a few days away... ',
'We can\'t wait! ',
'Get ready! ',
'',
])
params = {
'intro': intro,
'name': db_event.name,
'description': db_event.description,
'date': human_date,
#
'venue': venue,
'location': location,
'full_location': db_event.city,
#
'host': host,
}
# Add some callouts
callouts = ''
if host and host != venue:
callouts += random.choice([
'Hosted by our friends at %(host)s.',
'Thanks to our buddies at %(host)s for hosting!',
'Hitup the awesome %(host)s with any questions you\'ve got!',
])
params['callouts'] = callouts % params
# Possible lines:
#━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━
#_____________________________________________________________________
message = """
%(intro)s%(name)s
Date: %(date)s
Venue: %(venue)s, %(full_location)s
%(callouts)s
_____________________________________________________________________
Description:
%(description)s
"""
post_values['message'] = message % params
description = db_event.description
if len(description) > 10000:
post_values['description'] = description[:9999] + u"…"
else:
post_values['description'] = description
post_values['description'] = post_values['description']
cover = db_event.largest_cover
if cover:
post_values['picture'] = cover['source']
venue_id = db_event.venue_id
if venue_id:
post_values['place'] = venue_id
feed_targeting = get_targeting_data(fbl, db_event)
if feed_targeting:
# Ideally we'd do this as 'feed_targeting', but Facebook appears to return errors with that due to:
# {u'error': {u'message': u'Invalid parameter', u'code': 100, u'is_transient': False,
# u'error_user_title': u'Invalid Connection', u'error_subcode': 1487124, u'type': u'FacebookApiException',
# u'error_user_msg': u'You can only specify connections to objects you are an administrator or developer of.',
# u'error_data': {u'blame_field': u'targeting'}}}
post_values['targeting'] = json.dumps(feed_targeting)
logging.info("FB Feed Post Values: %s", post_values)
return fbl.fb.post(endpoint, None, post_values)
def runTest(self):
fbl = fb_api.FBLookup('uid', 'access_token')
fields_str = '%2C'.join(fb_api.OBJ_USER_FIELDS)
url = '/v2.9/uid?fields=%s' % fields_str
event_url = '/v2.9/uid/events?since=yesterday&fields=id,rsvp_status&limit=3000'
# Set up our facebook backend
fb_api.FBAPI.results = {
url: (200, {}),
event_url: (200, {}),
'/v2.9/uid/friends': (200, {}),
'/v2.9/uid/permissions': (200, {}),
}
# And fetching it then populates our memcache and db
result = fbl.get(fb_api.LookupUser, 'uid')
self.assertEqual(
result, {
'empty': None,
'friends': {},
'permissions': {},
'profile': {},
'rsvp_for_future_events': {},
})
# Now remove our facebook backend, and test all our caches
fb_api.FBAPI.results = {}
fbl.clear_local_cache()
# Check that if allow_cache=False, we cannot fetch anything
fbl.allow_cache = False
self.assertRaises(fb_api.NoFetchedDataException, fbl.get,
fb_api.LookupUser, 'uid')
fbl.allow_cache = True
# Rely on memcache/dbcache to fulfill this request now
fbl.clear_local_cache()
result = fbl.get(fb_api.LookupUser, 'uid')
self.assertEqual(
result, {
'empty': None,
'friends': {},
'permissions': {},
'profile': {},
'rsvp_for_future_events': {},
})
# Clear memcache...
user_key = (fb_api.LookupUser, '"uid"')
fbl.m.invalidate_keys([user_key])
fbl.clear_local_cache()
# Check that fetching with allow_db_cache=False, fails
fbl.allow_dbcache = False
self.assertRaises(fb_api.NoFetchedDataException, fbl.get,
fb_api.LookupUser, 'uid')
fbl.allow_dbcache = True
# But allowing db cache still works (and repopulates memcache)
result = fbl.get(fb_api.LookupUser, 'uid')
self.assertEqual(
result, {
'empty': None,
'friends': {},
'permissions': {},
'profile': {},
'rsvp_for_future_events': {},
})
# Clear dbcache, but still can work (because of memcache)
fbl.db.invalidate_keys([user_key])
fbl.clear_local_cache()
# Without allowing memcache read, it fails
fbl.allow_memcache_read = False
self.assertRaises(fb_api.NoFetchedDataException, fbl.get,
fb_api.LookupUser, 'uid')
fbl.allow_memcache_read = True
# But with memcache read, it works fine
result = fbl.get(fb_api.LookupUser, 'uid')
self.assertEqual(
result, {
'empty': None,
'friends': {},
'permissions': {},
'profile': {},
'rsvp_for_future_events': {},
})
# Clear memcache, now that db is empty, data is entirely gone, and it no longer works
fbl.m.invalidate_keys([user_key])
fbl.clear_local_cache()
self.assertRaises(fb_api.NoFetchedDataException, fbl.get,
fb_api.LookupUser, 'uid')
def setup_login_state(self, request):
#TODO(lambert): change fb api to not request access token, and instead pull it from the user
# only request the access token from FB when it's been longer than a day, and do it out-of-band to fetch-and-update-db-and-memcache
self.fb_uid = None
self.user = None
self.access_token = None
if len(request.get_all('nt')) > 1:
logging.error('Have too many nt= parameters, something is Very Wrong!')
for k, v in request.cookies.iteritems():
logging.info("DEBUG: cookie %r = %r", k, v)
fb_cookie_uid = login_logic.get_uid_from_fb_cookie(request.cookies)
our_cookie_uid, set_by_access_token_param = login_logic.get_uid_from_user_login_cookie(request.cookies)
# Normally, our trusted source of login id is the FB cookie,
# though we may override it if access_token_md5 is set from the app
if set_by_access_token_param:
trusted_cookie_uid = our_cookie_uid
logging.info("Validated cookie, logging in as %s", our_cookie_uid)
else:
trusted_cookie_uid = fb_cookie_uid
# If the user has changed facebook users, let's automatically re-login at dancedeets
if fb_cookie_uid and fb_cookie_uid != our_cookie_uid:
self.set_login_cookie(fb_cookie_uid)
our_cookie_uid = fb_cookie_uid
if self.request.cookies.get('user_login', ''):
logging.info("Deleting old-style user_login cookie")
self.response.set_cookie('user_login', '', max_age=0, path='/', domain=self._get_cookie_domain())
# Don't force-logout the user if there is a our_cookie_uid but not a trusted_cookie_uid
# The fb cookie probably expired after a couple hours, and we'd prefer to keep our users logged-in
# Logged-out view, just return without setting anything up
if not our_cookie_uid:
return
self.fb_uid = our_cookie_uid
self.user = users.User.get_by_id(self.fb_uid)
# If we have a user, grab the access token
if self.user:
if trusted_cookie_uid:
# Long-lived tokens should last "around" 60 days, so let's refresh-renew if there's only 40 days left
if self.user.fb_access_token_expires:
token_expires_soon = (self.user.fb_access_token_expires - datetime.datetime.now()) < datetime.timedelta(days=40)
else:
# These are either infinite-access tokens (which won't expire soon)
# or they are ancient tokens (in which case, our User reload mapreduce has already set user.expired_oauth_token)
token_expires_soon = False
# Update the access token if necessary
if self.user.expired_oauth_token or token_expires_soon or self.request.get('update_fb_access_token'):
try:
access_token, access_token_expires = self.get_long_lived_token_and_expires(request)
except TypeError:
logging.info("Could not access cookie ")
except facebook.AlreadyHasLongLivedToken:
logging.info("Already have long-lived token, FB wouldn't give us a new one, so no need to refresh anything.")
else:
logging.info("New access token from cookie: %s, expires %s", access_token, access_token_expires)
if access_token:
self.user = users.User.get_by_id(self.fb_uid)
self.user.fb_access_token = access_token
self.user.fb_access_token_expires = access_token_expires
self.user.expired_oauth_token = False
self.user.expired_oauth_token_reason = None
# this also sets to memcache
self.user.put()
logging.info("Stored the new access_token to the User db")
else:
logging.error("Got a cookie, but no access_token. Using the one from the existing user. Strange!")
if 'web' not in self.user.clients:
self.user = users.User.get_by_id(self.fb_uid)
self.user.clients.append('web')
self.user.put()
logging.info("Added the web client to the User db")
self.access_token = self.user.fb_access_token
else:
self.access_token = self.user.fb_access_token
logging.info("Have dd login cookie but no fb login cookie")
if self.user.expired_oauth_token:
self.fb_uid = None
self.user = None
self.access_token = None
return
elif trusted_cookie_uid:
# if we don't have a user but do have a token, the user has granted us permissions, so let's construct the user now
try:
access_token, access_token_expires = self.get_long_lived_token_and_expires(request)
except facebook.AlreadyHasLongLivedToken:
logging.warning(
"Don't have user, just trusted_cookie_uid. And unable to get long lived token for the incoming request. Giving up and doing logged-out"
)
self.fb_uid = None
self.access_token = None
self.user = None
return
self.access_token = access_token
# Fix this ugly import hack:
fbl = fb_api.FBLookup(self.fb_uid, self.access_token)
fbl.debug = 'fbl' in self.debug_list
fb_user = fbl.get(fb_api.LookupUser, self.fb_uid)
referer = self.get_cookie('User-Referer')
city = self.request.get('city') or self.get_location_from_headers() or get_location(fb_user)
logging.info("User passed in a city of %r, facebook city is %s", self.request.get('city'), get_location(fb_user))
ip = ips.get_remote_ip(self.request)
user_creation.create_user_with_fbuser(
self.fb_uid, fb_user, self.access_token, access_token_expires, city, ip, send_email=True, referer=referer, client='web'
)
# TODO(lambert): handle this MUUUCH better
logging.info("Not a /login request and there is no user object, constructed one realllly-quick, and continuing on.")
self.user = users.User.get_by_id(self.fb_uid)
# Should not happen:
if not self.user:
logging.error("We still don't have a user!")
self.fb_uid = None
self.access_token = None
self.user = None
return
else:
# no user, no trusted_cookie_uid, but we have fb_uid from the user_login cookie
logging.error("We have a user_login cookie, but no user, and no trusted_cookie_uid. Acting as logged-out")
self.fb_uid = None
self.access_token = None
self.user = None
return
logging.info("Logged in uid %s with name %s and token %s", self.fb_uid, self.user.full_name, self.access_token)
# Track last-logged-in state
hour_ago = datetime.datetime.now() - datetime.timedelta(hours=1)
if not getattr(self.user, 'last_login_time', None) or self.user.last_login_time < hour_ago:
# Do this in a separate request so we don't increase latency on this call
deferred.defer(update_last_login_time, self.user.fb_uid, datetime.datetime.now())
backgrounder.load_users([self.fb_uid], allow_cache=False)
def runTest(self):
fbl = fb_api.FBLookup('uid', 'access_token')
fbl.allow_cache = False
# Set up our facebook backend
fields_str = '%2C'.join(fb_api.OBJ_EVENT_FIELDS)
url = '/v2.9/eid?fields=%s' % fields_str
picture_url = '/v2.9/eid/picture?redirect=false&type=large'
# Inaccessible event
fb_api.FBAPI.results = {
url: (400, {
"error": {
"message": "Unsupported get request.",
"type": "GraphMethodException",
"code": 100
}
}),
picture_url: (400, {
"error": {
"message": "Unsupported get request.",
"type": "GraphMethodException",
"code": 100
}
}),
}
result = fbl.get(fb_api.LookupEvent, 'eid')
self.assertEqual(result['empty'],
fb_api.EMPTY_CAUSE_INSUFFICIENT_PERMISSIONS)
fbl.clear_local_cache()
# Partial timeout of optional field
fb_api.FBAPI.results = {
url: (200, {
'id': 'eid'
}),
'/?fields=images&ids=%7Bresult%3Dinfo%3A%24.cover.id%7D':
fb_api_stub.RESULT_TIMEOUT,
picture_url: (200, {}),
}
result = fbl.get(fb_api.LookupEvent, 'eid')
self.assertEqual(result['info']['id'], 'eid')
fbl.clear_local_cache()
# Partial timeout of required field
fb_api.FBAPI.results = {
url: (200, {
'id': 'eid'
}),
'/?fields=images&ids=%7Bresult%3Dinfo%3A%24.cover.id%7D':
fb_api_stub.RESULT_TIMEOUT,
picture_url: fb_api_stub.RESULT_TIMEOUT,
}
self.assertRaises(fb_api.NoFetchedDataException, fbl.get,
fb_api.LookupEvent, 'eid')
fbl.clear_local_cache()
# Event without a Cover field
fb_api.FBAPI.results = {
url: (200, {
"name": "Event Title",
"start_time": "2014-07-12T17:00:00-0400",
"id": "eid"
}),
'/?fields=images&ids=%7Bresult%3Dinfo%3A%24.cover.id%7D': (400, {
'error': {
'message': 'Cannot specify an empty identifier',
'code': 2500,
'type': 'OAuthException'
}
}),
picture_url: (200, {
"data": [{
"pic": "",
"all_members_count": 437,
}]
}),
}
result = fbl.get(fb_api.LookupEvent, 'eid')
self.assertEqual(result['empty'], None)
self.assertEqual(result['info']['id'], 'eid')
fbl.clear_local_cache()
fb_api.FBAPI.do_timeout = True
self.assertRaises(fb_api.NoFetchedDataException, fbl.get,
fb_api.LookupEvent, 'eid')
fb_api.FBAPI.do_timeout = False
fbl.clear_local_cache()
def setUp(self):
super(TestEventLocations, self).setUp()
self.fbl = fb_api.FBLookup("dummyid", None)
def get_fblookup(self):
fbl = fb_api.FBLookup(self.fb_uid, self.fb_access_token)
return fbl
def runTest(self):
ids = ['110312662362915']
fbl = fb_api.FBLookup(None, None)
page_ids = fb_api_util.filter_by_type(fbl, ids, 'page')
self.assertEqual(page_ids, ids)
def setUp(self):
super(TestClassifier, self).setUp()
self.fbl = fb_api.FBLookup("dummyid", unittest.get_local_access_token_for_testing())
def runTest(self):
fbl = fb_api.FBLookup('uid', 'access_token')
fbl.allow_cache = False
# Set up our facebook backend
fb_api.FBAPI.results = {
URL_111: (200, {
'id': '111',
'name': 'page 1'
}),
URL_111_FEED: (200, {
'data': []
}),
URL_111_EVENTS: (200, {
'data': []
}),
URL_111_INFO2: (200, {}),
URL_222: (200, {
'id': '222',
'name': 'page 2'
}),
URL_222_FEED: (200, {
'data': []
}),
URL_222_EVENTS: (200, {
'data': []
}),
URL_222_INFO2: (200, {}),
'/%s/111?metadata=1' % fb_api.LookupThingCommon.version: (200, {
'metadata': {
'type': 'page'
}
}),
'/%s/222?metadata=1' % fb_api.LookupThingCommon.version: (200, {
'metadata': {
'type': 'page'
}
}),
}
# Fetch it and construct a source
source = thing_db.create_source_from_id(fbl, '111')
source.num_all_events = 5
source.put()
source = thing_db.Source.get_by_key_name('111')
self.assertEquals(source.name, 'page 1')
self.mark_as_error_and_reload(fbl)
# Let's verify id 111 no longer exists
source = thing_db.Source.get_by_key_name('111')
# Now let's load id 222 and verify it exists
source = thing_db.Source.get_by_key_name('222')
# And verify that our data from the first one got carried over
self.assertEquals(source.num_all_events, 5)
# But the the remaining data comes from the latest FB values for this page id
self.assertEquals(source.name, 'page 2')
# Now let's create 111 again, to verify merge works
fb_api.FBAPI.results.update({
URL_111: (200, {
'id': '111',
'name': 'page 1'
}),
URL_111_FEED: (200, {
'data': []
}),
})
source = thing_db.create_source_from_id(fbl, '111')
source.num_all_events = 5
source.put()
pe = potential_events.PotentialEvent(key_name="333")
pe.set_sources([
potential_events.PESource("111", thing_db.FIELD_FEED),
potential_events.PESource("222", thing_db.FIELD_FEED)
])
pe.put()
self.mark_as_error_and_reload(fbl)
pe = potential_events.PotentialEvent.get_by_key_name("333")
#STR_ID_MIGRATE
self.assertEqual(
pe.sources(),
[potential_events.PESource("222", thing_db.FIELD_FEED)])
