def try_token(self, token):
user_clients = relational.UserClient.objects.filter(fbid=token.fbid, client__fb_app_id=token.appid)
client_codenames = user_clients.values_list('client__codename', flat=True)
app_stats = self.stats.get_many(client_codenames.iterator())
# We expect values of "expires" to be optimistic, meaning we trust dates
# in the past, but must confirm dates in the future.
# (We sometimes set field optimistically; and, user can invalidate our
# token, throwing its actual expires to 0.)
if not token.expires or token.expires == EPOCH_ZERO:
app_stats.increment_invalid()
return False
if token.expires <= epoch.utcnow():
app_stats.increment_expired()
return False
try:
secret = _SECRETS[token.appid]
except KeyError:
secret = relational.FBApp.objects.values_list('secret', flat=True).get(appid=token.appid)
_SECRETS[token.appid] = secret
# Confirm token expiration (and validity)
debug_result = facebook.client.debug_token(token.appid, secret, token.token)
debug_data = debug_result['data']
token_valid = debug_data['is_valid']
token_expires = debug_data['expires_at']
# Update token, if needed; but, restart if another process has changed
# the token (meaning it may now refer to new value):
token.expires = token_expires
updated = token.partial_save()
if token_valid and token.expires > epoch.utcnow():
app_stats.increment_valid()
elif token.expires > EPOCH_ZERO:
app_stats.increment_expired()
else:
app_stats.increment_invalid()
return updated
def test_extension(self, _urllib_mock):
models.relational.FBApp.objects.create(
appid=471727162864364,
name='Share!',
secret='sekret',
)
tokens = models.Token.items.filter(fbid__eq=100, appid__eq=471727162864364)
self.assertEqual(tokens.query_count(), 0)
now = epoch.utcnow()
facebook.extend_token(100, 471727162864364, 'xyz', '1.0')
token = tokens.filter_get()
self.assertEqual(token.token, 'tok1')
self.assertAlmostEqual(token.expires, now + timedelta(days=60),
delta=timedelta(seconds=1))
def handle(self, server, campaign_id, client_content_id, fbidfile, limit, debug, **options):
# Init and validate:
if fbidfile and (limit or options['random']):
raise CommandError("option --fbidfile incompatible with options --limit and --random")
self.campaign = relational.Campaign.objects.get(pk=campaign_id)
self.client_content = relational.ClientContent.objects.get(pk=client_content_id)
client = self.campaign.client
fb_app_id = client.fb_app_id
if client != self.client_content.client:
raise CommandError("Mismatched campaign and client content")
self.uri = server + reverse('targetshare:faces')
if not self.uri.startswith('http'):
self.uri = 'https://' + self.uri
self.timeout = options['timeout']
self.verbosity = int(options['verbosity'])
if fbidfile:
fh = sys.stdin if fbidfile == '-' else open(fbidfile)
# Grab fbids separated by any space (newline, space, tab):
fbids = chain.from_iterable(line.split() for line in fh.xreadlines())
keys = tuple({'fbid': int(fbid), 'appid': fb_app_id} for fbid in fbids)
tokens = dynamo.Token.items.batch_get(keys)
limit = len(keys)
else:
if limit is None:
limit = 100
if options['random']:
app_users = relational.UserClient.objects.filter(client__fb_app_id=fb_app_id)
random_fbids = app_users.values_list('fbid', flat=True).distinct().order_by('?')
# A lot of these might be no good when debugged, so get 10x
# TODO: Use scan and filter by expires?
# It'd be nice if boto allowed you to specify an (infinite)
# iterator of keys....
num_fbids = (limit * 10) if debug else limit
tokens = dynamo.Token.items.batch_get([{'fbid': fbid, 'appid': fb_app_id}
for fbid in random_fbids[:num_fbids].iterator()])
else:
tokens = dynamo.Token.items.filter(appid__eq=fb_app_id, expires__gt=epoch.utcnow())
if debug:
tokens = tokens.scan()
else:
tokens = tokens.scan(limit=limit)
tokens = tokens.iterable
if debug:
self._secrets = dict(relational.FBApp.objects.values_list('appid', 'secret').iterator())
tokens = self.debug_tokens(tokens, limit)
# Do one thread per primary for now:
queue = Queue()
results = []
for (count, token) in enumerate(tokens, 1):
thread = Thread(target=self.do_poll, args=(queue, results))
thread.setDaemon(True)
thread.start()
queue.put(token)
queue.join()
if len(results) < count:
self.perr("Completed polling of {} / {} tokens"
.format(len(results), count), 1)
for (fbid, result_time) in results:
self.stdout.write("{}: {:.1f}".format(fbid, result_time))
def crawl_user(fbid, retry_delay=0):
"""Enqueue crawl tasks for the user (`fbid`) and the users of his/her network."""
# Find a valid token for the user #
tokens = models.Token.items.query(fbid__eq=fbid)
# Iterate over user's tokens, starting with most recent:
for token in sorted(tokens, key=expires_safe, reverse=True):
# We expect values of "expires" to be optimistic, meaning we trust dates
# in the past, but must confirm dates in the future.
# (We sometimes set field optimistically; and, user can invalidate our
# token, throwing its actual expires to 0.)
if not token.expires or token.expires <= epoch.utcnow():
return # This, and any remaining, invalid
# Confirm token expiration (and validity)
secret = models.FBApp.objects.values_list('secret', flat=True).get(appid=token.appid)
try:
debug_result = facebook.client.debug_token(token.appid, secret, token.token)
debug_data = debug_result['data']
token_valid = debug_data['is_valid']
token_expires = debug_data['expires_at']
except (KeyError, IOError, RuntimeError) as exc:
# Facebook is being difficult; retry later (with increasing wait):
# (We would use self.request.retries rather than define
# retry_delay; but, we don't want to increase the countdown for
# *all* retries.)
crawl_user.retry((fbid, 2 * (retry_delay + 60)), {},
countdown=retry_delay, exc=exc)
# Update token, if needed; but, restart if another process has changed
# the token (meaning it may now refer to new value):
token.expires = token_expires
try:
token.partial_save()
except ConditionalCheckFailedException as exc:
# Token has changed since we loaded it; retry:
crawl_user.retry((fbid, retry_delay), {}, countdown=0, exc=exc)
if token_valid and token.expires > epoch.utcnow():
# We have our token, no lie!
break
else:
return # All tokens were invalid (and liars)
try:
edges = _bg_px4_crawl(token)
except urllib2.HTTPError as exc:
if 'invalid_token' in exc.headers.get('www-authenticate', ''):
return # dead token
raise
fb_sync_maps = _get_sync_maps(edges, token)
delay = 0
for (count, fbm) in enumerate(fb_sync_maps, 1):
if fbm.status == models.FBSyncMap.WAITING:
fbm.save_status(models.FBSyncMap.QUEUED)
initial_crawl.apply_async(
args=[fbm.fbid_primary, fbm.fbid_secondary],
countdown=delay
)
elif fbm.status == models.FBSyncMap.COMPLETE:
fbm.save_status(models.FBSyncMap.QUEUED)
incremental_crawl.apply_async(
args=[fbm.fbid_primary, fbm.fbid_secondary],
countdown=delay
)
delay += DELAY_INCREMENT if count % 100 == 0 else 0