def phid_to_name(phid):
if phid is None:
return None
if cache.has(phid):
return cache.get(phid)
phab = Phabricator()
data = phab.phid.lookup(names=[
phid,
])
name = data[phid]['name']
cache.set(phid, name)
return name
def lookup_many(user_ids):
""" """
method_name = 'twitter.users.lookup'
new_users = []
cached_users = []
for user in user_ids:
if cache.has(method_name,user):
cached_users.append(user)
else:
new_users.append(user)
data = {}
pp("Retrieving user metadata from cache")
for user in cached_users:
data[user] = lookup(user)
if type(lookup(user)) is list:
pp('Watch out for %s' % user)
pp("Using API to lookup data from %d new users" % len(new_users))
for user_slice in [new_users[x:x+100]
for x
in xrange(0,len(new_users),100)]:
query = ",".join([str(x) for x in user_slice])
logger.debug("Query is %s" % query)
try:
pp("Requesting user slice...")
metadatas = call_api(twitter.users.lookup,
{id_or_sn(query):query})
for user_data in metadatas:
screen_name = user_data['screen_name']
cache.write(method_name,screen_name,user_data)
data[screen_name] = user_data
pp("Successfully wrote slice to cache")
except TwitterHTTPError as e:
## is there something more graceful to be done with 404's here?
## maybe...if we handle unauthorized requests better.
print e
logger.error(e)
return data
def lookup_many(user_ids):
""" """
method_name = 'twitter.users.lookup'
new_users = []
cached_users = []
for user in user_ids:
if cache.has(method_name, user):
cached_users.append(user)
else:
new_users.append(user)
data = {}
pp("Retrieving user metadata from cache")
for user in cached_users:
data[user] = lookup(user)
if type(lookup(user)) is list:
pp('Watch out for %s' % user)
pp("Using API to lookup data from %d new users" % len(new_users))
for user_slice in [
new_users[x:x + 100] for x in xrange(0, len(new_users), 100)
]:
query = ",".join([str(x) for x in user_slice])
logger.debug("Query is %s" % query)
try:
pp("Requesting user slice...")
metadatas = call_api(twitter.users.lookup,
{id_or_sn(query): query})
for user_data in metadatas:
screen_name = user_data['screen_name']
cache.write(method_name, screen_name, user_data)
data[screen_name] = user_data
pp("Successfully wrote slice to cache")
except TwitterHTTPError as e:
## is there something more graceful to be done with 404's here?
## maybe...if we handle unauthorized requests better.
print e
logger.error(e)
return data
def call_api_with_cache(user_id, method, method_name,parameters={}):
# first check the cache
if cache.has(method_name,user_id):
logger.debug("%s %s exists in cache." % (method_name, user_id))
return cache.read(method_name,user_id)
else:
# if not in cache call the API
logger.debug("%s %s does not exists in cache. Will retrieve it from web." % (method_name, user_id))
try:
data = call_api(method,
dict([(id_or_sn(user_id),user_id)]
+ parameters.items()))
cache.write(method_name,user_id,data)
return data
except TwitterHTTPError as e:
logger.error(e)
#hack to prevent crawling this
return {'error': e}
def call_api_with_cache(user_id, method, method_name, parameters={}):
# first check the cache
if cache.has(method_name, user_id):
logger.debug("%s %s exists in cache." % (method_name, user_id))
return cache.read(method_name, user_id)
else:
# if not in cache call the API
logger.debug(
"%s %s does not exists in cache. Will retrieve it from web." %
(method_name, user_id))
try:
data = call_api(
method,
dict([(id_or_sn(user_id), user_id)] + parameters.items()))
cache.write(method_name, user_id, data)
return data
except TwitterHTTPError as e:
logger.error(e)
#hack to prevent crawling this
return {'error': e}
def lookup_many(user_ids):
""" """
method_name = 'twitter.users.lookup'
new_users = []
cached_users = []
for user in user_ids:
if cache.has(method_name,user):
cached_users.append(user)
else:
new_users.append(user)
data = {}
for user in cached_users:
data[user] = lookup(user)
if type(lookup(user)) is list:
pp('Watch out for %s' % user)
for user_slice in [new_users[x:x+100]
for x
in xrange(0,len(new_users),100)]:
query = ",".join([str(x) for x in user_slice])
logger.debug("Query is %s" % query)
try:
metadatas = call_api(twitter.users.lookup,
{id_or_sn(query):query})
for user_data in metadatas:
screen_name = user_data['screen_name']
cache.write(method_name,screen_name,user_data)
data[screen_name] = user_data
except TwitterHTTPError as e:
print e
logger.error(e)
return data
