def phid_to_name(phid): if phid is None: return None if cache.has(phid): return cache.get(phid) phab = Phabricator() data = phab.phid.lookup(names=[ phid, ]) name = data[phid]['name'] cache.set(phid, name) return name
def lookup_many(user_ids): """ """ method_name = 'twitter.users.lookup' new_users = [] cached_users = [] for user in user_ids: if cache.has(method_name,user): cached_users.append(user) else: new_users.append(user) data = {} pp("Retrieving user metadata from cache") for user in cached_users: data[user] = lookup(user) if type(lookup(user)) is list: pp('Watch out for %s' % user) pp("Using API to lookup data from %d new users" % len(new_users)) for user_slice in [new_users[x:x+100] for x in xrange(0,len(new_users),100)]: query = ",".join([str(x) for x in user_slice]) logger.debug("Query is %s" % query) try: pp("Requesting user slice...") metadatas = call_api(twitter.users.lookup, {id_or_sn(query):query}) for user_data in metadatas: screen_name = user_data['screen_name'] cache.write(method_name,screen_name,user_data) data[screen_name] = user_data pp("Successfully wrote slice to cache") except TwitterHTTPError as e: ## is there something more graceful to be done with 404's here? ## maybe...if we handle unauthorized requests better. print e logger.error(e) return data
def lookup_many(user_ids): """ """ method_name = 'twitter.users.lookup' new_users = [] cached_users = [] for user in user_ids: if cache.has(method_name, user): cached_users.append(user) else: new_users.append(user) data = {} pp("Retrieving user metadata from cache") for user in cached_users: data[user] = lookup(user) if type(lookup(user)) is list: pp('Watch out for %s' % user) pp("Using API to lookup data from %d new users" % len(new_users)) for user_slice in [ new_users[x:x + 100] for x in xrange(0, len(new_users), 100) ]: query = ",".join([str(x) for x in user_slice]) logger.debug("Query is %s" % query) try: pp("Requesting user slice...") metadatas = call_api(twitter.users.lookup, {id_or_sn(query): query}) for user_data in metadatas: screen_name = user_data['screen_name'] cache.write(method_name, screen_name, user_data) data[screen_name] = user_data pp("Successfully wrote slice to cache") except TwitterHTTPError as e: ## is there something more graceful to be done with 404's here? ## maybe...if we handle unauthorized requests better. print e logger.error(e) return data
def call_api_with_cache(user_id, method, method_name,parameters={}): # first check the cache if cache.has(method_name,user_id): logger.debug("%s %s exists in cache." % (method_name, user_id)) return cache.read(method_name,user_id) else: # if not in cache call the API logger.debug("%s %s does not exists in cache. Will retrieve it from web." % (method_name, user_id)) try: data = call_api(method, dict([(id_or_sn(user_id),user_id)] + parameters.items())) cache.write(method_name,user_id,data) return data except TwitterHTTPError as e: logger.error(e) #hack to prevent crawling this return {'error': e}
def call_api_with_cache(user_id, method, method_name, parameters={}): # first check the cache if cache.has(method_name, user_id): logger.debug("%s %s exists in cache." % (method_name, user_id)) return cache.read(method_name, user_id) else: # if not in cache call the API logger.debug( "%s %s does not exists in cache. Will retrieve it from web." % (method_name, user_id)) try: data = call_api( method, dict([(id_or_sn(user_id), user_id)] + parameters.items())) cache.write(method_name, user_id, data) return data except TwitterHTTPError as e: logger.error(e) #hack to prevent crawling this return {'error': e}
def lookup_many(user_ids): """ """ method_name = 'twitter.users.lookup' new_users = [] cached_users = [] for user in user_ids: if cache.has(method_name,user): cached_users.append(user) else: new_users.append(user) data = {} for user in cached_users: data[user] = lookup(user) if type(lookup(user)) is list: pp('Watch out for %s' % user) for user_slice in [new_users[x:x+100] for x in xrange(0,len(new_users),100)]: query = ",".join([str(x) for x in user_slice]) logger.debug("Query is %s" % query) try: metadatas = call_api(twitter.users.lookup, {id_or_sn(query):query}) for user_data in metadatas: screen_name = user_data['screen_name'] cache.write(method_name,screen_name,user_data) data[screen_name] = user_data except TwitterHTTPError as e: print e logger.error(e) return data