def resolvefb(): # in order to preserve the comment block at the top of the file, # copy it over into a new RtYamlList instance. We do this because # Python list instances can't hold other random attributes. import rtyaml updated_media = rtyaml.RtYamlList() if hasattr(media, '__initial_comment_block'): updated_media.__initial_comment_block = getattr( media, '__initial_comment_block') for m in media: social = m['social'] if ('facebook' in social and social['facebook']) and ('facebook_id' not in social): graph_url = "https://graph.facebook.com/%s" % social['facebook'] if re.match('\d+', social['facebook']): social['facebook_id'] = social['facebook'] print("Looking up graph username for %s" % social['facebook']) fbobj = requests.get(graph_url).json() if 'username' in fbobj: print("\tGot graph username of %s" % fbobj['username']) social['facebook'] = fbobj['username'] else: print("\tUnable to get graph username") else: try: print("Looking up graph ID for %s" % social['facebook']) fbobj = requests.get(graph_url).json() if 'id' in fbobj: print("\tGot graph ID of %s" % fbobj['id']) social['facebook_id'] = fbobj['id'] else: print("\tUnable to get graph ID") except: print("\tUnable to get graph ID for: %s" % social['facebook']) social['facebook_id'] = None updated_media.append(m) print("Saving social media...") save_data(updated_media, "legislators-social-media.yaml")
def resolveig(): # in order to preserve the comment block at the top of the file, # copy it over into a new RtYamlList instance. We do this because # Python list instances can't hold other random attributes. import rtyaml updated_media = rtyaml.RtYamlList() if hasattr(media, '__initial_comment_block'): updated_media.__initial_comment_block = getattr( media, '__initial_comment_block') client_id_file = open('cache/instagram_client_id', 'r') client_id = client_id_file.read() bioguide = utils.flags().get('bioguide', None) for m in media: if bioguide and (m['id']['bioguide'] != bioguide): updated_media.append(m) continue social = m['social'] if 'instagram' not in social and 'instagram_id' not in social: updated_media.append(m) continue instagram_handle = social['instagram'] query_url = "https://api.instagram.com/v1/users/search?q={query}&client_id={client_id}".format( query=instagram_handle, client_id=client_id) instagram_user_search = requests.get(query_url).json() for user in instagram_user_search['data']: time.sleep(0.5) if user['username'] == instagram_handle: m['social']['instagram_id'] = int(user['id']) print( "matched instagram_id {instagram_id} to {instagram_handle}" .format(instagram_id=social['instagram_id'], instagram_handle=instagram_handle)) updated_media.append(m) save_data(updated_media, "legislators-social-media.yaml")
def resolvetw(): """ Does two batch lookups: 1. All entries with `twitter_id`: Checks to see if the corresponding Twitter profile has the same screen_name as found in the entry's `twitter`. If not, the `twitter` value is updated. 2. All entries with `twitter` (but not `twitter_id`): fetches the corresponding Twitter profile by screen_name and inserts ID. If no profile is found, the `twitter` value is deleted. Note: cache/twitter_client_id must be a formatted JSON dict: { "consumer_secret": "xyz", "access_token": "abc", "access_token_secret": "def", "consumer_key": "jk" } """ import rtyaml from social.twitter import get_api, fetch_profiles updated_media = rtyaml.RtYamlList() if hasattr(media, '__initial_comment_block'): updated_media.__initial_comment_block = getattr( media, '__initial_comment_block') client_id_file = open('cache/twitter_client_id', 'r') _c = json.load(client_id_file) api = get_api(_c['access_token'], _c['access_token_secret'], _c['consumer_key'], _c['consumer_secret']) bioguide = utils.flags().get('bioguide', None) lookups = { 'screen_names': [], 'ids': [] } # store members that have `twitter` or `twitter_id` info for m in media: # we start with appending to updated_media so that we keep the same order of entries # as found in the loaded file updated_media.append(m) if bioguide and (m['id']['bioguide'] != bioguide): continue social = m['social'] # now we add entries to either the `ids` or the `screen_names` list to batch lookup if 'twitter_id' in social: # add to the queue to be batched-looked-up lookups['ids'].append(m) # append elif 'twitter' in social: lookups['screen_names'].append(m) ####################################### # perform Twitter batch lookup for ids: if lookups['screen_names']: arr = lookups['screen_names'] print("Looking up Twitter ids for", len(arr), "names.") tw_names = [m['social']['twitter'] for m in arr] tw_profiles = fetch_profiles(api, screen_names=tw_names) for m in arr: social = m['social'] # find profile that corresponds to a given screen_name twitter_handle = social['twitter'] twp = next( (p for p in tw_profiles if p['screen_name'].lower() == twitter_handle.lower()), None) if twp: m['social']['twitter_id'] = int(twp['id']) print("Matched twitter_id `%s` to `%s`" % (social['twitter_id'], twitter_handle)) else: # Remove errant Twitter entry for now print("No Twitter user profile for:", twitter_handle) m['social'].pop('twitter') print("\t ! removing Twitter handle:", twitter_handle) ########################################## # perform Twitter batch lookup for names by id, to update any renamings: if lookups['ids']: arr = lookups['ids'] print("Looking up Twitter screen_names for", len(arr), "ids.") tw_ids = [m['social']['twitter_id'] for m in arr] tw_profiles = fetch_profiles(api, ids=tw_ids) any_renames_needed = False for m in arr: social = m['social'] # find profile that corresponds to a given screen_name t_id = social['twitter_id'] t_name = social.get('twitter') twp = next((p for p in tw_profiles if int(p['id']) == t_id), None) if twp: # Be silent if there is no change to screen name if t_name and (twp['screen_name'].lower() == t_name.lower()): pass else: any_renames_needed = True m['social']['twitter'] = twp['screen_name'] print("For twitter_id `%s`, renamed `%s` to `%s`" % (t_id, t_name, m['social']['twitter'])) else: # No entry found for this twitter id print("No Twitter user profile for %s, %s" % (t_id, t_name)) m['social'].pop('twitter_id') print("\t ! removing Twitter id:", t_id) if not any_renames_needed: print("No renames needed") # all done with Twitter save_data(updated_media, "legislators-social-media.yaml")