def _parse_json_response(self, query, results): """ Parses Google Custom Search's JSON response and returns as an ifind Response. Args: query (ifind Query): object encapsulating details of a search query. results : requests library response object containing search results. Returns: ifind Response: object encapsulating a search request's results. Usage: Private method. """ response = Response(query.terms, query) content = json.loads(results.text) # The query object wasn't mutated earlier and the result type isn't passed to this function. # Check for a result_type or set it to default. result_type = query.result_type if not result_type: result_type = self.default_result_type # Check for a next page token. next_page_token = content.get(u'nextPageToken') if next_page_token: # A page token exists, create the URL which will fetch the next page response.next_page = "{}&pageToken={}".format(self._create_query_string(query), next_page_token) rank_counter = 1 if result_type == 'web' or not query.result_type: for result in content[u'items']: title = result[u'title'] url = result[u'link'] summary = result[u'snippet'] response.add_result(title=title, url=url, summary=summary, rank=rank_counter) rank_counter+=1 return response
def _parse_json_response(self, query, results): """ Parses Googleplus's JSON response and returns as an ifind Response. Args: query (ifind Query): object encapsulating details of a search query. results : requests library response object containing search results. Returns: ifind Response: object encapsulating a search request's results. Usage: Private method. """ response = Response(query.terms, query) content = json.loads(results.text) # The query object wasn't mutated earlier and the result type isn't passed to this function. # Check for a result_type or set it to default. result_type = query.result_type if not result_type: result_type = self.default_result_type # Check for a next page token. next_page_token = content.get(u'nextPageToken') if next_page_token: # A page token exists, create the URL which will fetch the next page response.next_page = "{}&pageToken={}".format(self._create_query_string(query), next_page_token) if result_type == 'people': # Build the ifind response for a people search for user in content[u'items']: name = user[u'displayName'] url = user[u'url'] imageurl = Googleplus._resize_image(user[u'image'][u'url']) summary = '' # Kwargs id = user[u'id'] # Add the result to the response response.add_result(title=name, url=url, summary=summary, imageurl=imageurl, id=id) elif result_type == 'activities': # Build the ifind response for an activity search for activity in content[u'items']: # The three dictionaries below are passed as keyword arguments to the result object activity_dict = { 'url': activity.get(u'url'), 'verb': activity.get(u'verb'), 'title': activity.get(u'title'), 'published': activity.get(u'published'), 'updated': activity.get(u'updated'), 'kind': activity.get(u'kind'), 'id': activity.get(u'id') } actor_dict = { 'display_name': activity.get(u'actor').get(u'displayName'), 'url': activity.get(u'actor').get(u'url'), 'image': activity.get(u'actor').get(u'image').get(u'url'), 'id': activity.get(u'actor').get(u'id') } object_dict = { 'type': activity.get(u'object').get(u'objectType'), 'content': activity.get(u'object').get(u'content').encode('utf-8'), 'url': activity.get(u'object').get(u'url'), } title = u"{} ({})".format(activity_dict.get('title'), activity_dict.get('verb')) url = activity_dict.get('url') summary = Googleplus._build_activity_summary(activity) imageurl = Googleplus._resize_image(actor_dict.get('image')) # Attachments is a list of dictionaries with keys: # u'objectType', u'displayName', u'content', u'url' and potentially nested dictionaries, # such as u'embed', u'image', u'thumbnails (list of dicts). attachments = activity.get(u'object').get(u'attachments') # Add the result to the response. response.add_result(title=title, url=url, summary=summary, imageurl=imageurl, actor=actor_dict, object=object_dict, activity=activity_dict, attachments=attachments) elif result_type == 'person_lookup': # Build the ifind response for a person lookup. No loop as the content is for a single person. title = content[u'displayName'] url = content[u'url'] imageurl = content[u'image'][u'url'] summary = Googleplus._build_person_summary(content) about_me = content.get(u'aboutMe') occupation = content.get(u'occupation') verified = content.get(u'verified') circled_count = content.get(u'circledByCount') is_plus_user = content.get(u'isPlusUser') birthday = content.get(u'birthday') bragging_rights = content.get(u'braggingRights') emails = content.get(u'emails') skills = content.get(u'skills') relationship_status = content.get(u'relationshipStatus') places_lived = content.get(u'placesLived') organizations = content.get(u'organizations') tagline = content.get(u'tagline') # Kwargs below person = {'about_me': about_me, 'occupation':occupation, 'verified': verified, 'emails': emails, 'circled_count': circled_count, 'is_plus_user': is_plus_user, 'birthday': birthday, 'bragging_rights': bragging_rights, 'skills': skills, 'relationship_status': relationship_status, 'places_lived': places_lived, 'organizations': organizations, 'tagline': tagline } # Add the result to the response. response.add_result(title=title, url=url, summary=summary, imageurl=imageurl, person=person) return response
def _parse_json_response(self, query, results): """ Parses Twitter's JSON response and returns as an ifind Response. Args: query (ifind Query): object encapsulating details of a search query. results : requests library response object containing search results. Returns: ifind Response: object encapsulating a search request's results. Usage: Private method. """ response = Response(query.terms, query) content = json.loads(results.text) # Check to see if there are more results. next_results = content[u'search_metadata'].get(u'next_results', '') if next_results: # Create a dictionary from the string found in u'next_results' params = next_results[1:] params = params.split('&') for index in range(len(params)): params[index] = params[index].split('=') param_dic = {} # At this point params looks like: [['someparam', 'somevalue'], ['someparam', 'somevalue']....] for lis in params: param_dic[lis[0]] = lis[1] # Set the next page URL in the response. response.next_page = self._create_query_string(query, search_params=param_dic) else: # No more results, set the flag in the response response.no_more_results = True for result in content[u'statuses']: text = result[u'text'] result_id = str(result[u'id']) # User dictionary user = {'user_id': result[u'user'][u'id_str'], 'profile_image': result.get(u'user').get(u'profile_image_url'), 'geo_enabled': result.get(u'user').get(u'geo_enabled'), 'description': result.get(u'user').get(u'description'), 'follower_count': result.get(u'user').get(u'followers_count'), 'protected': result.get(u'user').get(u'protected'), 'location': result.get(u'user').get(u'location'), 'utc_offset': result.get(u'user').get(u'utc_offset'), 'time_zone': result.get(u'user').get(u'time_zone'), 'name': result.get(u'user').get(u'name'), 'screen_name': result.get(u'user').get(u'screen_name'), 'member_since': result.get(u'user').get(u'created_at') } # TODO clean this up stamp = result[u'created_at'].split() # Created at in format: '01 Jan, 2014 @ 20:23' created_at = "{} {}, {} @ {}".format(stamp[2], stamp[1], stamp[5], stamp[3][:-3]) url = 'https://www.twitter.com/{0}/status/{1}'.format(user['user_id'], result_id) imageurl = user.get('profile_image') title = u"{} ({}) - {}".format(user['name'], user['screen_name'], created_at) # Kwargs below source = result.get(u'source') coordinates = result.get(u'coordinates') place = result.get(u'place') hashtags= result.get(u'entities').get(u'hashtags') user_info = user reply_to_screen_name = result.get(u'in_reply_to_screen_name') reply_to_userid = result.get(u'in_reply_to_user_id_str') reply_to_status = result.get(u'in_reply_to_status_id_str') tweet_id = result_id # List of links in the tweet. Each item in the list is a dictionary with keys: # u'url, u'indices', u'expanded_url, u'display_url' links = result.get(u'entities').get(u'urls') # List of media items in the tweet. Each item in the list is a dictionary with keys: # u'expanded_url', u'sizes', u'url', u'media_url_https', # u'id_str', u'indices', u'media_url', u'type', u'id', u'display_url' media = result.get(u'entities').get(u'media') # List of users mentioned in the tweet. Each item in the list is a dictionary with keys: # u'indices', 'u'screen_name', u'PSG_inside', u'id', u'name', u'id_str' user_mentions = result.get(u'entities').get(u'user_mentions') response.add_result(title=title, url=url, summary=text, imageurl=imageurl, stamp=stamp, user_info=user_info, media=media, links=links, user_mentions=user_mentions, source=source, coordinates=coordinates, place=place, hashtags=hashtags, reply_to_screen_name=reply_to_screen_name, reply_to_status=reply_to_status, reply_to_userid=reply_to_userid, tweet_id=tweet_id) if len(response) == query.top: break return response