def _parse_json_response(self, query, results): """ Parses Neutrinogeoaddress's JSON response and returns as an ifind Response. Args: query (ifind Query): object encapsulating details of a search query. results : requests library response object containing search results. Returns: ifind Response: object encapsulating a search request's results. Usage: Private method. """ response = Response(query.terms, query) content = json.loads(results.text) # Results aren't paginated, no more to get. response.no_more_results = True url_base = 'https://www.google.co.uk/maps/place/' trans_table = maketrans(u' ', u'+') # Switch spaces with + for the google maps url locations = content.get(u'locations') if locations: # There are results present, iterate over them. for loc in locations: # Kwargs below address = loc.get(u'address', '') latitude = loc.get(u'latitude', '') longitude = loc.get(u'longitude', '') country = loc.get(u'country', '') country_code = loc.get(u'country-code', '') city = loc.get(u'city', '') postcode = loc.get(u'postal-code', '') # The iframe_url must be placed in an iframe in order to render the map. if self.google_api_key: iframe_url = self._build_iframe_url(address, trans_table) else: iframe_url = None url = url_base + encode_symbols(address.encode('utf-8').translate(trans_table)) text = Neutrinogeoaddress._build_summary(address, city, country, postcode, latitude, longitude) response.add_result(title=address, url=url, summary=text, imageurl=None, address=address, latitude=latitude, longitude=longitude, country=country, country_code=country_code, city=city, postcode=postcode, iframe_url=iframe_url) return response
def _parse_json_response(query, results): """ Parses Pipl's JSON response and returns as an ifind Response. Args: query (ifind Query): object encapsulating details of a search query. results : requests library response object containing search results. Returns: ifind Response: object encapsulating a search request's results. Usage: Private method. """ response = Response(query.terms, query) content = json.loads(results.text) # Pipl only returns 20 results, so there are no more. response.no_more_results = True for record in content[u'records']: try: name = record[u'names'][0][u'display'] except KeyError: name = record.get('source').get('url', "_no title_") url = record[u'source'][u'url'] imageurl = None try: imageurl = record[u'images'][0][u'url'] except: pass summary = Pipl._build_summary(record) # Kwargs below # Each keyword contains a list of (potentially empty) dictionary objects. usernames = record.get(u'usernames') addresses = record.get(u'addresses') relationships = record.get(u'relationships') jobs = record.get(u'jobs') educations = record.get(u'educations') tags = record.get(u'tags') response.add_result(title=name, url=url, summary=summary, imageurl=imageurl, usernames=usernames, addresses=addresses, relationships=relationships, jobs=jobs, educations=educations, tags=tags) return response
def _parse_json_response(self, query, results): """ Parses Twitter's JSON response and returns as an ifind Response. Args: query (ifind Query): object encapsulating details of a search query. results : requests library response object containing search results. Returns: ifind Response: object encapsulating a search request's results. Usage: Private method. """ response = Response(query.terms, query) content = json.loads(results.text) # Check to see if there are more results. next_results = content[u'search_metadata'].get(u'next_results', '') if next_results: # Create a dictionary from the string found in u'next_results' params = next_results[1:] params = params.split('&') for index in range(len(params)): params[index] = params[index].split('=') param_dic = {} # At this point params looks like: [['someparam', 'somevalue'], ['someparam', 'somevalue']....] for lis in params: param_dic[lis[0]] = lis[1] # Set the next page URL in the response. response.next_page = self._create_query_string(query, search_params=param_dic) else: # No more results, set the flag in the response response.no_more_results = True for result in content[u'statuses']: text = result[u'text'] result_id = str(result[u'id']) # User dictionary user = {'user_id': result[u'user'][u'id_str'], 'profile_image': result.get(u'user').get(u'profile_image_url'), 'geo_enabled': result.get(u'user').get(u'geo_enabled'), 'description': result.get(u'user').get(u'description'), 'follower_count': result.get(u'user').get(u'followers_count'), 'protected': result.get(u'user').get(u'protected'), 'location': result.get(u'user').get(u'location'), 'utc_offset': result.get(u'user').get(u'utc_offset'), 'time_zone': result.get(u'user').get(u'time_zone'), 'name': result.get(u'user').get(u'name'), 'screen_name': result.get(u'user').get(u'screen_name'), 'member_since': result.get(u'user').get(u'created_at') } # TODO clean this up stamp = result[u'created_at'].split() # Created at in format: '01 Jan, 2014 @ 20:23' created_at = "{} {}, {} @ {}".format(stamp[2], stamp[1], stamp[5], stamp[3][:-3]) url = 'https://www.twitter.com/{0}/status/{1}'.format(user['user_id'], result_id) imageurl = user.get('profile_image') title = u"{} ({}) - {}".format(user['name'], user['screen_name'], created_at) # Kwargs below source = result.get(u'source') coordinates = result.get(u'coordinates') place = result.get(u'place') hashtags= result.get(u'entities').get(u'hashtags') user_info = user reply_to_screen_name = result.get(u'in_reply_to_screen_name') reply_to_userid = result.get(u'in_reply_to_user_id_str') reply_to_status = result.get(u'in_reply_to_status_id_str') tweet_id = result_id # List of links in the tweet. Each item in the list is a dictionary with keys: # u'url, u'indices', u'expanded_url, u'display_url' links = result.get(u'entities').get(u'urls') # List of media items in the tweet. Each item in the list is a dictionary with keys: # u'expanded_url', u'sizes', u'url', u'media_url_https', # u'id_str', u'indices', u'media_url', u'type', u'id', u'display_url' media = result.get(u'entities').get(u'media') # List of users mentioned in the tweet. Each item in the list is a dictionary with keys: # u'indices', 'u'screen_name', u'PSG_inside', u'id', u'name', u'id_str' user_mentions = result.get(u'entities').get(u'user_mentions') response.add_result(title=title, url=url, summary=text, imageurl=imageurl, stamp=stamp, user_info=user_info, media=media, links=links, user_mentions=user_mentions, source=source, coordinates=coordinates, place=place, hashtags=hashtags, reply_to_screen_name=reply_to_screen_name, reply_to_status=reply_to_status, reply_to_userid=reply_to_userid, tweet_id=tweet_id) if len(response) == query.top: break return response
def _parse_json_response(self, query, results): """ Parses Companycheck's JSON response and returns as an ifind Response. Args: query (ifind Query): object encapsulating details of a search query. results : requests library response object containing search results. Returns: ifind Response: object encapsulating a search request's results. Usage: Private method. """ response = Response(query.terms, query) content = json.loads(results.text) # The base URL to add the director or company number to, which provides the complete link. url_base = 'http://companycheck.co.uk/' # Since the object isn't mutated, set the default again if there is nothing present. result_type = query.result_type if not result_type: result_type = self.default_result_type # CompanyCheck returns all the results it has, it has no further results. response.no_more_results = True if result_type == 'company' or not result_type: # Create the ifind response for company searches for company in content: name = company[u'name'] url = url_base + 'company/' + str(company[u'number']) imageurl = None summary = Companycheck._build_company_summary(company) # Keyword args below number = company[u'number'] country = company[u'country'] address = company[u'address'] sic = company[u'sic'] status = company[u'status'] # Add result object to the response response.add_result(title=name, url=url, summary=summary, imageurl=imageurl, number=number, country=country, address=address, sic=sic, status=status) elif result_type == 'director': # Create the ifind response for director searches for director in content: name = director[u'name'] url = url_base + 'director/' + str(director[u'number']) imageurl = None sum_dic = Companycheck._build_director_summary(director) summary = sum_dic.get('summary') # Keyword args below postcodes = sum_dic.get('postcode_list') number = director[u'number'] # Add result object to the response response.add_result(title=name, url=url, summary=summary, imageurl=imageurl, postcodes=postcodes, number=number) return response