Beispiel #1
0
    def _parse_json_response(self, query, results):
        """
        Parses Neutrinogeoaddress's JSON response and returns as an ifind Response.

        Args:
            query (ifind Query): object encapsulating details of a search query.
            results : requests library response object containing search results.

        Returns:
            ifind Response: object encapsulating a search request's results.

        Usage:
            Private method.
        """

        response = Response(query.terms, query)
        content = json.loads(results.text)

        # Results aren't paginated, no more to get.
        response.no_more_results = True

        url_base = 'https://www.google.co.uk/maps/place/'
        trans_table = maketrans(u' ', u'+')  # Switch spaces with + for the google maps url

        locations = content.get(u'locations')
        if locations:
        # There are results present, iterate over them.
            for loc in locations:
                # Kwargs below
                address = loc.get(u'address', '')
                latitude = loc.get(u'latitude', '')
                longitude = loc.get(u'longitude', '')
                country = loc.get(u'country', '')
                country_code = loc.get(u'country-code', '')
                city = loc.get(u'city', '')
                postcode = loc.get(u'postal-code', '')

                # The iframe_url must be placed in an iframe in order to render the map.
                if self.google_api_key:
                    iframe_url = self._build_iframe_url(address, trans_table)
                else:
                    iframe_url = None

                url = url_base + encode_symbols(address.encode('utf-8').translate(trans_table))
                text = Neutrinogeoaddress._build_summary(address, city, country, postcode, latitude, longitude)

                response.add_result(title=address, url=url, summary=text, imageurl=None,
                                    address=address, latitude=latitude, longitude=longitude, country=country,
                                    country_code=country_code, city=city, postcode=postcode, iframe_url=iframe_url)

        return response
Beispiel #2
0
    def _parse_json_response(query, results):
        """
        Parses Pipl's JSON response and returns as an ifind Response.

        Args:
            query (ifind Query): object encapsulating details of a search query.
            results : requests library response object containing search results.

        Returns:
            ifind Response: object encapsulating a search request's results.

        Usage:
            Private method.
        """

        response = Response(query.terms, query)
        content = json.loads(results.text)

        # Pipl only returns 20 results, so there are no more.
        response.no_more_results = True

        for record in content[u'records']:
            try:
                name = record[u'names'][0][u'display']
            except KeyError:
                name = record.get('source').get('url', "_no title_")
            url = record[u'source'][u'url']
            imageurl = None
            try:
                imageurl = record[u'images'][0][u'url']
            except:
                pass
            summary = Pipl._build_summary(record)

            # Kwargs below
            # Each keyword contains a list of (potentially empty) dictionary objects.
            usernames = record.get(u'usernames')
            addresses = record.get(u'addresses')
            relationships = record.get(u'relationships')
            jobs = record.get(u'jobs')
            educations = record.get(u'educations')
            tags = record.get(u'tags')


            response.add_result(title=name, url=url, summary=summary, imageurl=imageurl,
                                usernames=usernames, addresses=addresses, relationships=relationships,
                                jobs=jobs, educations=educations, tags=tags)

        return response
Beispiel #3
0
    def _parse_json_response(self, query, results):
        """
        Parses Twitter's JSON response and returns as an ifind Response.

        Args:
            query (ifind Query): object encapsulating details of a search query.
            results : requests library response object containing search results.

        Returns:
            ifind Response: object encapsulating a search request's results.

        Usage:
            Private method.

        """
        response = Response(query.terms, query)
        content = json.loads(results.text)

        # Check to see if there are more results.
        next_results = content[u'search_metadata'].get(u'next_results', '')
        if next_results:
            # Create a dictionary from the string found in u'next_results'
            params = next_results[1:]
            params = params.split('&')
            for index in range(len(params)):
                params[index] = params[index].split('=')
            param_dic = {}
            # At this point params looks like: [['someparam', 'somevalue'], ['someparam', 'somevalue']....]
            for lis in params:
                param_dic[lis[0]] = lis[1]

            # Set the next page URL in the response.
            response.next_page = self._create_query_string(query, search_params=param_dic)
        else:
            # No more results, set the flag in the response
            response.no_more_results = True

        for result in content[u'statuses']:

            text = result[u'text']
            result_id = str(result[u'id'])

            # User dictionary
            user = {'user_id': result[u'user'][u'id_str'],
                    'profile_image': result.get(u'user').get(u'profile_image_url'),
                    'geo_enabled': result.get(u'user').get(u'geo_enabled'),
                    'description': result.get(u'user').get(u'description'),
                    'follower_count': result.get(u'user').get(u'followers_count'),
                    'protected': result.get(u'user').get(u'protected'),
                    'location': result.get(u'user').get(u'location'),
                    'utc_offset': result.get(u'user').get(u'utc_offset'),
                    'time_zone': result.get(u'user').get(u'time_zone'),
                    'name': result.get(u'user').get(u'name'),
                    'screen_name': result.get(u'user').get(u'screen_name'),
                    'member_since': result.get(u'user').get(u'created_at')
            }

            # TODO clean this up
            stamp = result[u'created_at'].split()
            # Created at in format: '01 Jan, 2014 @ 20:23'
            created_at = "{} {}, {} @ {}".format(stamp[2], stamp[1], stamp[5], stamp[3][:-3])

            url = 'https://www.twitter.com/{0}/status/{1}'.format(user['user_id'], result_id)
            imageurl = user.get('profile_image')
            title = u"{} ({}) - {}".format(user['name'], user['screen_name'], created_at)

            # Kwargs below
            source = result.get(u'source')
            coordinates = result.get(u'coordinates')
            place = result.get(u'place')
            hashtags= result.get(u'entities').get(u'hashtags')
            user_info = user
            reply_to_screen_name = result.get(u'in_reply_to_screen_name')
            reply_to_userid = result.get(u'in_reply_to_user_id_str')
            reply_to_status = result.get(u'in_reply_to_status_id_str')
            tweet_id = result_id


            # List of links in the tweet. Each item in the list is a dictionary with keys:
            # u'url, u'indices', u'expanded_url, u'display_url'
            links = result.get(u'entities').get(u'urls')

            # List of media items in the tweet. Each item in the list is a dictionary with keys:
            # u'expanded_url', u'sizes', u'url', u'media_url_https',
            # u'id_str', u'indices', u'media_url', u'type', u'id', u'display_url'
            media = result.get(u'entities').get(u'media')

            # List of users mentioned in the tweet. Each item in the list is a dictionary with keys:
            # u'indices', 'u'screen_name', u'PSG_inside', u'id', u'name', u'id_str'
            user_mentions = result.get(u'entities').get(u'user_mentions')


            response.add_result(title=title, url=url, summary=text, imageurl=imageurl, stamp=stamp,
                                user_info=user_info, media=media, links=links, user_mentions=user_mentions,
                                source=source, coordinates=coordinates, place=place,
                                hashtags=hashtags,  reply_to_screen_name=reply_to_screen_name,
                                reply_to_status=reply_to_status, reply_to_userid=reply_to_userid, tweet_id=tweet_id)

            if len(response) == query.top:
                break

        return response
Beispiel #4
0
    def _parse_json_response(self, query, results):
        """
        Parses Companycheck's JSON response and returns as an ifind Response.

        Args:
            query (ifind Query): object encapsulating details of a search query.
            results : requests library response object containing search results.

        Returns:
            ifind Response: object encapsulating a search request's results.

        Usage:
            Private method.
        """

        response = Response(query.terms, query)
        content = json.loads(results.text)

        # The base URL to add the director or company number to, which provides the complete link.
        url_base = 'http://companycheck.co.uk/'

        # Since the object isn't mutated, set the default again if there is nothing present.
        result_type = query.result_type
        if not result_type:
            result_type = self.default_result_type


        # CompanyCheck returns all the results it has, it has no further results.
        response.no_more_results = True

        if result_type == 'company' or not result_type:
            # Create the ifind response for company searches
            for company in content:
                name = company[u'name']
                url =  url_base + 'company/' + str(company[u'number'])
                imageurl = None
                summary = Companycheck._build_company_summary(company)
                # Keyword args below
                number = company[u'number']
                country = company[u'country']
                address = company[u'address']
                sic = company[u'sic']
                status = company[u'status']
                # Add result object to the response
                response.add_result(title=name, url=url, summary=summary, imageurl=imageurl,
                                    number=number, country=country, address=address, sic=sic, status=status)

        elif result_type == 'director':
            # Create the ifind response for director searches
            for director in content:
                name = director[u'name']
                url =  url_base + 'director/' + str(director[u'number'])
                imageurl = None
                sum_dic = Companycheck._build_director_summary(director)
                summary = sum_dic.get('summary')
                # Keyword args below
                postcodes = sum_dic.get('postcode_list')
                number = director[u'number']
                # Add result object to the response
                response.add_result(title=name, url=url, summary=summary, imageurl=imageurl, postcodes=postcodes,
                                    number=number)

        return response