Example #1
0
def request_followers(application_id: str) -> None:
    """Request the followers for all main accounts. Uses the fetcher service to request followers in parallel"""
    follower_count = 200
    request_limit = 15
    request_chunks = 5
    with engine.connect() as connection:
        requests_left = request_limit

        while requests_left > 0:
            try:
                limit = min(requests_left, request_chunks)
                main_users = list(
                    models.account.select_main_with_followers(
                        application_id, SOURCES['TWITTER'], limit, connection))

                if not main_users:
                    return

                requests_left -= len(main_users)
                request = create_timeline_payload(application_id,
                                                  follower_count, main_users)
                followers_response = urlopen(request)
                followers_response_code = followers_response.getcode()

                if followers_response_code == 200:
                    response = json.loads(read_response(followers_response))

                    for followers in response:
                        result = followers['result']
                        name = followers['screenName']
                        users = result['users']
                        cursor = result['next_cursor']
                        user_followers = [
                            user['screen_name'] for user in users
                            if not user['protected']
                        ]
                        models.account_relationship.insert_multiple(
                            application_id, name, user_followers,
                            SOURCES['TWITTER'], cursor, connection)
                else:
                    raise IOError('Invalid response from Fetcher Service')

            except (ValueError, HTTPException) as e:
                print(e)
            except HTTPError as e:
                print(e.read())
Example #2
0
    def handle_request(self, expect_cmd=None):
        try:
            sreq = util.read_response(self.sock)
        except UnicodeDecodeError:
            logger.warn('%r: client is sending garbage!', self.addr, exc_info=True)
            raise AWIPRequestInvalid('bad encoded data')

        logger.debug('%r: Got request: %r', self.addr, sreq)
        if sreq is None:
            raise AWIPClientDisconnected
        req = Request(sreq)

        if expect_cmd and req.cmd != expect_cmd:
            raise AWIPRequestInvalid('Please set mode first')

        method = 'handle_cmd_%s' % req.cmd
        logger.info('%r: %s', self.addr, method)
        f = getattr(self, method, None)
        if f is None:
            raise AWIPClientError('No such request command')
        d = f(req)
        if 'status' not in d:
            d['status'] = 'ok'
        self.reply(d)
Example #3
0
 def do_cmd(self, cmd, **other):
     d = { 'cmd': cmd }
     d.update(other)
     write_response(self.sock, d)
     return read_response(self.sock)
Example #4
0
def request_user_timelines(application_id: str,
                           is_main=True,
                           request_limit=1500) -> int:
    """Requests the twitter timeline for all accounts in the database. Either main or not. Uses the fetcher service."""
    request_chunks = 20
    with engine.connect() as connection:
        requests_left = request_limit
        while requests_left > 0:
            limit = min(requests_left, request_chunks)
            account_ids = map_to_attribute(
                'id',
                account_model.select_multiple_incomplete(application_id,
                                                         SOURCES['TWITTER'],
                                                         connection,
                                                         ismain=is_main,
                                                         limit=limit))
            # There are no more accounts that need to be fetched
            if not account_ids:
                return requests_left

            requests_left -= len(account_ids)

            accounts_timeline = account_model.select_oldest_timelines(
                account_ids, connection)
            timeline_payload = create_timeline_payload(accounts_timeline)
            request = create_post_request(TWITTER_TIMELINE, {
                'applicationId': application_id,
                'accounts': timeline_payload
            })

            try:
                timeline_response = urlopen(request)
                timeline_response_code = timeline_response.getcode()

                if timeline_response_code != 200:
                    continue
                raw_response = read_response(timeline_response)
                response = json.loads(raw_response)

                for account in response['success']:
                    with connection.begin():
                        name = account['screenName']
                        timeline = account['timeline']
                        statuses = []
                        if isinstance(timeline, str):
                            continue
                        for status in timeline:
                            statuses.append({
                                'text':
                                status['text'],
                                'id':
                                status['id'],
                                'date':
                                convert_twitter_date(status['created_at'])
                            })

                        if len(statuses) > 1:
                            account_timeline.insert_multiple(
                                application_id, name, SOURCES['TWITTER'],
                                statuses[1:], connection)
                        else:
                            account_id = account_model.select_one_id(
                                application_id, name, SOURCES['TWITTER'],
                                connection)
                            account_model.update_one_iscomplete(
                                account_id, True, connection)
                for res in response['errors']:
                    with connection.begin():
                        name = res['screenName']
                        status = res['status']
                        if status == 404:
                            account_id = account_model.select_one_id(
                                application_id, name, SOURCES['TWITTER'],
                                connection)
                            if not account_id:
                                continue
                            account = account_model.select_one_by_id(
                                account_id, connection)
                            # Delete account from accounts Table if it has been removed from Twitter
                            if not account['isMain']:
                                account_rel_model.delete_follower_account_rel(
                                    account_id, connection)
                                account_model.delete_one_by_id(
                                    account_id, connection)
            except (ValueError, HTTPException) as e:
                print(e)
            except HTTPError as e:
                print(e.read())

        return requests_left