def request_followers(application_id: str) -> None:
"""Request the followers for all main accounts. Uses the fetcher service to request followers in parallel"""
follower_count = 200
request_limit = 15
request_chunks = 5
with engine.connect() as connection:
requests_left = request_limit
while requests_left > 0:
try:
limit = min(requests_left, request_chunks)
main_users = list(
models.account.select_main_with_followers(
application_id, SOURCES['TWITTER'], limit, connection))
if not main_users:
return
requests_left -= len(main_users)
request = create_timeline_payload(application_id,
follower_count, main_users)
followers_response = urlopen(request)
followers_response_code = followers_response.getcode()
if followers_response_code == 200:
response = json.loads(read_response(followers_response))
for followers in response:
result = followers['result']
name = followers['screenName']
users = result['users']
cursor = result['next_cursor']
user_followers = [
user['screen_name'] for user in users
if not user['protected']
]
models.account_relationship.insert_multiple(
application_id, name, user_followers,
SOURCES['TWITTER'], cursor, connection)
else:
raise IOError('Invalid response from Fetcher Service')
except (ValueError, HTTPException) as e:
print(e)
except HTTPError as e:
print(e.read())
def handle_request(self, expect_cmd=None):
try:
sreq = util.read_response(self.sock)
except UnicodeDecodeError:
logger.warn('%r: client is sending garbage!', self.addr, exc_info=True)
raise AWIPRequestInvalid('bad encoded data')
logger.debug('%r: Got request: %r', self.addr, sreq)
if sreq is None:
raise AWIPClientDisconnected
req = Request(sreq)
if expect_cmd and req.cmd != expect_cmd:
raise AWIPRequestInvalid('Please set mode first')
method = 'handle_cmd_%s' % req.cmd
logger.info('%r: %s', self.addr, method)
f = getattr(self, method, None)
if f is None:
raise AWIPClientError('No such request command')
d = f(req)
if 'status' not in d:
d['status'] = 'ok'
self.reply(d)
def do_cmd(self, cmd, **other):
d = { 'cmd': cmd }
d.update(other)
write_response(self.sock, d)
return read_response(self.sock)
def request_user_timelines(application_id: str,
is_main=True,
request_limit=1500) -> int:
"""Requests the twitter timeline for all accounts in the database. Either main or not. Uses the fetcher service."""
request_chunks = 20
with engine.connect() as connection:
requests_left = request_limit
while requests_left > 0:
limit = min(requests_left, request_chunks)
account_ids = map_to_attribute(
'id',
account_model.select_multiple_incomplete(application_id,
SOURCES['TWITTER'],
connection,
ismain=is_main,
limit=limit))
# There are no more accounts that need to be fetched
if not account_ids:
return requests_left
requests_left -= len(account_ids)
accounts_timeline = account_model.select_oldest_timelines(
account_ids, connection)
timeline_payload = create_timeline_payload(accounts_timeline)
request = create_post_request(TWITTER_TIMELINE, {
'applicationId': application_id,
'accounts': timeline_payload
})
try:
timeline_response = urlopen(request)
timeline_response_code = timeline_response.getcode()
if timeline_response_code != 200:
continue
raw_response = read_response(timeline_response)
response = json.loads(raw_response)
for account in response['success']:
with connection.begin():
name = account['screenName']
timeline = account['timeline']
statuses = []
if isinstance(timeline, str):
continue
for status in timeline:
statuses.append({
'text':
status['text'],
'id':
status['id'],
'date':
convert_twitter_date(status['created_at'])
})
if len(statuses) > 1:
account_timeline.insert_multiple(
application_id, name, SOURCES['TWITTER'],
statuses[1:], connection)
else:
account_id = account_model.select_one_id(
application_id, name, SOURCES['TWITTER'],
connection)
account_model.update_one_iscomplete(
account_id, True, connection)
for res in response['errors']:
with connection.begin():
name = res['screenName']
status = res['status']
if status == 404:
account_id = account_model.select_one_id(
application_id, name, SOURCES['TWITTER'],
connection)
if not account_id:
continue
account = account_model.select_one_by_id(
account_id, connection)
# Delete account from accounts Table if it has been removed from Twitter
if not account['isMain']:
account_rel_model.delete_follower_account_rel(
account_id, connection)
account_model.delete_one_by_id(
account_id, connection)
except (ValueError, HTTPException) as e:
print(e)
except HTTPError as e:
print(e.read())
return requests_left