def _find_and_respond_to_twitter_events(self) -> None:
"""Looks for TwitterEvent objects that have not yet been responded to and
begins the process of creating a response. Additionally, failed events are
rerun to provide a correct response, particularly useful in cases where
external apis are down for maintenance.
"""
interval = (
self.PRODUCTION_APP_RATE_LIMITING_INTERVAL_IN_SECONDS if self._is_production()
else self.DEVELOPMENT_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS)
self._events_iteration += 1
LOG.debug(
f'Looking up twitter events on iteration {self._events_iteration}')
# set up timer
twitter_events_thread = threading.Timer(
interval, self._find_and_respond_to_twitter_events)
self._lookup_threads.append(twitter_events_thread)
# start timer
twitter_events_thread.start()
try:
new_events: list[TwitterEvent] = TwitterEvent.get_all_by(
is_duplicate=False,
responded_to=False,
response_in_progress=False)
LOG.debug(f'new events: {new_events}')
failed_events: list[TwitterEvent] = TwitterEvent.get_all_by(
is_duplicate=False,
error_on_lookup=True,
responded_to=True,
response_in_progress=False)
failed_events_that_need_response: list[TwitterEvent] = self._filter_failed_twitter_events(failed_events)
events_to_respond_to: [list[TwitterEvent]] = new_events + failed_events_that_need_response
LOG.debug(f'events to respond to: {events_to_respond_to}')
for event in events_to_respond_to:
self._process_twitter_event(event=event)
except Exception as e:
LOG.error(e)
LOG.error(str(e))
LOG.error(e.args)
logging.exception("stack trace")
finally:
TwitterEvent.query.session.close()
def test_find_and_respond_to_twitter_events(self, twitter_event_mock):
db_id = 1
random_id = random.randint(10000000000000000000, 20000000000000000000)
event_type = 'status'
user_handle = 'bdhowald'
user_id = random.randint(1000000000, 2000000000)
event_text = '@HowsMyDrivingNY abc1234:ny'
timestamp = random.randint(1500000000000, 1700000000000)
in_reply_to_message_id = random.randint(
10000000000000000000, 20000000000000000000)
location = 'Queens, NY'
responded_to = 0
user_mentions = '@HowsMyDrivingNY'
twitter_event = TwitterEvent(
id=db_id,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
location=location,
responded_to=responded_to,
user_handle=user_handle,
user_id=user_id,
user_mentions=user_mentions)
lookup_request = AccountActivityAPIDirectMessage(
message=TwitterEvent(
id=1,
created_at=timestamp,
event_id=random_id,
event_text='@howsmydrivingny ny:hme6483',
event_type='direct_message',
user_handle=user_handle,
user_id=30139847),
message_source='api')
twitter_event_mock.get_all_by.return_value = [twitter_event]
twitter_event_mock.query.filter_by().filter().count.return_value = 0
initiate_reply_mock = MagicMock(name='initiate_reply')
self.tweeter.aggregator.initiate_reply = initiate_reply_mock
build_reply_data_mock = MagicMock(name='build_reply_data')
build_reply_data_mock.return_value = lookup_request
self.tweeter.reply_argument_builder.build_reply_data = build_reply_data_mock
self.tweeter._find_and_respond_to_twitter_events()
self.tweeter.aggregator.initiate_reply.assert_called_with(
lookup_request=lookup_request)
def _add_twitter_events_for_missed_statuses(self, messages: List[tweepy.Status]):
"""Creates TwitterEvent objects when the Account Activity API fails to send us
status events via webhooks to have them created by the HowsMyDrivingNY API.
:param messages: List[tweepy.Status]: The statuses returned via the Twitter
Search API via Tweepy.
"""
undetected_messages = 0
for message in messages:
existing_event: Optional[TwitterEvent] = TwitterEvent.query.filter(TwitterEvent.event_id == message.id).first()
if not existing_event and message.user.id != HMDNY_TWITTER_USER_ID:
undetected_messages += 1
event: TwitterEvent = TwitterEvent(
event_type=TwitterMessageType.STATUS.value,
event_id=message.id,
user_handle=message.user.screen_name,
user_id=message.user.id,
event_text=message.full_text,
created_at=message.created_at.replace(tzinfo=pytz.timezone('UTC')).timestamp() * MILLISECONDS_PER_SECOND,
in_reply_to_message_id=message.in_reply_to_status_id,
location=message.place and message.place.full_name,
user_mentions=' '.join([user['screen_name'] for user in message.entities['user_mentions']]),
detected_via_account_activity_api=False)
TwitterEvent.query.session.add(event)
TwitterEvent.query.session.commit()
LOG.debug(
f"Found {undetected_messages} status{'' if undetected_messages == 1 else 'es'} that "
f"{'was' if undetected_messages == 1 else 'were'} previously undetected.")
def test_process_response_direct_message(self, mocked_is_production):
""" Test direct message and new format """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
lookup_request = AccountActivityAPIDirectMessage(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny ny:hme6483',
event_type='direct_message',
user_handle=username,
user_id=30139847),
message_source='direct_message')
combined_message = "@bdhowald #NY_HME6483 has been queried 1 time.\n\nTotal parking and camera violation tickets: 15\n\n4 | No Standing - Day/Time Limits\n3 | No Parking - Street Cleaning\n1 | Failure To Display Meter Receipt\n1 | No Violation Description Available\n1 | Bus Lane Violation\n\n@bdhowald Parking and camera violation tickets for #NY_HME6483, cont'd:\n\n1 | Failure To Stop At Red Light\n1 | No Standing - Commercial Meter Zone\n1 | Expired Meter\n1 | Double Parking\n1 | No Angle Parking\n\n@bdhowald Violations by year for #NY_HME6483:\n\n10 | 2017\n15 | 2018\n\n@bdhowald Known fines for #NY_HME6483:\n\n$200.00 | Fined\n$125.00 | Outstanding\n$75.00 | Paid\n"
mocked_is_production.return_value = True
send_direct_message_mock = MagicMock('send_direct_message_mock')
application_api_mock = MagicMock(name='application_api')
application_api_mock.send_direct_message = send_direct_message_mock
self.tweeter._app_api = application_api_mock
self.tweeter._process_response(
request_object=lookup_request,
response_parts=[[combined_message]],
successful_lookup=True)
send_direct_message_mock.assert_called_with(
recipient_id=30139847,
text=combined_message)
def test_process_response_campaign_only_lookup(self,
recursively_process_status_updates_mock):
""" Test campaign-only lookup """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
campaign_hashtag = '#SaferSkillman'
campaign_tickets = random.randint(1000, 2000)
campaign_vehicles = random.randint(100, 200)
response_parts = [[(f"@{username} {'{:,}'.format(campaign_vehicles)} vehicles with a total of "
f"{'{:,}'.format(campaign_tickets)} tickets have been tagged with {campaign_hashtag}.\n\n")]]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text=f'@howsmydrivingny {campaign_hashtag}',
event_type='status',
user_handle=username,
user_id=30139847,
user_mention_ids='813286,19834403,1230933768342528001,37687633,66379182',
user_mentions='@BarackObama @NYCMayor @GoodNYCMayor @NYC_DOT @NYCTSubway'
),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(
request_object=lookup_request,
response_parts=response_parts,
successful_lookup=True)
recursively_process_status_updates_mock.assert_called_with(
response_parts=response_parts,
message_id=message_id,
user_mention_ids=[
'813286',
'19834403',
'1230933768342528001',
'37687633',
'66379182'
])
def test_process_response_status_legacy_format(
self, recursively_process_status_updates_mock):
""" Test status and old format """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
response_parts = [[
'@BarackObama #PA_GLF7467 has been queried 1 time.\n\nTotal parking and camera violation tickets: 49\n\n17 | No Parking - Street Cleaning\n6 | Expired Meter\n5 | No Violation Description Available\n3 | Fire Hydrant\n3 | No Parking - Day/Time Limits\n',
"@BarackObama Parking and camera violation tickets for #PA_GLF7467, cont'd:\n\n3 | Failure To Display Meter Receipt\n3 | School Zone Speed Camera Violation\n2 | No Parking - Except Authorized Vehicles\n2 | Bus Lane Violation\n1 | Failure To Stop At Red Light\n",
"@BarackObama Parking and camera violation tickets for #PA_GLF7467, cont'd:\n\n1 | No Standing - Day/Time Limits\n1 | No Standing - Except Authorized Vehicle\n1 | Obstructing Traffic Or Intersection\n1 | Double Parking\n",
'@BarackObama Known fines for #PA_GLF7467:\n\n$1,000.00 | Fined\n$225.00 | Outstanding\n$775.00 | Paid\n'
]]
lookup_request = AccountActivityAPIStatus(message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny plate:glf7467 state:pa',
event_type='status',
user_handle=username,
user_id=30139847),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
is_production_mock = MagicMock(name='is_production')
is_production_mock.return_value = True
create_favorite_mock = MagicMock(name='is_production')
create_favorite_mock.return_value = True
application_api_mock = MagicMock(name='application_api')
application_api_mock.create_favorite = create_favorite_mock
self.tweeter._is_production = is_production_mock
self.tweeter._app_api = application_api_mock
reply_event_args['username'] = username
self.tweeter._process_response(request_object=lookup_request,
response_parts=response_parts,
successful_lookup=True)
recursively_process_status_updates_mock.assert_called_with(
response_parts=response_parts, message_id=message_id)
def test_process_response_with_error(self,
recursively_process_status_updates_mock):
""" Test error handling """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny plate:glf7467 state:pa',
event_type='status',
user_handle=username,
user_id=30139847,
user_mention_ids='813286,19834403,1230933768342528001,37687633,66379182',
user_mentions='@BarackObama @NYCMayor @GoodNYCMayor @NYC_DOT @NYCTSubway'
),
message_source='status'
)
response_parts = [
['@' + username + " Sorry, I encountered an error. Tagging @bdhowald."]]
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(
request_object=lookup_request,
response_parts=response_parts,
successful_lookup=True)
recursively_process_status_updates_mock.assert_called_with(
response_parts=response_parts,
message_id=message_id,
user_mention_ids=[
'813286',
'19834403',
'1230933768342528001',
'37687633',
'66379182'
])
def test_process_response_with_direct_message_api_direct_message(self,
recursively_process_status_updates_mock):
""" Test plateless lookup """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
response_parts = [
['@' + username + " I think you're trying to look up a plate, but can't be sure.\n\nJust a reminder, the format is <state|province|territory>:<plate>, e.g. NY:abc1234"]]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny the state is ny',
event_type='status',
user_handle=username,
user_id=30139847,
user_mention_ids='813286,19834403,1230933768342528001,37687633,66379182',
user_mentions='@BarackObama @NYCMayor @GoodNYCMayor @NYC_DOT @NYCTSubway'
),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(
request_object=lookup_request,
response_parts=response_parts,
successful_lookup=True)
recursively_process_status_updates_mock.assert_called_with(
response_parts=response_parts,
message_id=message_id,
user_mention_ids=[
'813286',
'19834403',
'1230933768342528001',
'37687633',
'66379182'
])
def test_build_account_activity_api_status(self):
account_activity_api_status_twitter_event = TwitterEvent(
id=1,
created_at=random.randint(self.CREATED_AT_MIN,
self.CREATED_AT_MAX),
event_id=random.randint(self.EVENT_ID_MIN, self.EVENT_ID_MAX),
event_text='hi there',
event_type='status',
user_handle=self.OTHER_TWITTER_HANDLE,
user_id=random.randint(self.USER_ID_MIN, self.USER_ID_MAX),
user_mentions=self.HMDNY_TWITTER_HANDLE)
req = self.reply_argument_builder.build_reply_data(
account_activity_api_status_twitter_event, LookupSource.STATUS)
self.assertIsInstance(req, AccountActivityAPIStatus)
def _add_twitter_events_for_missed_direct_messages(self, messages: list[tweepy.models.Status]) -> None:
"""Creates TwitterEvent objects when the Account Activity API fails to send us
direct message events via webhooks to have them created by the HowsMyDrivingNY API.
:param messages: list[tweepy.DirectMessage]: The direct messages returned via the
Twitter Search API via Tweepy.
"""
undetected_messages = 0
sender_ids = set(int(message.message_create['sender_id']) for message in messages)
sender_objects = self._get_twitter_client_api().lookup_users(user_id=sender_ids)
senders = {sender.id_str:sender for sender in sender_objects}
for message in messages:
existing_event: Optional[TwitterEvent] = TwitterEvent.query.filter(TwitterEvent.event_id == message.id).first()
if not existing_event and int(message.message_create['sender_id']) != HMDNY_TWITTER_USER_ID:
undetected_messages += 1
sender = senders[message.message_create['sender_id']]
event: TwitterEvent = TwitterEvent(
event_type=TwitterMessageType.DIRECT_MESSAGE.value,
event_id=message.id,
user_handle=sender.screen_name,
user_id=sender.id,
event_text=message.message_create['message_data']['text'],
created_at=message.created_timestamp,
in_reply_to_message_id=None,
location=None,
user_mention_ids=','.join([user['id_str'] for user in message.message_create['message_data']['entities']['user_mentions']]),
user_mentions=' '.join([user['screen_name'] for user in message.message_create['message_data']['entities']['user_mentions']]),
detected_via_account_activity_api=False)
TwitterEvent.query.session.add(event)
TwitterEvent.query.session.commit()
LOG.debug(
f"Found {undetected_messages} direct message{'' if undetected_messages == 1 else 's'} that "
f"{'was' if undetected_messages == 1 else 'were'} previously undetected.")
def test_process_response_with_error(
self, recursively_process_status_updates_mock):
""" Test error handling """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
lookup_request = AccountActivityAPIStatus(message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny plate:glf7467 state:pa',
event_type='status',
user_handle=username,
user_id=30139847),
message_source='status')
response_parts = [[
'@' + username +
" Sorry, I encountered an error. Tagging @bdhowald."
]]
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(request_object=lookup_request,
response_parts=response_parts,
successful_lookup=True)
recursively_process_status_updates_mock.assert_called_with(
response_parts=response_parts, message_id=message_id)
def test_process_response_campaign_only_lookup(self,
recursively_process_status_updates_mock):
""" Test campaign-only lookup """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
campaign_hashtag = '#SaferSkillman'
campaign_tickets = random.randint(1000, 2000)
campaign_vehicles = random.randint(100, 200)
response_parts = [[(f"@{username} {'{:,}'.format(campaign_vehicles)} vehicles with a total of "
f"{'{:,}'.format(campaign_tickets)} tickets have been tagged with {campaign_hashtag}.\n\n")]]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text=f'@howsmydrivingny {campaign_hashtag}',
event_type='status',
user_handle=username,
user_id=30139847),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(reply_event_args)
recursively_process_status_updates_mock.assert_called_with(
response_parts, message_id, username)
def test_process_response_with_search_status(
self, recursively_process_status_updates_mock):
""" Test plateless lookup """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
response_parts = [[
'@' + username +
' I’d be happy to look that up for you!\n\nJust a reminder, the format is <state|province|territory>:<plate>, e.g. NY:abc1234'
]]
lookup_request = AccountActivityAPIStatus(message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny plate dkr9364 state ny',
event_type='status',
user_handle=username,
user_id=30139847),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(request_object=lookup_request,
response_parts=response_parts,
successful_lookup=True)
recursively_process_status_updates_mock.assert_called_with(
response_parts=response_parts, message_id=message_id)
def test_process_response_with_direct_message_api_direct_message(self,
recursively_process_status_updates_mock):
""" Test plateless lookup """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
response_parts = [
['@' + username + " I think you're trying to look up a plate, but can't be sure.\n\nJust a reminder, the format is <state|province|territory>:<plate>, e.g. NY:abc1234"]]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny the state is ny',
event_type='status',
user_handle=username,
user_id=30139847),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(reply_event_args)
recursively_process_status_updates_mock.assert_called_with(
response_parts, message_id, username)
def _process_twitter_event(self, event: TwitterEvent):
LOG.debug(f'Beginning response for event: {event.id}')
# search for duplicates
is_event_duplicate: bool = TwitterEvent.query.filter_by(
event_type=event.event_type,
event_id=event.event_id,
user_handle=event.user_handle,
responded_to=True
).filter(
TwitterEvent.id != event.id).count() > 0
if is_event_duplicate:
event.is_duplicate = True
TwitterEvent.query.session.commit()
LOG.info(f'Event {event.id} is a duplicate, skipping.')
else:
event.response_in_progress = True
TwitterEvent.query.session.commit()
try:
message_source: str = LookupSource(event.event_type)
# build request
lookup_request: Type[BaseLookupRequest] = self.reply_argument_builder.build_reply_data(
message=event,
message_source=message_source)
user_is_follower = lookup_request.requesting_user_is_follower(
follower_ids=self._get_follower_ids())
perform_lookup_for_user: bool = (user_is_follower or
event.user_favorited_non_follower_reply)
if self.aggregator.lookup_has_valid_plates(
lookup_request=lookup_request) and not perform_lookup_for_user:
response_parts: List[Any]
if lookup_request.is_direct_message():
response_parts = [L10N.NON_FOLLOWER_DIRECT_MESSAGE_REPLY_STRING]
elif lookup_request.is_status():
response_parts = [L10N.NON_FOLLOWER_TWEET_REPLY_STRING]
try:
reply_message_id = self._process_response(
request_object=lookup_request,
response_parts=response_parts)
# Save the reply id, so that when the user favorites it,
# we can trigger the search.
non_follower_reply = NonFollowerReply(
created_at=(int(datetime.utcnow().timestamp() *
MILLISECONDS_PER_SECOND)),
event_type=event.event_type,
event_id=reply_message_id,
in_reply_to_message_id=event.event_id,
user_handle=event.user_handle,
user_id=event.user_id)
NonFollowerReply.query.session.add(
non_follower_reply)
NonFollowerReply.query.session.commit()
except tweepy.error.TweepError as e:
event.error_on_lookup = True
else:
# Reply to the event.
reply_to_event = self.aggregator.initiate_reply(
lookup_request=lookup_request)
success = reply_to_event['success']
if success:
# There's no need to tell people that
# there was an error more than once.
if not (reply_to_event[
'error_on_lookup'] and event.error_on_lookup):
try:
self._process_response(
request_object=reply_to_event['request_object'],
response_parts=reply_to_event['response_parts'],
successful_lookup=reply_to_event.get('successful_lookup'))
except tweepy.error.TweepError as e:
reply_to_event['error_on_lookup'] = True
# Update error status
if reply_to_event['error_on_lookup']:
event.error_on_lookup = True
else:
event.error_on_lookup = False
# We've responded!
event.response_in_progress = False
event.responded_to = True
TwitterEvent.query.session.commit()
except ValueError as e:
LOG.error(
f'Encountered unknown event type. '
f'Response is not possible.')
def _find_and_respond_to_twitter_events(self):
"""Looks for TwitterEvent objects that have not yet been responded to and
begins the process of creating a response. Additionally, failed events are
rerun to provide a correct response, particularly useful in cases where
external apis are down for maintenance.
"""
interval = 3.0 if self._is_production() else self.DEVELOPMENT_TIME_INTERVAL
self.events_iteration += 1
LOG.debug(
f'Looking up twitter events on iteration {self.events_iteration}')
# start timer
threading.Timer(
interval, self._find_and_respond_to_twitter_events).start()
try:
new_events: [List[TwitterEvent]] = TwitterEvent.get_all_by(
is_duplicate=False,
responded_to=False,
response_in_progress=False)
LOG.debug(f'new events: {new_events}')
failed_events: [List[TwitterEvent]] = TwitterEvent.get_all_by(
is_duplicate=False,
error_on_lookup=True,
responded_to=True,
response_in_progress=False)
LOG.debug(f'failed events: {failed_events}')
events_to_respond_to: [List[TwitterEvent]] = new_events + failed_events
LOG.debug(f'events to respond to: {events_to_respond_to}')
for event in events_to_respond_to:
LOG.debug(f'Beginning response for event: {event.id}')
# search for duplicates
is_event_duplicate: bool = TwitterEvent.query.filter_by(
event_type=event.event_type,
event_id=event.event_id,
user_handle=event.user_handle,
responded_to=True
).filter(
TwitterEvent.id != event.id).count() > 0
if is_event_duplicate:
event.is_duplicate = True
TwitterEvent.query.session.commit()
LOG.info(f'Event {event.id} is a duplicate, skipping.')
else:
event.response_in_progress = True
TwitterEvent.query.session.commit()
try:
message_source = LookupSource(event.event_type)
# build request
lookup_request: Type[BaseLookupRequest] = self.reply_argument_builder.build_reply_data(
message=event,
message_source=message_source)
# Reply to the event.
reply_event = self.aggregator.initiate_reply(
lookup_request=lookup_request)
success = reply_event.get('success', False)
if success:
# Need username for statuses
reply_event['username'] = event.user_handle
# There's need to tell people that there was an error more than once
if not (reply_event.get(
'error_on_lookup') and event.error_on_lookup):
try:
self._process_response(reply_event)
except tweepy.error.TweepError as e:
reply_event['error_on_lookup'] = True
# We've responded!
event.response_in_progress = False
event.responded_to = True
# Update error status
if reply_event.get('error_on_lookup'):
event.error_on_lookup = True
else:
event.error_on_lookup = False
TwitterEvent.query.session.commit()
except ValueError as e:
LOG.error(
f'Encountered unknown event type. '
f'Response is not possible.')
except Exception as e:
LOG.error(e)
LOG.error(str(e))
LOG.error(e.args)
logging.exception("stack trace")
finally:
TwitterEvent.query.session.close()
def test_find_and_respond_to_missed_direct_messages(self, twitter_event_mock):
db_id = 1
event_id = random.randint(10000000000000000000, 20000000000000000000)
event_text = 'abc1234:ny'
event_type = 'direct_message'
timestamp = random.randint(1500000000000, 1700000000000)
user_handle = 'bdhowald'
user_id = random.randint(1000000000, 2000000000)
new_twitter_event = TwitterEvent(
created_at=timestamp + 1,
detected_via_account_activity_api=False,
event_id=event_id + 1,
event_text=event_text + '!',
event_type=event_type,
in_reply_to_message_id=None,
location=None,
responded_to=False,
user_handle=user_handle,
user_id=user_id,
user_mentions=[])
message_needing_event = MagicMock(
id=event_id + 1,
created_timestamp = timestamp + 1,
message_create={
'message_data': {
'entities': {
'user_mentions': [
{
'id': 123,
'id_str': '123',
'screen_name': user_handle
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
},
'text': event_text + '!'
},
'sender_id': f'{user_id}'
},
name='message_needing_event')
message_not_needing_event = MagicMock(
id=event_id - 1,
created_timestamp = timestamp - 1,
message_create={
'message_data': {
'entities': {
'user_mentions': [
{
'id': 123,
'id_str': '123',
'screen_name': user_handle
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
},
'text': event_text,
},
'sender_id': f'{user_id}'
},
name='message_not_needing_event')
sender = MagicMock(
id=user_id,
id_str=f'{user_id}',
name='sender',
screen_name=user_handle)
twitter_event_mock.return_value = new_twitter_event
twitter_event_mock.query.filter.return_value.first.side_effect = [
None, message_not_needing_event]
client_api_mock = MagicMock(name='client_api')
client_api_mock.get_direct_messages.return_value = [
message_needing_event, message_not_needing_event]
client_api_mock.lookup_users.return_value = [sender]
self.tweeter._client_api = client_api_mock
self.tweeter._find_and_respond_to_missed_direct_messages()
twitter_event_mock.query.session.add.assert_called_once_with(new_twitter_event)
twitter_event_mock.query.session.commit.assert_called_once_with()
self.mocked_log.debug.assert_called_with('Found 1 direct message that was previously undetected.')
self.tweeter.terminate_lookups()
def test_find_and_respond_to_missed_statuses(self, twitter_event_mock):
db_id = 1
event_id = random.randint(10000000000000000000, 20000000000000000000)
event_text = '@HowsMyDrivingNY @bdhowald abc1234:ny'
event_type = 'status'
in_reply_to_message_id = random.randint(
10000000000000000000, 20000000000000000000)
location = 'Queens, NY'
now = datetime.utcnow()
place = MagicMock(
full_name=location,
name='place')
user_id = random.randint(1000000000, 2000000000)
user = MagicMock(
id=user_id,
name='user')
user_handle = 'bdhowald'
older_twitter_event = TwitterEvent(
id=db_id,
created_at=now.replace(tzinfo=pytz.timezone('UTC')).timestamp() * 1000,
detected_via_account_activity_api=False,
event_id=event_id,
event_text=event_text,
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
location=location,
responded_to=True,
user_handle=user_handle,
user_id=user_id,
user_mention_ids=[user_id],
user_mentions=[
{
'id': 123,
'id_str': '123',
'screen_name': user_handle
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
])
new_twitter_event = TwitterEvent(
created_at=now.replace(tzinfo=pytz.timezone('UTC')).timestamp() * 1000,
detected_via_account_activity_api=False,
event_id=event_id + 1,
event_text=event_text + '!',
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
location=location,
responded_to=False,
user_handle=user_handle,
user_id=user_id,
user_mention_ids=[user_id],
user_mentions=user_handle)
status_needing_event = MagicMock(
id=event_id + 1,
created_at = now,
entities={
'user_mentions': [
{
'id': 123,
'id_str': '123',
'screen_name': user_handle
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
},
full_text=event_text + '!',
in_reply_to_message_id=in_reply_to_message_id,
name='status_needing_event',
place=place,
user=user)
status_not_needing_event = MagicMock(
id=event_id - 1,
created_at = now - relativedelta(minutes=1),
entities={
'user_mentions': [
{
'id': 123,
'id_str': '123',
'screen_name': user_handle
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
},
full_text=event_text,
in_reply_to_message_id=in_reply_to_message_id,
name='status_not_needing_event',
place=place,
user=user)
twitter_event_mock.return_value = new_twitter_event
twitter_event_mock.query.filter().order_by().first.return_value = older_twitter_event
twitter_event_mock.query.filter().first.side_effect = [
None, status_not_needing_event]
client_api_mock = MagicMock(name='client_api')
client_api_mock.mentions_timeline.side_effect = [[
status_needing_event, status_not_needing_event], []]
self.tweeter._client_api = client_api_mock
self.tweeter._find_and_respond_to_missed_statuses()
twitter_event_mock.query.session.add.assert_called_once_with(new_twitter_event)
twitter_event_mock.query.session.commit.assert_called_once_with()
self.mocked_log.debug.assert_called_with('Found 1 status that was previously undetected.')
self.tweeter.terminate_lookups()
def test_find_and_respond_to_failed_twitter_events(self,
twitter_event_mock: MagicMock,
event_type: str,
expect_called: bool,
num_times_failed: int,
last_failed_time: timedelta = timedelta(minutes=0),
tweet_exists: bool = True):
db_id = 1
random_id = random.randint(1000000000000000000, 2000000000000000000)
user_handle = 'bdhowald'
user_id = random.randint(1000000000, 2000000000)
event_text = '@HowsMyDrivingNY abc1234:ny'
timestamp = random.randint(1500000000000, 1700000000000)
in_reply_to_message_id = random.randint(
10000000000000000000, 20000000000000000000)
location = 'Queens, NY'
responded_to = 0
user_mentions = [
{
'id': 123,
'id_str': '123',
'screen_name': 'HowsMyDrivingNY'
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
twitter_event = TwitterEvent(
id=db_id,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
last_failed_at_time=(datetime.utcnow() - last_failed_time),
location=location,
num_times_failed=num_times_failed,
responded_to=responded_to,
user_handle=user_handle,
user_id=user_id,
user_mentions=user_mentions)
direct_message_lookup_request = AccountActivityAPIDirectMessage(
message=TwitterEvent(
id=1,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
user_handle=user_handle,
user_id=user_id,
user_favorited_non_follower_reply=False),
message_source=event_type)
status_lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
user_handle=user_handle,
user_id=user_id,
user_favorited_non_follower_reply=False),
message_source=event_type)
lookup_request = (direct_message_lookup_request if
event_type == 'direct_message' else status_lookup_request)
twitter_event_mock.get_all_by.side_effect = [[], [twitter_event]]
twitter_event_mock.query.filter_by().filter().count.return_value = 0
tweet_exists_mock = MagicMock(name='tweet_exists')
tweet_exists_mock.return_value = True if tweet_exists else False
self.tweeter.tweet_detection_service.tweet_exists = tweet_exists_mock
initiate_reply_mock = MagicMock(name='initiate_reply')
self.tweeter.aggregator.initiate_reply = initiate_reply_mock
build_reply_data_mock = MagicMock(name='build_reply_data')
build_reply_data_mock.return_value = lookup_request
self.tweeter.reply_argument_builder.build_reply_data = build_reply_data_mock
application_api_mock = MagicMock(name='application_api')
application_api_mock.get_follower_ids.return_value = ([user_id], (123, 0))
self.tweeter._app_api = application_api_mock
process_response_mock = MagicMock(name='process_response')
process_response_mock.return_value = random_id
self.tweeter._process_response = process_response_mock
self.tweeter._find_and_respond_to_twitter_events()
if expect_called:
self.tweeter.aggregator.initiate_reply.assert_called_with(
lookup_request=lookup_request)
else:
self.tweeter.aggregator.initiate_reply.assert_not_called()
self.tweeter.terminate_lookups()
def test_find_and_respond_to_twitter_events(self,
twitter_event_mock: MagicMock,
event_type: str,
is_follower: bool,
response_parts: Optional[list[str]] = None):
db_id = 1
random_id = random.randint(1000000000000000000, 2000000000000000000)
user_handle = 'bdhowald'
user_id = random.randint(1000000000, 2000000000)
event_text = '@HowsMyDrivingNY abc1234:ny'
timestamp = random.randint(1500000000000, 1700000000000)
in_reply_to_message_id = random.randint(
10000000000000000000, 20000000000000000000)
location = 'Queens, NY'
responded_to = 0
user_mentions = [
{
'id': 123,
'id_str': '123',
'screen_name': 'HowsMyDrivingNY'
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
twitter_event = TwitterEvent(
id=db_id,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
last_failed_at_time=None,
location=location,
num_times_failed=0,
responded_to=responded_to,
user_handle=user_handle,
user_id=user_id,
user_mentions=user_mentions)
lookup_request = AccountActivityAPIDirectMessage(
message=TwitterEvent(
id=1,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
user_handle=user_handle,
user_id=user_id,
user_favorited_non_follower_reply=False),
message_source=event_type)
twitter_event_mock.get_all_by.return_value = [twitter_event]
twitter_event_mock.query.filter_by().filter().count.return_value = 0
initiate_reply_mock = MagicMock(name='initiate_reply')
self.tweeter.aggregator.initiate_reply = initiate_reply_mock
build_reply_data_mock = MagicMock(name='build_reply_data')
build_reply_data_mock.return_value = lookup_request
self.tweeter.reply_argument_builder.build_reply_data = build_reply_data_mock
application_api_mock = MagicMock(name='application_api')
application_api_mock.get_follower_ids.return_value = ([
user_id if is_follower else (user_id + 1)], (123, 0))
self.tweeter._app_api = application_api_mock
process_response_mock = MagicMock(name='process_response')
process_response_mock.return_value = random_id
self.tweeter._process_response = process_response_mock
self.tweeter._find_and_respond_to_twitter_events()
if is_follower:
self.tweeter.aggregator.initiate_reply.assert_called_with(
lookup_request=lookup_request)
else:
self.tweeter.aggregator.initiate_reply.assert_not_called()
self.tweeter._process_response.assert_called_with(
request_object=lookup_request,
response_parts=response_parts)
self.tweeter.terminate_lookups()
