def setUp(self):
self.tweeter = TrafficViolationsTweeter()
self.tweeter._app_api = MagicMock(name='app_api')
self.tweeter._client_api = MagicMock(name='client_api')
# mock followers ids to be empty so that asking if the
# requesting user is a follower always returns True
self.tweeter._client_api.get_follower_ids.return_value = []
self.log_patcher = mock.patch(
'traffic_violations.services.twitter_service.LOG')
self.mocked_log = self.log_patcher.start()
def setUp(self):
self.listener = TrafficViolationsStreamListener(
TrafficViolationsTweeter())
self.log_patcher = mock.patch(
'traffic_violations.traffic_violations_stream_listener.LOG')
self.mocked_log = self.log_patcher.start()
def run():
tweeter = TrafficViolationsTweeter()
if sys.argv[-1] == 'print_daily_summary':
tweeter._print_daily_summary()
elif sys.argv[-1] == 'print_featured_plate':
tweeter._print_featured_plate()
else:
tweeter._find_and_respond_to_requests()
def perform(self, *args, **kwargs):
is_dry_run: bool = kwargs.get('is_dry_run') or False
use_dvaa_thresholds: bool = kwargs.get('use_dvaa_thresholds') or False
use_only_visible_tweets: bool = kwargs.get('use_only_visible_tweets') or False
tweet_detection_service = TweetDetectionService()
tweeter = TrafficViolationsTweeter()
eastern = pytz.timezone('US/Eastern')
utc = pytz.timezone('UTC')
now = datetime.now()
# If today is leap day, there were no lookups a year ago today
if (now.day == self.LEAP_DAY_DATE and
now.month == self.LEAP_DAY_MONTH):
return
# If today is March 1, and last year was a leap year, show lookups
# from the previous February 29.
one_year_after_leap_day = True if (
now.day == self.POST_LEAP_DAY_DATE and
now.month == self.POST_LEAP_DAY_MONTH and
self._is_leap_year(now.year - 1)) else False
top_of_the_hour_last_year = now.replace(
microsecond=0,
minute=0,
second=0) - relativedelta(years=1)
top_of_the_next_hour_last_year = (
top_of_the_hour_last_year + relativedelta(hours=1))
if use_dvaa_thresholds:
threshold_attribute: str = 'boot_eligible_under_dvaa_threshold'
else:
threshold_attribute: str = 'boot_eligible_under_rdaa_threshold'
base_query: list[Tuple[int]] = PlateLookup.query.session.query(
func.max(PlateLookup.id).label('most_recent_vehicle_lookup')
).filter(
or_(
and_(PlateLookup.created_at >= top_of_the_hour_last_year,
PlateLookup.created_at < top_of_the_next_hour_last_year,
getattr(PlateLookup, threshold_attribute) == True,
PlateLookup.count_towards_frequency == True),
and_(one_year_after_leap_day,
PlateLookup.created_at >= (top_of_the_hour_last_year - relativedelta(days=1)),
PlateLookup.created_at < (top_of_the_next_hour_last_year - relativedelta(days=1)),
getattr(PlateLookup, threshold_attribute) == True,
PlateLookup.count_towards_frequency == True)
)
)
if use_only_visible_tweets:
base_query = base_query.filter(
PlateLookup.message_source == lookup_sources.LookupSource.STATUS.value)
recent_plate_lookup_ids = base_query.group_by(
PlateLookup.plate,
PlateLookup.state
).all()
lookup_ids_to_update: list[int] = [id[0] for id in recent_plate_lookup_ids]
lookups_to_update: list[PlateLookup] = PlateLookup.get_all_in(
id=lookup_ids_to_update)
if not lookups_to_update:
LOG.debug(f'No vehicles for which to perform retrospective job '
f'between {top_of_the_hour_last_year} and '
f'and {top_of_the_next_hour_last_year}.')
for previous_lookup in lookups_to_update:
LOG.debug(f'Performing retrospective job for '
f'{L10N.VEHICLE_HASHTAG.format(previous_lookup.state, previous_lookup.plate)} ')
plate_query: PlateQuery = PlateQuery(created_at=now,
message_source=previous_lookup.message_source,
plate=previous_lookup.plate,
plate_types=previous_lookup.plate_types,
state=previous_lookup.state)
nyc_open_data_service: OpenDataService = OpenDataService()
data_before_query: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query,
until=previous_lookup.created_at)
lookup_before_query: OpenDataServicePlateLookup = data_before_query.data
camera_streak_data_before_query: CameraStreakData = lookup_before_query.camera_streak_data['Mixed']
data_after_query: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query,
since=previous_lookup.created_at,
until=previous_lookup.created_at + relativedelta(years=1))
lookup_after_query: OpenDataServicePlateLookup = data_after_query.data
new_bus_lane_camera_violations: Optional[int] = None
new_speed_camera_violations: Optional[int] = None
new_red_light_camera_violations: Optional[int] = None
for violation_type_summary in lookup_after_query.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary['title'] == 'Bus Lane Violation':
new_bus_lane_camera_violations = violation_count
if violation_type_summary['title'] == 'Failure To Stop At Red Light':
new_red_light_camera_violations = violation_count
if violation_type_summary['title'] == 'School Zone Speed Camera Violation':
new_speed_camera_violations = violation_count
if new_bus_lane_camera_violations is None:
new_bus_lane_camera_violations = 0
if new_red_light_camera_violations is None:
new_red_light_camera_violations = 0
if new_speed_camera_violations is None:
new_speed_camera_violations = 0
new_boot_eligible_violations = (new_red_light_camera_violations +
new_speed_camera_violations)
if new_boot_eligible_violations > 0:
vehicle_hashtag = L10N.VEHICLE_HASHTAG.format(
previous_lookup.state, previous_lookup.plate)
previous_lookup_created_at = utc.localize(
previous_lookup.created_at)
previous_lookup_date = previous_lookup_created_at.astimezone(eastern).strftime(
L10N.REPEAT_LOOKUP_DATE_FORMAT)
previous_lookup_time = previous_lookup_created_at.astimezone(eastern).strftime(
L10N.REPEAT_LOOKUP_TIME_FORMAT)
red_light_camera_violations_string = (
f'{new_red_light_camera_violations} | Red Light Camera Violations\n'
if new_red_light_camera_violations > 0 else '')
speed_camera_violations_string = (
f'{new_speed_camera_violations} | Speed Safety Camera Violations\n'
if new_speed_camera_violations > 0 else '')
reckless_driver_summary_string = (
f'{vehicle_hashtag} was originally '
f'queried on {previous_lookup_date} '
f'at {previous_lookup_time}')
# assume we can't link
can_link_tweet = False
# Where did this come from?
if previous_lookup.message_source == lookup_sources.LookupSource.STATUS.value:
# Determine if tweet is still visible:
if tweet_detection_service.tweet_exists(id=previous_lookup.message_id,
username=previous_lookup.username):
can_link_tweet = True
if can_link_tweet:
reckless_driver_summary_string += L10N.PREVIOUS_LOOKUP_STATUS_STRING.format(
previous_lookup.username,
previous_lookup.username,
previous_lookup.message_id)
else:
reckless_driver_summary_string += '.'
if use_only_visible_tweets and not can_link_tweet:
# If we're only displaying tweets we can quote tweet,
# skip this one since we can'tt.
continue
reckless_driver_update_string = (
f'From {camera_streak_data_before_query.min_streak_date} to '
f'{camera_streak_data_before_query.max_streak_date}, this vehicle '
f'received {camera_streak_data_before_query.max_streak} camera '
f'violations. Over the past 12 months, this vehicle '
f'received {new_boot_eligible_violations} new camera violation'
f"{'' if new_boot_eligible_violations == 1 else 's'}: \n\n"
f'{red_light_camera_violations_string}'
f'{speed_camera_violations_string}')
messages: list[str] = [
reckless_driver_summary_string,
reckless_driver_update_string]
if not is_dry_run:
success: bool = tweeter.send_status(
message_parts=messages,
on_error_message=(
f'Error printing reckless driver update. '
f'Tagging @bdhowald.'))
if success:
LOG.debug('Reckless driver retrospective job '
'ran successfully.')
else:
print(reckless_driver_update_string)
def perform(self, *args, **kwargs):
is_dry_run: bool = kwargs.get('is_dry_run') or False
tweeter = TrafficViolationsTweeter()
repeat_camera_offender: Optional[
RepeatCameraOffender] = RepeatCameraOffender.query.filter(
and_(RepeatCameraOffender.times_featured == 0,
RepeatCameraOffender.total_camera_violations >=
25)).order_by(func.random()).first()
if repeat_camera_offender:
# get the number of vehicles that have the same number
# of violations
tied_with = RepeatCameraOffender.query.filter(
RepeatCameraOffender.total_camera_violations ==
repeat_camera_offender.total_camera_violations).count()
# since the vehicles are in descending order of violations,
# we find the record that has the same number of violations
# and the lowest id...
min_id = RepeatCameraOffender.query.session.query(
func.min(RepeatCameraOffender.id)).filter(
RepeatCameraOffender.total_camera_violations ==
repeat_camera_offender.total_camera_violations).one()[0]
# nth place is simply the sum of the two values minus one.
nth_place = tied_with + min_id - 1
red_light_camera_violations = \
repeat_camera_offender.red_light_camera_violations
speed_camera_violations = \
repeat_camera_offender.speed_camera_violations
vehicle_hashtag: str = L10N.VEHICLE_HASHTAG.format(
repeat_camera_offender.state, repeat_camera_offender.plate_id)
# one of 'st', 'nd', 'rd', 'th'
suffix: str = string_utils.determine_ordinal_indicator(nth_place)
# how bad is this vehicle?
worst_substring: str = (f'{nth_place}{suffix}-worst'
if nth_place > 1 else 'worst')
tied_substring: str = ' tied for' if tied_with != 1 else ''
spaces_needed: int = twitter_utils.padding_spaces_needed(
red_light_camera_violations, speed_camera_violations)
featured_string = L10N.REPEAT_CAMERA_OFFENDER_STRING.format(
vehicle_hashtag,
repeat_camera_offender.total_camera_violations,
str(red_light_camera_violations).ljust(
spaces_needed - len(str(red_light_camera_violations))),
str(speed_camera_violations).ljust(
spaces_needed - len(str(speed_camera_violations))),
vehicle_hashtag, tied_substring, worst_substring)
messages: List[str] = [featured_string]
if not is_dry_run:
success: bool = tweeter.send_status(
message_parts=messages,
on_error_message=(f'Error printing featured plate. '
f'Tagging @bdhowald.'))
if success:
repeat_camera_offender.times_featured += 1
RepeatCameraOffender.query.session.commit()
LOG.debug('Featured job plate ran successfully.')
class TestTrafficViolationsTweeter(unittest.TestCase):
def setUp(self):
self.tweeter = TrafficViolationsTweeter()
self.tweeter._app_api = MagicMock(name='app_api')
self.tweeter._client_api = MagicMock(name='client_api')
# mock followers ids to be empty so that asking if the
# requesting user is a follower always returns True
self.tweeter._client_api.get_follower_ids.return_value = []
self.log_patcher = mock.patch(
'traffic_violations.services.twitter_service.LOG')
self.mocked_log = self.log_patcher.start()
def tearDown(self):
self.log_patcher.stop()
def test_find_and_respond_to_requests(self):
direct_messages_mock = MagicMock(
name='_find_and_respond_to_missed_direct_messages')
statuses_mock = MagicMock(
name='_find_and_respond_to_missed_statuses')
twitter_events_mock = MagicMock(
name='_find_and_respond_to_twitter_events')
self.tweeter._find_and_respond_to_twitter_events = twitter_events_mock
self.tweeter._find_and_respond_to_missed_direct_messages = direct_messages_mock
self.tweeter._find_and_respond_to_missed_statuses = statuses_mock
self.tweeter.find_and_respond_to_requests()
direct_messages_mock.assert_called_with()
statuses_mock.assert_called_with()
twitter_events_mock.assert_called_with()
@mock.patch(
'traffic_violations.services.twitter_service.TwitterEvent')
def test_find_and_respond_to_missed_direct_messages(self, twitter_event_mock):
db_id = 1
event_id = random.randint(10000000000000000000, 20000000000000000000)
event_text = 'abc1234:ny'
event_type = 'direct_message'
timestamp = random.randint(1500000000000, 1700000000000)
user_handle = 'bdhowald'
user_id = random.randint(1000000000, 2000000000)
new_twitter_event = TwitterEvent(
created_at=timestamp + 1,
detected_via_account_activity_api=False,
event_id=event_id + 1,
event_text=event_text + '!',
event_type=event_type,
in_reply_to_message_id=None,
location=None,
responded_to=False,
user_handle=user_handle,
user_id=user_id,
user_mentions=[])
message_needing_event = MagicMock(
id=event_id + 1,
created_timestamp = timestamp + 1,
message_create={
'message_data': {
'entities': {
'user_mentions': [
{
'id': 123,
'id_str': '123',
'screen_name': user_handle
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
},
'text': event_text + '!'
},
'sender_id': f'{user_id}'
},
name='message_needing_event')
message_not_needing_event = MagicMock(
id=event_id - 1,
created_timestamp = timestamp - 1,
message_create={
'message_data': {
'entities': {
'user_mentions': [
{
'id': 123,
'id_str': '123',
'screen_name': user_handle
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
},
'text': event_text,
},
'sender_id': f'{user_id}'
},
name='message_not_needing_event')
sender = MagicMock(
id=user_id,
id_str=f'{user_id}',
name='sender',
screen_name=user_handle)
twitter_event_mock.return_value = new_twitter_event
twitter_event_mock.query.filter.return_value.first.side_effect = [
None, message_not_needing_event]
client_api_mock = MagicMock(name='client_api')
client_api_mock.get_direct_messages.return_value = [
message_needing_event, message_not_needing_event]
client_api_mock.lookup_users.return_value = [sender]
self.tweeter._client_api = client_api_mock
self.tweeter._find_and_respond_to_missed_direct_messages()
twitter_event_mock.query.session.add.assert_called_once_with(new_twitter_event)
twitter_event_mock.query.session.commit.assert_called_once_with()
self.mocked_log.debug.assert_called_with('Found 1 direct message that was previously undetected.')
self.tweeter.terminate_lookups()
@mock.patch(
'traffic_violations.services.twitter_service.TwitterEvent')
def test_find_and_respond_to_missed_statuses(self, twitter_event_mock):
db_id = 1
event_id = random.randint(10000000000000000000, 20000000000000000000)
event_text = '@HowsMyDrivingNY @bdhowald abc1234:ny'
event_type = 'status'
in_reply_to_message_id = random.randint(
10000000000000000000, 20000000000000000000)
location = 'Queens, NY'
now = datetime.utcnow()
place = MagicMock(
full_name=location,
name='place')
user_id = random.randint(1000000000, 2000000000)
user = MagicMock(
id=user_id,
name='user')
user_handle = 'bdhowald'
older_twitter_event = TwitterEvent(
id=db_id,
created_at=now.replace(tzinfo=pytz.timezone('UTC')).timestamp() * 1000,
detected_via_account_activity_api=False,
event_id=event_id,
event_text=event_text,
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
location=location,
responded_to=True,
user_handle=user_handle,
user_id=user_id,
user_mention_ids=[user_id],
user_mentions=[
{
'id': 123,
'id_str': '123',
'screen_name': user_handle
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
])
new_twitter_event = TwitterEvent(
created_at=now.replace(tzinfo=pytz.timezone('UTC')).timestamp() * 1000,
detected_via_account_activity_api=False,
event_id=event_id + 1,
event_text=event_text + '!',
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
location=location,
responded_to=False,
user_handle=user_handle,
user_id=user_id,
user_mention_ids=[user_id],
user_mentions=user_handle)
status_needing_event = MagicMock(
id=event_id + 1,
created_at = now,
entities={
'user_mentions': [
{
'id': 123,
'id_str': '123',
'screen_name': user_handle
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
},
full_text=event_text + '!',
in_reply_to_message_id=in_reply_to_message_id,
name='status_needing_event',
place=place,
user=user)
status_not_needing_event = MagicMock(
id=event_id - 1,
created_at = now - relativedelta(minutes=1),
entities={
'user_mentions': [
{
'id': 123,
'id_str': '123',
'screen_name': user_handle
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
},
full_text=event_text,
in_reply_to_message_id=in_reply_to_message_id,
name='status_not_needing_event',
place=place,
user=user)
twitter_event_mock.return_value = new_twitter_event
twitter_event_mock.query.filter().order_by().first.return_value = older_twitter_event
twitter_event_mock.query.filter().first.side_effect = [
None, status_not_needing_event]
client_api_mock = MagicMock(name='client_api')
client_api_mock.mentions_timeline.side_effect = [[
status_needing_event, status_not_needing_event], []]
self.tweeter._client_api = client_api_mock
self.tweeter._find_and_respond_to_missed_statuses()
twitter_event_mock.query.session.add.assert_called_once_with(new_twitter_event)
twitter_event_mock.query.session.commit.assert_called_once_with()
self.mocked_log.debug.assert_called_with('Found 1 status that was previously undetected.')
self.tweeter.terminate_lookups()
@mock.patch(
'traffic_violations.services.twitter_service.TwitterEvent')
def test_find_and_respond_to_missed_statuses_with_no_undetected_events(self, twitter_event_mock):
twitter_event_mock.query.filter().order_by().first.return_value = None
self.tweeter._find_and_respond_to_missed_statuses()
twitter_event_mock.query.session().add.assert_not_called()
twitter_event_mock.query.session().commit.assert_not_called()
self.tweeter.terminate_lookups()
@ddt.data({
'event_type': 'direct_message',
'is_follower': True
}, {
'event_type': 'direct_message',
'is_follower': False,
'response_parts': [
'If you would like to look up plates via direct message, '
'please follow @HowsMyDrivingNY and try again.']
}, {
'event_type': 'status',
'is_follower': False,
'response_parts': [
"It appears that you don't follow @HowsMyDrivingNY.\n\n"
'No worries, simply like this tweet to perform a query '
'or visit https://howsmydrivingny.nyc.']
})
@ddt.unpack
@mock.patch(
'traffic_violations.services.twitter_service.TwitterEvent')
def test_find_and_respond_to_twitter_events(self,
twitter_event_mock: MagicMock,
event_type: str,
is_follower: bool,
response_parts: Optional[list[str]] = None):
db_id = 1
random_id = random.randint(1000000000000000000, 2000000000000000000)
user_handle = 'bdhowald'
user_id = random.randint(1000000000, 2000000000)
event_text = '@HowsMyDrivingNY abc1234:ny'
timestamp = random.randint(1500000000000, 1700000000000)
in_reply_to_message_id = random.randint(
10000000000000000000, 20000000000000000000)
location = 'Queens, NY'
responded_to = 0
user_mentions = [
{
'id': 123,
'id_str': '123',
'screen_name': 'HowsMyDrivingNY'
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
twitter_event = TwitterEvent(
id=db_id,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
last_failed_at_time=None,
location=location,
num_times_failed=0,
responded_to=responded_to,
user_handle=user_handle,
user_id=user_id,
user_mentions=user_mentions)
lookup_request = AccountActivityAPIDirectMessage(
message=TwitterEvent(
id=1,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
user_handle=user_handle,
user_id=user_id,
user_favorited_non_follower_reply=False),
message_source=event_type)
twitter_event_mock.get_all_by.return_value = [twitter_event]
twitter_event_mock.query.filter_by().filter().count.return_value = 0
initiate_reply_mock = MagicMock(name='initiate_reply')
self.tweeter.aggregator.initiate_reply = initiate_reply_mock
build_reply_data_mock = MagicMock(name='build_reply_data')
build_reply_data_mock.return_value = lookup_request
self.tweeter.reply_argument_builder.build_reply_data = build_reply_data_mock
application_api_mock = MagicMock(name='application_api')
application_api_mock.get_follower_ids.return_value = ([
user_id if is_follower else (user_id + 1)], (123, 0))
self.tweeter._app_api = application_api_mock
process_response_mock = MagicMock(name='process_response')
process_response_mock.return_value = random_id
self.tweeter._process_response = process_response_mock
self.tweeter._find_and_respond_to_twitter_events()
if is_follower:
self.tweeter.aggregator.initiate_reply.assert_called_with(
lookup_request=lookup_request)
else:
self.tweeter.aggregator.initiate_reply.assert_not_called()
self.tweeter._process_response.assert_called_with(
request_object=lookup_request,
response_parts=response_parts)
self.tweeter.terminate_lookups()
@ddt.data({
'event_type': 'direct_message',
'expect_called': True,
'num_times_failed': 0
}, {
'event_type': 'direct_message',
'expect_called': True,
'num_times_failed': 0
}, {
'event_type': 'status',
'expect_called': False,
'num_times_failed': 0,
'tweet_exists': False,
}, {
'event_type': 'status',
'expect_called': False,
'last_failed_time': timedelta(minutes=4),
'num_times_failed': 1
}, {
'event_type': 'status',
'expect_called': True,
'last_failed_time': timedelta(minutes=6),
'num_times_failed': 1
}, {
'event_type': 'status',
'expect_called': False,
'last_failed_time': timedelta(minutes=59),
'num_times_failed': 2
}, {
'event_type': 'status',
'expect_called': True,
'last_failed_time': timedelta(minutes=61),
'num_times_failed': 2
}, {
'event_type': 'status',
'expect_called': False,
'last_failed_time': timedelta(hours=2),
'num_times_failed': 3
}, {
'event_type': 'status',
'expect_called': True,
'last_failed_time': timedelta(hours=4),
'num_times_failed': 3
}, {
'event_type': 'status',
'expect_called': False,
'last_failed_time': timedelta(hours=23),
'num_times_failed': 4
}, {
'event_type': 'status',
'expect_called': True,
'last_failed_time': timedelta(hours=25),
'num_times_failed': 4
}, {
'event_type': 'status',
'expect_called': False,
'last_failed_time': timedelta(hours=25),
'num_times_failed': 5
})
@ddt.unpack
@mock.patch(
'traffic_violations.services.twitter_service.TwitterEvent')
def test_find_and_respond_to_failed_twitter_events(self,
twitter_event_mock: MagicMock,
event_type: str,
expect_called: bool,
num_times_failed: int,
last_failed_time: timedelta = timedelta(minutes=0),
tweet_exists: bool = True):
db_id = 1
random_id = random.randint(1000000000000000000, 2000000000000000000)
user_handle = 'bdhowald'
user_id = random.randint(1000000000, 2000000000)
event_text = '@HowsMyDrivingNY abc1234:ny'
timestamp = random.randint(1500000000000, 1700000000000)
in_reply_to_message_id = random.randint(
10000000000000000000, 20000000000000000000)
location = 'Queens, NY'
responded_to = 0
user_mentions = [
{
'id': 123,
'id_str': '123',
'screen_name': 'HowsMyDrivingNY'
},
{
'id': 456,
'id_str': '456',
'screen_name': 'OtherUser'
},
{
'id': 789,
'id_str': '456',
'screen_name': 'SomeOtherUser'
}
]
twitter_event = TwitterEvent(
id=db_id,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
last_failed_at_time=(datetime.utcnow() - last_failed_time),
location=location,
num_times_failed=num_times_failed,
responded_to=responded_to,
user_handle=user_handle,
user_id=user_id,
user_mentions=user_mentions)
direct_message_lookup_request = AccountActivityAPIDirectMessage(
message=TwitterEvent(
id=1,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
user_handle=user_handle,
user_id=user_id,
user_favorited_non_follower_reply=False),
message_source=event_type)
status_lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
user_handle=user_handle,
user_id=user_id,
user_favorited_non_follower_reply=False),
message_source=event_type)
lookup_request = (direct_message_lookup_request if
event_type == 'direct_message' else status_lookup_request)
twitter_event_mock.get_all_by.side_effect = [[], [twitter_event]]
twitter_event_mock.query.filter_by().filter().count.return_value = 0
tweet_exists_mock = MagicMock(name='tweet_exists')
tweet_exists_mock.return_value = True if tweet_exists else False
self.tweeter.tweet_detection_service.tweet_exists = tweet_exists_mock
initiate_reply_mock = MagicMock(name='initiate_reply')
self.tweeter.aggregator.initiate_reply = initiate_reply_mock
build_reply_data_mock = MagicMock(name='build_reply_data')
build_reply_data_mock.return_value = lookup_request
self.tweeter.reply_argument_builder.build_reply_data = build_reply_data_mock
application_api_mock = MagicMock(name='application_api')
application_api_mock.get_follower_ids.return_value = ([user_id], (123, 0))
self.tweeter._app_api = application_api_mock
process_response_mock = MagicMock(name='process_response')
process_response_mock.return_value = random_id
self.tweeter._process_response = process_response_mock
self.tweeter._find_and_respond_to_twitter_events()
if expect_called:
self.tweeter.aggregator.initiate_reply.assert_called_with(
lookup_request=lookup_request)
else:
self.tweeter.aggregator.initiate_reply.assert_not_called()
self.tweeter.terminate_lookups()
@ddt.data({
'expect_called': True,
'minutes_ago': 20
}, {
'expect_called': False,
'minutes_ago': 10
})
@ddt.unpack
def test_get_follower_ids(self, expect_called: bool, minutes_ago: int):
application_api_mock = MagicMock(name='application_api')
application_api_mock.get_follower_ids.return_value = ([1], (0, 0))
self.tweeter._app_api = application_api_mock
self.tweeter._follower_ids_last_fetched = datetime.utcnow() - timedelta(
minutes=minutes_ago)
self.tweeter._get_follower_ids()
if expect_called:
application_api_mock.get_follower_ids.assert_called_with(cursor=-1)
else:
application_api_mock.get_follower_ids.assert_not_called()
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._is_production')
def test_process_response_direct_message(self, mocked_is_production):
""" Test direct message and new format """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
lookup_request = AccountActivityAPIDirectMessage(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny ny:hme6483',
event_type='direct_message',
user_handle=username,
user_id=30139847),
message_source='direct_message')
combined_message = "@bdhowald #NY_HME6483 has been queried 1 time.\n\nTotal parking and camera violation tickets: 15\n\n4 | No Standing - Day/Time Limits\n3 | No Parking - Street Cleaning\n1 | Failure To Display Meter Receipt\n1 | No Violation Description Available\n1 | Bus Lane Violation\n\n@bdhowald Parking and camera violation tickets for #NY_HME6483, cont'd:\n\n1 | Failure To Stop At Red Light\n1 | No Standing - Commercial Meter Zone\n1 | Expired Meter\n1 | Double Parking\n1 | No Angle Parking\n\n@bdhowald Violations by year for #NY_HME6483:\n\n10 | 2017\n15 | 2018\n\n@bdhowald Known fines for #NY_HME6483:\n\n$200.00 | Fined\n$125.00 | Outstanding\n$75.00 | Paid\n"
mocked_is_production.return_value = True
send_direct_message_mock = MagicMock('send_direct_message_mock')
application_api_mock = MagicMock(name='application_api')
application_api_mock.send_direct_message = send_direct_message_mock
self.tweeter._app_api = application_api_mock
self.tweeter._process_response(
request_object=lookup_request,
response_parts=[[combined_message]],
successful_lookup=True)
send_direct_message_mock.assert_called_with(
recipient_id=30139847,
text=combined_message)
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._recursively_process_status_updates')
def test_process_response_status_legacy_format(self,
recursively_process_status_updates_mock):
""" Test status and old format """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
response_parts = [['@BarackObama #PA_GLF7467 has been queried 1 time.\n\nTotal parking and camera violation tickets: 49\n\n17 | No Parking - Street Cleaning\n6 | Expired Meter\n5 | No Violation Description Available\n3 | Fire Hydrant\n3 | No Parking - Day/Time Limits\n', "@BarackObama Parking and camera violation tickets for #PA_GLF7467, cont'd:\n\n3 | Failure To Display Meter Receipt\n3 | School Zone Speed Camera Violation\n2 | No Parking - Except Authorized Vehicles\n2 | Bus Lane Violation\n1 | Failure To Stop At Red Light\n",
"@BarackObama Parking and camera violation tickets for #PA_GLF7467, cont'd:\n\n1 | No Standing - Day/Time Limits\n1 | No Standing - Except Authorized Vehicle\n1 | Obstructing Traffic Or Intersection\n1 | Double Parking\n", '@BarackObama Known fines for #PA_GLF7467:\n\n$1,000.00 | Fined\n$225.00 | Outstanding\n$775.00 | Paid\n']]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny plate:glf7467 state:pa',
event_type='status',
user_handle=username,
user_id=30139847,
user_mention_ids='813286,19834403,1230933768342528001,37687633,66379182',
user_mentions='@BarackObama @NYCMayor @GoodNYCMayor @NYC_DOT @NYCTSubway'
),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
is_production_mock = MagicMock(name='is_production')
is_production_mock.return_value = True
create_favorite_mock = MagicMock(name='is_production')
create_favorite_mock.return_value = True
application_api_mock = MagicMock(name='application_api')
application_api_mock.create_favorite = create_favorite_mock
self.tweeter._is_production = is_production_mock
self.tweeter._app_api = application_api_mock
reply_event_args['username'] = username
self.tweeter._process_response(
request_object=lookup_request,
response_parts=response_parts,
successful_lookup=True)
recursively_process_status_updates_mock.assert_called_with(
response_parts=response_parts,
message_id=message_id,
user_mention_ids=[
'813286',
'19834403',
'1230933768342528001',
'37687633',
'66379182'
])
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._recursively_process_status_updates')
def test_process_response_campaign_only_lookup(self,
recursively_process_status_updates_mock):
""" Test campaign-only lookup """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
campaign_hashtag = '#SaferSkillman'
campaign_tickets = random.randint(1000, 2000)
campaign_vehicles = random.randint(100, 200)
response_parts = [[(f"@{username} {'{:,}'.format(campaign_vehicles)} vehicles with a total of "
f"{'{:,}'.format(campaign_tickets)} tickets have been tagged with {campaign_hashtag}.\n\n")]]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text=f'@howsmydrivingny {campaign_hashtag}',
event_type='status',
user_handle=username,
user_id=30139847,
user_mention_ids='813286,19834403,1230933768342528001,37687633,66379182',
user_mentions='@BarackObama @NYCMayor @GoodNYCMayor @NYC_DOT @NYCTSubway'
),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(
request_object=lookup_request,
response_parts=response_parts,
successful_lookup=True)
recursively_process_status_updates_mock.assert_called_with(
response_parts=response_parts,
message_id=message_id,
user_mention_ids=[
'813286',
'19834403',
'1230933768342528001',
'37687633',
'66379182'
])
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._recursively_process_status_updates')
def test_process_response_with_search_status(self,
recursively_process_status_updates_mock):
""" Test plateless lookup """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
response_parts = [
['@' + username + ' I’d be happy to look that up for you!\n\nJust a reminder, the format is <state|province|territory>:<plate>, e.g. NY:abc1234']]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny plate dkr9364 state ny',
event_type='status',
user_handle=username,
user_id=30139847,
user_mention_ids='813286,19834403,1230933768342528001,37687633,66379182',
user_mentions='@BarackObama @NYCMayor @GoodNYCMayor @NYC_DOT @NYCTSubway'
),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(
request_object=lookup_request,
response_parts=response_parts,
successful_lookup=True)
recursively_process_status_updates_mock.assert_called_with(
response_parts=response_parts,
message_id=message_id,
user_mention_ids=[
'813286',
'19834403',
'1230933768342528001',
'37687633',
'66379182'
])
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._recursively_process_status_updates')
def test_process_response_with_direct_message_api_direct_message(self,
recursively_process_status_updates_mock):
""" Test plateless lookup """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
response_parts = [
['@' + username + " I think you're trying to look up a plate, but can't be sure.\n\nJust a reminder, the format is <state|province|territory>:<plate>, e.g. NY:abc1234"]]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny the state is ny',
event_type='status',
user_handle=username,
user_id=30139847,
user_mention_ids='813286,19834403,1230933768342528001,37687633,66379182',
user_mentions='@BarackObama @NYCMayor @GoodNYCMayor @NYC_DOT @NYCTSubway'
),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(
request_object=lookup_request,
response_parts=response_parts,
successful_lookup=True)
recursively_process_status_updates_mock.assert_called_with(
response_parts=response_parts,
message_id=message_id,
user_mention_ids=[
'813286',
'19834403',
'1230933768342528001',
'37687633',
'66379182'
])
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._recursively_process_status_updates')
def test_process_response_with_error(self,
recursively_process_status_updates_mock):
""" Test error handling """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny plate:glf7467 state:pa',
event_type='status',
user_handle=username,
user_id=30139847,
user_mention_ids='813286,19834403,1230933768342528001,37687633,66379182',
user_mentions='@BarackObama @NYCMayor @GoodNYCMayor @NYC_DOT @NYCTSubway'
),
message_source='status'
)
response_parts = [
['@' + username + " Sorry, I encountered an error. Tagging @bdhowald."]]
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(
request_object=lookup_request,
response_parts=response_parts,
successful_lookup=True)
recursively_process_status_updates_mock.assert_called_with(
response_parts=response_parts,
message_id=message_id,
user_mention_ids=[
'813286',
'19834403',
'1230933768342528001',
'37687633',
'66379182'
])
def test_recursively_compile_direct_messages(self):
str1 = 'Some stuff\n'
str2 = 'Some other stuff\nSome more Stuff'
str3 = 'Yet more stuff'
response_parts = [
[str1], str2, str3
]
result_str = "\n".join([str1, str2, str3])
self.assertEqual(self.tweeter._recursively_compile_direct_messages(
response_parts), result_str)
@ddt.data({
'response_parts': [
[
'Some stuff\n'
],
'Some other stuff\nSome more Stuff',
'Yet more stuff'
]
}, {
'response_parts': [
'Some stuff\n',
'Some other stuff\nSome more Stuff',
'Yet more stuff'
]
})
@ddt.unpack
@mock.patch(
'traffic_violations.services.twitter_service.LOG')
def test_recursively_process_status_updates(
self,
mocked_log: MagicMock,
response_parts: list[str]
) -> None:
original_id = 1
user_mention_ids = ['1','2','3']
update_status_mock = MagicMock(name='update_status')
update_status_mock.side_effect = [
MagicMock(id=123), MagicMock(id=456), MagicMock(id=789)
]
application_api_mock = MagicMock(name='application_api')
application_api_mock.update_status = update_status_mock
is_production_mock = MagicMock(name='is_production')
is_production_mock.return_value = True
self.tweeter._app_api = application_api_mock
self.tweeter._is_production = is_production_mock
self.tweeter._recursively_process_status_updates(
response_parts, original_id, user_mention_ids)
mocked_log.debug.assert_has_calls([
call("message_id: 123"),
call("message_id: 456"),
call("message_id: 789")
])
update_status_mock.assert_has_calls([
call(
status='Some stuff\n',
in_reply_to_status_id=original_id,
exclude_reply_user_ids=user_mention_ids
),
call(
status='Some other stuff\nSome more Stuff',
in_reply_to_status_id=123,
exclude_reply_user_ids=user_mention_ids
),
call(
status='Yet more stuff',
in_reply_to_status_id=456,
exclude_reply_user_ids=user_mention_ids
)
])
def setUp(self):
self.tweeter = TrafficViolationsTweeter()
self.log_patcher = mock.patch(
'traffic_violations.services.twitter_service.LOG')
self.mocked_log = self.log_patcher.start()
class TestTrafficViolationsTweeter(unittest.TestCase):
def setUp(self):
self.tweeter = TrafficViolationsTweeter()
self.log_patcher = mock.patch(
'traffic_violations.services.twitter_service.LOG')
self.mocked_log = self.log_patcher.start()
def tearDown(self):
self.log_patcher.stop()
def test_find_and_respond_to_requests(self):
direct_messages_mock = MagicMock(
name='_find_and_respond_to_missed_direct_messages')
statuses_mock = MagicMock(
name='_find_and_respond_to_missed_statuses')
twitter_events_mock = MagicMock(
name='_find_and_respond_to_twitter_events')
self.tweeter._find_and_respond_to_twitter_events = twitter_events_mock
self.tweeter._find_and_respond_to_missed_direct_messages = direct_messages_mock
self.tweeter._find_and_respond_to_missed_statuses = statuses_mock
self.tweeter._find_and_respond_to_requests()
direct_messages_mock.assert_called_with()
statuses_mock.assert_called_with()
twitter_events_mock.assert_called_with()
@mock.patch(
'traffic_violations.services.twitter_service.TwitterEvent')
def test_find_and_respond_to_missed_direct_messages(self, twitter_event_mock):
db_id = 1
event_id = random.randint(10000000000000000000, 20000000000000000000)
event_text = 'abc1234:ny'
event_type = 'direct_message'
timestamp = random.randint(1500000000000, 1700000000000)
user_handle = 'bdhowald'
user_id = random.randint(1000000000, 2000000000)
new_twitter_event = TwitterEvent(
created_at=timestamp + 1,
detected_via_account_activity_api=False,
event_id=event_id + 1,
event_text=event_text + '!',
event_type=event_type,
in_reply_to_message_id=None,
location=None,
responded_to=False,
user_handle=user_handle,
user_id=user_id,
user_mentions='')
message_needing_event = MagicMock(
id=event_id + 1,
created_timestamp = timestamp + 1,
message_create={
'message_data': {
'entities': {
'user_mentions': [
{
'screen_name': user_handle
}
]
},
'text': event_text + '!'
},
'sender_id': f'{user_id}'
},
name='message_needing_event')
message_not_needing_event = MagicMock(
id=event_id - 1,
created_timestamp = timestamp - 1,
message_create={
'message_data': {
'entities': {
'user_mentions': [{'screen_name': user_handle}]
},
'text': event_text,
},
'sender_id': f'{user_id}'
},
name='message_not_needing_event')
sender = MagicMock(
id=user_id,
id_str=f'{user_id}',
name='sender',
screen_name=user_handle)
twitter_event_mock.return_value = new_twitter_event
twitter_event_mock.query.filter.return_value.first.side_effect = [
None, message_not_needing_event]
api_mock = MagicMock(name='api')
api_mock.list_direct_messages.return_value = [
message_needing_event, message_not_needing_event]
api_mock.lookup_users.return_value = [sender]
self.tweeter.client_api = api_mock
self.tweeter._find_and_respond_to_missed_direct_messages()
twitter_event_mock.query.session.add.assert_called_once_with(new_twitter_event)
twitter_event_mock.query.session.commit.assert_called_once_with()
self.mocked_log.debug.assert_called_with('Found 1 direct message that was previously undetected.')
@mock.patch(
'traffic_violations.services.twitter_service.TwitterEvent')
def test_find_and_respond_to_missed_statuses(self, twitter_event_mock):
db_id = 1
event_id = random.randint(10000000000000000000, 20000000000000000000)
event_text = '@HowsMyDrivingNY @bdhowald abc1234:ny'
event_type = 'status'
in_reply_to_message_id = random.randint(
10000000000000000000, 20000000000000000000)
location = 'Queens, NY'
now = datetime.utcnow()
place = MagicMock(
full_name=location,
name='place')
user_id = random.randint(1000000000, 2000000000)
user = MagicMock(
id=user_id,
name='user')
user_handle = 'bdhowald'
older_twitter_event = TwitterEvent(
id=db_id,
created_at=now.replace(tzinfo=pytz.timezone('UTC')).timestamp() * 1000,
detected_via_account_activity_api=False,
event_id=event_id,
event_text=event_text,
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
location=location,
responded_to=True,
user_handle=user_handle,
user_id=user_id,
user_mentions=user_handle)
new_twitter_event = TwitterEvent(
created_at=now.replace(tzinfo=pytz.timezone('UTC')).timestamp() * 1000,
detected_via_account_activity_api=False,
event_id=event_id + 1,
event_text=event_text + '!',
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
location=location,
responded_to=False,
user_handle=user_handle,
user_id=user_id,
user_mentions=user_handle)
status_needing_event = MagicMock(
id=event_id + 1,
created_at = now,
entities={
'user_mentions': [{
'screen_name': user_handle
}]
},
full_text=event_text + '!',
in_reply_to_message_id=in_reply_to_message_id,
name='status_needing_event',
place=place,
user=user)
status_not_needing_event = MagicMock(
id=event_id - 1,
created_at = now - relativedelta(minutes=1),
entities={
'user_mentions': [{
'screen_name': user_handle
}]
},
full_text=event_text,
in_reply_to_message_id=in_reply_to_message_id,
name='status_not_needing_event',
place=place,
user=user)
twitter_event_mock.return_value = new_twitter_event
twitter_event_mock.query.filter.order_by.first.return_value = older_twitter_event
twitter_event_mock.query.filter.return_value.first.side_effect = [
None, status_not_needing_event]
api_mock = MagicMock(name='api')
api_mock.mentions_timeline.return_value = [
status_needing_event, status_not_needing_event]
self.tweeter.client_api = api_mock
self.tweeter._find_and_respond_to_missed_statuses()
twitter_event_mock.query.session.add.assert_called_once_with(new_twitter_event)
twitter_event_mock.query.session.commit.assert_called_once_with()
self.mocked_log.debug.assert_called_with('Found 1 status that was previously undetected.')
@mock.patch(
'traffic_violations.services.twitter_service.TwitterEvent')
def test_find_and_respond_to_missed_statuses_with_no_undetected_events(self, twitter_event_mock):
twitter_event_mock.query.filter.order_by.first.return_value = None
self.tweeter._find_and_respond_to_missed_statuses()
twitter_event_mock.query.session.add.assert_not_called()
twitter_event_mock.query.session.commit.assert_not_called()
@mock.patch(
'traffic_violations.services.twitter_service.TwitterEvent')
def test_find_and_respond_to_twitter_events(self, twitter_event_mock):
db_id = 1
random_id = random.randint(10000000000000000000, 20000000000000000000)
event_type = 'status'
user_handle = 'bdhowald'
user_id = random.randint(1000000000, 2000000000)
event_text = '@HowsMyDrivingNY abc1234:ny'
timestamp = random.randint(1500000000000, 1700000000000)
in_reply_to_message_id = random.randint(
10000000000000000000, 20000000000000000000)
location = 'Queens, NY'
responded_to = 0
user_mentions = '@HowsMyDrivingNY'
twitter_event = TwitterEvent(
id=db_id,
created_at=timestamp,
event_id=random_id,
event_text=event_text,
event_type=event_type,
in_reply_to_message_id=in_reply_to_message_id,
location=location,
responded_to=responded_to,
user_handle=user_handle,
user_id=user_id,
user_mentions=user_mentions)
lookup_request = AccountActivityAPIDirectMessage(
message=TwitterEvent(
id=1,
created_at=timestamp,
event_id=random_id,
event_text='@howsmydrivingny ny:hme6483',
event_type='direct_message',
user_handle=user_handle,
user_id=30139847),
message_source='api')
twitter_event_mock.get_all_by.return_value = [twitter_event]
twitter_event_mock.query.filter_by().filter().count.return_value = 0
initiate_reply_mock = MagicMock(name='initiate_reply')
self.tweeter.aggregator.initiate_reply = initiate_reply_mock
build_reply_data_mock = MagicMock(name='build_reply_data')
build_reply_data_mock.return_value = lookup_request
self.tweeter.reply_argument_builder.build_reply_data = build_reply_data_mock
self.tweeter._find_and_respond_to_twitter_events()
self.tweeter.aggregator.initiate_reply.assert_called_with(
lookup_request=lookup_request)
def test_is_production(self):
self.assertEqual(self.tweeter._is_production(),
(os.environ.get('ENV') == 'production'))
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._is_production')
def test_process_response_direct_message(self, mocked_is_production):
""" Test direct message and new format """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
lookup_request = AccountActivityAPIDirectMessage(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny ny:hme6483',
event_type='direct_message',
user_handle=username,
user_id=30139847),
message_source='direct_message')
combined_message = "@bdhowald #NY_HME6483 has been queried 1 time.\n\nTotal parking and camera violation tickets: 15\n\n4 | No Standing - Day/Time Limits\n3 | No Parking - Street Cleaning\n1 | Failure To Display Meter Receipt\n1 | No Violation Description Available\n1 | Bus Lane Violation\n\n@bdhowald Parking and camera violation tickets for #NY_HME6483, cont'd:\n\n1 | Failure To Stop At Red Light\n1 | No Standing - Commercial Meter Zone\n1 | Expired Meter\n1 | Double Parking\n1 | No Angle Parking\n\n@bdhowald Violations by year for #NY_HME6483:\n\n10 | 2017\n15 | 2018\n\n@bdhowald Known fines for #NY_HME6483:\n\n$200.00 | Fined\n$125.00 | Outstanding\n$75.00 | Paid\n"
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': [[combined_message]],
'success': True,
'successful_lookup': True,
'username': username
}
mocked_is_production.return_value = True
send_direct_message_mock = MagicMock('send_direct_message_mock')
api_mock = MagicMock(name='api')
api_mock.send_direct_message = send_direct_message_mock
self.tweeter.app_api = api_mock
self.tweeter._process_response(reply_event_args)
send_direct_message_mock.assert_called_with(
recipient_id=30139847,
text=combined_message)
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._recursively_process_status_updates')
def test_process_response_status_legacy_format(self,
recursively_process_status_updates_mock):
""" Test status and old format """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
response_parts = [['@BarackObama #PA_GLF7467 has been queried 1 time.\n\nTotal parking and camera violation tickets: 49\n\n17 | No Parking - Street Cleaning\n6 | Expired Meter\n5 | No Violation Description Available\n3 | Fire Hydrant\n3 | No Parking - Day/Time Limits\n', "@BarackObama Parking and camera violation tickets for #PA_GLF7467, cont'd:\n\n3 | Failure To Display Meter Receipt\n3 | School Zone Speed Camera Violation\n2 | No Parking - Except Authorized Vehicles\n2 | Bus Lane Violation\n1 | Failure To Stop At Red Light\n",
"@BarackObama Parking and camera violation tickets for #PA_GLF7467, cont'd:\n\n1 | No Standing - Day/Time Limits\n1 | No Standing - Except Authorized Vehicle\n1 | Obstructing Traffic Or Intersection\n1 | Double Parking\n", '@BarackObama Known fines for #PA_GLF7467:\n\n$1,000.00 | Fined\n$225.00 | Outstanding\n$775.00 | Paid\n']]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny plate:glf7467 state:pa',
event_type='status',
user_handle=username,
user_id=30139847),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
is_production_mock = MagicMock(name='is_production')
is_production_mock.return_value = True
create_favorite_mock = MagicMock(name='is_production')
create_favorite_mock.return_value = True
api_mock = MagicMock(name='api')
api_mock.create_favorite = create_favorite_mock
self.tweeter._is_production = is_production_mock
self.tweeter.app_api = api_mock
reply_event_args['username'] = username
self.tweeter._process_response(reply_event_args)
recursively_process_status_updates_mock.assert_called_with(
response_parts, message_id, username)
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._recursively_process_status_updates')
def test_process_response_campaign_only_lookup(self,
recursively_process_status_updates_mock):
""" Test campaign-only lookup """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
campaign_hashtag = '#SaferSkillman'
campaign_tickets = random.randint(1000, 2000)
campaign_vehicles = random.randint(100, 200)
response_parts = [[(f"@{username} {'{:,}'.format(campaign_vehicles)} vehicles with a total of "
f"{'{:,}'.format(campaign_tickets)} tickets have been tagged with {campaign_hashtag}.\n\n")]]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text=f'@howsmydrivingny {campaign_hashtag}',
event_type='status',
user_handle=username,
user_id=30139847),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(reply_event_args)
recursively_process_status_updates_mock.assert_called_with(
response_parts, message_id, username)
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._recursively_process_status_updates')
def test_process_response_with_search_status(self,
recursively_process_status_updates_mock):
""" Test plateless lookup """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
response_parts = [
['@' + username + ' I’d be happy to look that up for you!\n\nJust a reminder, the format is <state|province|territory>:<plate>, e.g. NY:abc1234']]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny plate dkr9364 state ny',
event_type='status',
user_handle=username,
user_id=30139847),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(reply_event_args)
recursively_process_status_updates_mock.assert_called_with(
response_parts, message_id, username)
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._recursively_process_status_updates')
def test_process_response_with_direct_message_api_direct_message(self,
recursively_process_status_updates_mock):
""" Test plateless lookup """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
response_parts = [
['@' + username + " I think you're trying to look up a plate, but can't be sure.\n\nJust a reminder, the format is <state|province|territory>:<plate>, e.g. NY:abc1234"]]
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny the state is ny',
event_type='status',
user_handle=username,
user_id=30139847),
message_source='status')
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(reply_event_args)
recursively_process_status_updates_mock.assert_called_with(
response_parts, message_id, username)
@mock.patch(
'traffic_violations.services.twitter_service.TrafficViolationsTweeter._recursively_process_status_updates')
def test_process_response_with_error(self,
recursively_process_status_updates_mock):
""" Test error handling """
username = '******'
message_id = random.randint(1000000000000000000, 2000000000000000000)
lookup_request = AccountActivityAPIStatus(
message=TwitterEvent(
id=1,
created_at=random.randint(1_000_000_000_000, 2_000_000_000_000),
event_id=message_id,
event_text='@howsmydrivingny plate:glf7467 state:pa',
event_type='status',
user_handle=username,
user_id=30139847),
message_source='status'
)
response_parts = [
['@' + username + " Sorry, I encountered an error. Tagging @bdhowald."]]
reply_event_args = {
'error_on_lookup': False,
'request_object': lookup_request,
'response_parts': response_parts,
'success': True,
'successful_lookup': True,
'username': username
}
reply_event_args['username'] = username
self.tweeter._process_response(reply_event_args)
recursively_process_status_updates_mock.assert_called_with(
response_parts, message_id, username)
def test_recursively_process_direct_messages(self):
str1 = 'Some stuff\n'
str2 = 'Some other stuff\nSome more Stuff'
str3 = 'Yet more stuff'
response_parts = [
[str1], str2, str3
]
result_str = "\n".join([str1, str2, str3])
self.assertEqual(self.tweeter._recursively_process_direct_messages(
response_parts), result_str)
def test_recursively_process_status_updates(self):
str1 = 'Some stuff\n'
str2 = 'Some other stuff\nSome more Stuff'
str3 = 'Yet more stuff'
original_id = 1
response_parts = [
[str1], str2, str3
]
api_mock = MagicMock(name='api')
api_mock.update_status = inc
is_production_mock = MagicMock(name='is_production')
is_production_mock.return_value = True
self.tweeter.app_api = api_mock
self.tweeter._is_production = is_production_mock
self.assertEqual(self.tweeter._recursively_process_status_updates(
response_parts, original_id, 'BarackObama'), original_id + len(response_parts))
self.assertEqual(self.tweeter._recursively_process_status_updates(
response_parts, original_id, 'BarackObama'), original_id + len(response_parts))
def perform(self, *args, **kwargs):
is_dry_run: bool = kwargs.get('is_dry_run') or False
days_in_period = 22.0
days_in_year = 366.0
periods_in_year = days_in_year / days_in_period
eastern = pytz.timezone('US/Eastern')
start_date = datetime(2020, 3, 10)
end_date = start_date + timedelta(days=days_in_period, seconds=-1)
tweeter = TrafficViolationsTweeter()
nyc_open_data_service: OpenDataService = OpenDataService()
covid_19_camera_offender_raw_data: List[Dict[
str,
str]] = nyc_open_data_service.lookup_covid_19_camera_violations()
for vehicle in covid_19_camera_offender_raw_data:
plate = vehicle['plate']
state = vehicle['state']
offender: Optional[
Covid19CameraOffender] = Covid19CameraOffender.get_by(
plate_id=plate, state=state)
if offender:
LOG.debug(
f'COVID-19 speeder - {L10N.VEHICLE_HASHTAG.format(state, plate)} '
f"with {vehicle['count']} camera violations has been seen before."
)
continue
LOG.debug(
f'COVID-19 speeder - {L10N.VEHICLE_HASHTAG.format(state, plate)} '
f"with {vehicle['count']} camera violations has not been seen before."
)
plate_query: PlateQuery = PlateQuery(
created_at=datetime.now(),
message_source=LookupSource.API,
plate=plate,
plate_types=None,
state=state)
response: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query, since=start_date, until=end_date)
plate_lookup: OpenDataServicePlateLookup = response.data
red_light_camera_violations = 0
speed_camera_violations = 0
for violation_type_summary in plate_lookup.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary[
'title'] == self.RED_LIGHT_CAMERA_VIOLATION_DESCRIPTION:
red_light_camera_violations = violation_count
if violation_type_summary[
'title'] == self.SPEED_CAMERA_VIOLATION_DESCRIPTION:
speed_camera_violations = violation_count
total_camera_violations = (red_light_camera_violations +
speed_camera_violations)
vehicle_hashtag = L10N.VEHICLE_HASHTAG.format(state, plate)
red_light_camera_violations_string = (
f'{red_light_camera_violations} | Red Light Camera Violations\n'
if red_light_camera_violations > 0 else '')
speed_camera_violations_string = (
f'{speed_camera_violations} | Speed Safety Camera Violations\n'
if speed_camera_violations > 0 else '')
covid_19_reckless_driver_string = (
f"From {start_date.strftime('%B %-d, %Y')} to "
f"{end_date.strftime('%B %-d, %Y')}, {vehicle_hashtag} "
f'received {total_camera_violations} camera '
f'violations:\n\n'
f'{red_light_camera_violations_string}'
f'{speed_camera_violations_string}')
dval_string = (
'At this rate, this vehicle will receive '
f'{round(periods_in_year * total_camera_violations)} '
'speed safety camera violations over '
'a year, qualifying it for towing or booting under '
'@bradlander\'s Dangerous Vehicle Abatement Law and '
'requiring its driver to take a course on the consequences '
'of reckless driving.')
speeding_string = (
'With such little traffic, many drivers are speeding '
'regularly, putting New Yorkers at increased risk of '
'ending up in a hospital at a time our hospitals are '
'stretched to their limits. It\'s also hard to practice '
'social distancing when walking on our narrow sidewalks.')
open_streets_string = (
'Other cities are eating our lunch, @NYCMayor:\n\n'
f'{random.choice(self.COVID_19_OPEN_STREETS_TWEETS)}')
messages: List[str] = [
covid_19_reckless_driver_string, dval_string,
[speeding_string, open_streets_string]
]
if not is_dry_run:
success: bool = tweeter.send_status(
message_parts=messages,
on_error_message=(
f'Error printing COVID-19 reckless driver update. '
f'Tagging @bdhowald.'))
if success:
offender = Covid19CameraOffender(
plate_id=plate,
state=state,
red_light_camera_violations=red_light_camera_violations,
speed_camera_violations=speed_camera_violations)
Covid19CameraOffender.query.session.add(offender)
try:
Covid19CameraOffender.query.session.commit()
LOG.debug('COVID-19 Reckless driver retrospective job '
'ran successfully.')
except:
tweeter.send_status(message_parts=[(
f'Error printing COVID-19 reckless driver update. '
f'Tagging @bdhowald.')])
# Only do one at a time.
break
else:
print(covid_19_reckless_driver_string)
print(dval_string)
print(speeding_string)
print(open_streets_string)
break
def perform(self, *args, **kwargs):
""" Tweet out daily summary of yesterday's lookups """
is_dry_run: bool = kwargs.get('is_dry_run') or False
tweeter = TrafficViolationsTweeter()
utc = pytz.timezone('UTC')
eastern = pytz.timezone('US/Eastern')
today = datetime.now(eastern).date()
midnight_yesterday = (
eastern.localize(datetime.combine(today, time.min)) -
timedelta(days=1)).astimezone(utc)
end_of_yesterday = (
eastern.localize(datetime.combine(today, time.min)) -
timedelta(seconds=1)).astimezone(utc)
# find all of yesterday's lookups, using only the most
# recent of yesterday's queries for a vehicle.
subquery = PlateLookup.query.session.query(
PlateLookup.plate, PlateLookup.state,
func.max(
PlateLookup.id).label('most_recent_vehicle_lookup')).filter(
and_(
PlateLookup.created_at >= midnight_yesterday,
PlateLookup.created_at <= end_of_yesterday,
PlateLookup.count_towards_frequency == True)).group_by(
PlateLookup.plate,
PlateLookup.state).subquery('subquery')
full_query = PlateLookup.query.join(
subquery,
(PlateLookup.id == subquery.c.most_recent_vehicle_lookup))
yesterdays_lookups: List[PlateLookup] = full_query.all()
num_lookups: int = len(yesterdays_lookups)
ticket_counts: int = [
lookup.num_tickets for lookup in yesterdays_lookups
]
total_tickets: int = sum(ticket_counts)
num_empty_lookups: int = len([
lookup for lookup in yesterdays_lookups if lookup.num_tickets == 0
])
num_reckless_drivers: int = len([
lookup for lookup in yesterdays_lookups
if lookup.boot_eligible == True
])
total_reckless_drivers = PlateLookup.query.session.query(
PlateLookup.plate, PlateLookup.state).distinct().filter(
and_(PlateLookup.boot_eligible == True,
PlateLookup.count_towards_frequency)).count()
lookups_summary_string = (
f'On {midnight_yesterday.strftime("%A, %B %-d, %Y")}, '
f"users requested {num_lookups} lookup{L10N.pluralize(num_lookups)}. "
)
if num_lookups > 0:
median = statistics.median(ticket_counts)
lookups_summary_string += (
f"{'That vehicle has' if num_lookups == 1 else 'Collectively, those vehicles have'} "
f"received {'{:,}'.format(total_tickets)} ticket{L10N.pluralize(total_tickets)} "
f"for an average of {round(total_tickets / num_lookups, 2)} ticket{L10N.pluralize(total_tickets / num_lookups)} "
f"and a median of {median} ticket{L10N.pluralize(median)} per vehicle. "
f"{num_empty_lookups} lookup{L10N.pluralize(num_empty_lookups)} returned no tickets."
)
reckless_drivers_summary_string = (
f"{num_reckless_drivers} {'vehicle was' if num_reckless_drivers == 1 else 'vehicles were'} "
f"eligible to be booted or impounded under @bradlander's "
f"proposed legislation ({'{:,}'.format(total_reckless_drivers)} such lookups "
f"since June 6, 2018).")
messages: List[str] = [
lookups_summary_string, reckless_drivers_summary_string
]
if not is_dry_run:
success: bool = tweeter.send_status(
message_parts=messages,
on_error_message=(f'Error printing daily summary. '
f'Tagging @bdhowald.'))
if success:
LOG.debug('Daily summary plate ran successfully.')
def perform(self, *args, **kwargs):
is_dry_run: bool = kwargs.get('is_dry_run') or False
start_date = datetime.datetime(2020, 3, 10, 0, 0)
end_date = datetime.datetime(2021, 11, 26, 23, 59, 59)
days_in_period = (end_date - start_date).days
num_years = days_in_period / 365.0
tweeter = TrafficViolationsTweeter()
nyc_open_data_service: OpenDataService = OpenDataService()
covid_19_camera_offender_raw_data: list[dict[str, str]] = nyc_open_data_service.lookup_covid_19_camera_violations()
for vehicle in covid_19_camera_offender_raw_data:
plate = vehicle['plate']
state = vehicle['state']
offender: Optional[Covid19CameraOffender] = Covid19CameraOffender.get_by(
plate_id=plate,
state=state,
count_as_queried=True)
if offender:
LOG.debug(f'COVID-19 speeder - {L10N.VEHICLE_HASHTAG.format(state, plate)} '
f"with {vehicle['total_camera_violations']} camera violations has been seen before.")
continue
LOG.debug(f'COVID-19 speeder - {L10N.VEHICLE_HASHTAG.format(state, plate)} '
f"with {vehicle['total_camera_violations']} camera violations has not been seen before.")
plate_query: PlateQuery = PlateQuery(created_at=datetime.datetime.now(),
message_source=LookupSource.API,
plate=plate,
plate_types=None,
state=state)
response: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query,
since=start_date,
until=end_date)
plate_lookup: OpenDataServicePlateLookup = response.data
red_light_camera_violations = 0
speed_camera_violations = 0
for violation_type_summary in plate_lookup.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary['title'] == self.RED_LIGHT_CAMERA_VIOLATION_DESCRIPTION:
red_light_camera_violations = violation_count
if violation_type_summary['title'] == self.SPEED_CAMERA_VIOLATION_DESCRIPTION:
speed_camera_violations = violation_count
speed_camera_violations_per_year = speed_camera_violations / num_years
# If this driver doesn't meet the threshold, continue
if speed_camera_violations_per_year < self.DVAL_SPEED_CAMERA_THRESHOLD:
continue
total_camera_violations = speed_camera_violations + red_light_camera_violations
vehicle_hashtag = L10N.VEHICLE_HASHTAG.format(
state, plate)
red_light_camera_violations_string = (
f'{red_light_camera_violations} | Red Light Camera Violations\n'
if red_light_camera_violations > 0 else '')
speed_camera_violations_string = (
f'{speed_camera_violations} | Speed Safety Camera Violations\n'
if speed_camera_violations > 0 else '')
covid_19_reckless_driver_string = (
f"From {start_date.strftime('%B %-d, %Y')} to "
f"{end_date.strftime('%B %-d, %Y')}, {vehicle_hashtag} "
f'received {total_camera_violations} camera '
f'violations:\n\n'
f'{red_light_camera_violations_string}'
f'{speed_camera_violations_string}')
dval_string = (
'This vehicle has received an average of '
f'{round(speed_camera_violations / num_years, 1)} '
'speed safety camera violations per year, '
'qualifying it for towing or booting under '
'@bradlander\'s Dangerous Vehicle Abatement Law and '
'requiring its driver to take a course on the consequences '
'of reckless driving.')
messages: list[str] = [covid_19_reckless_driver_string, dval_string]
if not is_dry_run:
success: bool = tweeter.send_status(
message_parts=messages,
on_error_message=(
f'Error printing COVID-19 reckless driver update. '
f'Tagging @bdhowald.'))
if success:
offender = Covid19CameraOffender(plate_id=plate,
state=state,
red_light_camera_violations=red_light_camera_violations,
speed_camera_violations=speed_camera_violations)
Covid19CameraOffender.query.session.add(offender)
try:
Covid19CameraOffender.query.session.commit()
LOG.debug('COVID-19 Reckless driver retrospective job '
'ran successfully.')
except:
tweeter.send_status(message_parts=[(
f'Error printing COVID-19 reckless driver update. '
f'Tagging @bdhowald.')])
# Only do one at a time.
break
else:
print(covid_19_reckless_driver_string)
print(dval_string)
break