def update_lookups(self, lookups: List[PlateLookup]):
for previous_lookup in lookups:
now = datetime.utcnow()
plate_query: PlateQuery = PlateQuery(created_at=now,
message_source=previous_lookup.message_source,
plate=previous_lookup.plate,
plate_types=previous_lookup.plate_types,
state=previous_lookup.state)
nyc_open_data_service: OpenDataService = OpenDataService()
open_data_response: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query,
until=previous_lookup.created_at)
open_data_plate_lookup: OpenDataServicePlateLookup = open_data_response.data
for violation_type_summary in open_data_plate_lookup.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary['title'] == 'Bus Lane Violation':
previous_lookup.bus_lane_camera_violations = violation_count
if violation_type_summary['title'] == 'Failure To Stop At Red Light':
previous_lookup.red_light_camera_violations = violation_count
elif violation_type_summary['title'] == 'School Zone Speed Camera Violation':
previous_lookup.speed_camera_violations = violation_count
if not previous_lookup.bus_lane_camera_violations:
previous_lookup.bus_lane_camera_violations = 0
if not previous_lookup.red_light_camera_violations:
previous_lookup.red_light_camera_violations = 0
if not previous_lookup.speed_camera_violations:
previous_lookup.speed_camera_violations = 0
LOG.debug(f'updating lookup {previous_lookup.id}')
print(f'updating lookup {previous_lookup.id}')
def _perform_plate_lookup(
self, campaigns: List[Campaign], plate_query: PlateQuery,
unique_identifier: str) -> OpenDataServiceResponse:
LOG.debug('Performing lookup for plate.')
nyc_open_data_service: OpenDataService = OpenDataService()
open_data_response: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query)
LOG.debug(f'Violation data: {open_data_response}')
if open_data_response.success:
open_data_plate_lookup: OpenDataServicePlateLookup = open_data_response.data
bus_lane_camera_violations = 0
red_light_camera_violations = 0
speed_camera_violations = 0
for violation_type_summary in open_data_plate_lookup.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary['title'] == 'Bus Lane Violation':
bus_lane_camera_violations = violation_count
if violation_type_summary[
'title'] == 'Failure To Stop At Red Light':
red_light_camera_violations = violation_count
elif violation_type_summary[
'title'] == 'School Zone Speed Camera Violation':
speed_camera_violations = violation_count
camera_streak_data: CameraStreakData = open_data_plate_lookup.camera_streak_data
# If this came from message, add it to the plate_lookups table.
if plate_query.message_source and plate_query.message_id and plate_query.created_at:
new_lookup = PlateLookup(
boot_eligible=camera_streak_data.max_streak >= 5
if camera_streak_data else False,
bus_lane_camera_violations=bus_lane_camera_violations,
created_at=plate_query.created_at,
message_id=plate_query.message_id,
message_source=plate_query.message_source,
num_tickets=open_data_plate_lookup.num_violations,
plate=plate_query.plate,
plate_types=plate_query.plate_types,
red_light_camera_violations=red_light_camera_violations,
speed_camera_violations=speed_camera_violations,
state=plate_query.state,
unique_identifier=unique_identifier,
username=plate_query.username)
# Iterate through included campaigns to tie lookup to each
for campaign in campaigns:
# insert join record for campaign lookup
new_lookup.campaigns.append(campaign)
# Insert plate lookup
PlateLookup.query.session.add(new_lookup)
PlateLookup.query.session.commit()
else:
LOG.info(f'open data plate lookup failed')
return open_data_response
def setUp(self):
self.open_data_service = OpenDataService()
def perform(self, *args, **kwargs):
is_dry_run: bool = kwargs.get('is_dry_run') or False
use_dvaa_thresholds: bool = kwargs.get('use_dvaa_thresholds') or False
use_only_visible_tweets: bool = kwargs.get('use_only_visible_tweets') or False
tweet_detection_service = TweetDetectionService()
tweeter = TrafficViolationsTweeter()
eastern = pytz.timezone('US/Eastern')
utc = pytz.timezone('UTC')
now = datetime.now()
# If today is leap day, there were no lookups a year ago today
if (now.day == self.LEAP_DAY_DATE and
now.month == self.LEAP_DAY_MONTH):
return
# If today is March 1, and last year was a leap year, show lookups
# from the previous February 29.
one_year_after_leap_day = True if (
now.day == self.POST_LEAP_DAY_DATE and
now.month == self.POST_LEAP_DAY_MONTH and
self._is_leap_year(now.year - 1)) else False
top_of_the_hour_last_year = now.replace(
microsecond=0,
minute=0,
second=0) - relativedelta(years=1)
top_of_the_next_hour_last_year = (
top_of_the_hour_last_year + relativedelta(hours=1))
if use_dvaa_thresholds:
threshold_attribute: str = 'boot_eligible_under_dvaa_threshold'
else:
threshold_attribute: str = 'boot_eligible_under_rdaa_threshold'
base_query: list[Tuple[int]] = PlateLookup.query.session.query(
func.max(PlateLookup.id).label('most_recent_vehicle_lookup')
).filter(
or_(
and_(PlateLookup.created_at >= top_of_the_hour_last_year,
PlateLookup.created_at < top_of_the_next_hour_last_year,
getattr(PlateLookup, threshold_attribute) == True,
PlateLookup.count_towards_frequency == True),
and_(one_year_after_leap_day,
PlateLookup.created_at >= (top_of_the_hour_last_year - relativedelta(days=1)),
PlateLookup.created_at < (top_of_the_next_hour_last_year - relativedelta(days=1)),
getattr(PlateLookup, threshold_attribute) == True,
PlateLookup.count_towards_frequency == True)
)
)
if use_only_visible_tweets:
base_query = base_query.filter(
PlateLookup.message_source == lookup_sources.LookupSource.STATUS.value)
recent_plate_lookup_ids = base_query.group_by(
PlateLookup.plate,
PlateLookup.state
).all()
lookup_ids_to_update: list[int] = [id[0] for id in recent_plate_lookup_ids]
lookups_to_update: list[PlateLookup] = PlateLookup.get_all_in(
id=lookup_ids_to_update)
if not lookups_to_update:
LOG.debug(f'No vehicles for which to perform retrospective job '
f'between {top_of_the_hour_last_year} and '
f'and {top_of_the_next_hour_last_year}.')
for previous_lookup in lookups_to_update:
LOG.debug(f'Performing retrospective job for '
f'{L10N.VEHICLE_HASHTAG.format(previous_lookup.state, previous_lookup.plate)} ')
plate_query: PlateQuery = PlateQuery(created_at=now,
message_source=previous_lookup.message_source,
plate=previous_lookup.plate,
plate_types=previous_lookup.plate_types,
state=previous_lookup.state)
nyc_open_data_service: OpenDataService = OpenDataService()
data_before_query: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query,
until=previous_lookup.created_at)
lookup_before_query: OpenDataServicePlateLookup = data_before_query.data
camera_streak_data_before_query: CameraStreakData = lookup_before_query.camera_streak_data['Mixed']
data_after_query: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query,
since=previous_lookup.created_at,
until=previous_lookup.created_at + relativedelta(years=1))
lookup_after_query: OpenDataServicePlateLookup = data_after_query.data
new_bus_lane_camera_violations: Optional[int] = None
new_speed_camera_violations: Optional[int] = None
new_red_light_camera_violations: Optional[int] = None
for violation_type_summary in lookup_after_query.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary['title'] == 'Bus Lane Violation':
new_bus_lane_camera_violations = violation_count
if violation_type_summary['title'] == 'Failure To Stop At Red Light':
new_red_light_camera_violations = violation_count
if violation_type_summary['title'] == 'School Zone Speed Camera Violation':
new_speed_camera_violations = violation_count
if new_bus_lane_camera_violations is None:
new_bus_lane_camera_violations = 0
if new_red_light_camera_violations is None:
new_red_light_camera_violations = 0
if new_speed_camera_violations is None:
new_speed_camera_violations = 0
new_boot_eligible_violations = (new_red_light_camera_violations +
new_speed_camera_violations)
if new_boot_eligible_violations > 0:
vehicle_hashtag = L10N.VEHICLE_HASHTAG.format(
previous_lookup.state, previous_lookup.plate)
previous_lookup_created_at = utc.localize(
previous_lookup.created_at)
previous_lookup_date = previous_lookup_created_at.astimezone(eastern).strftime(
L10N.REPEAT_LOOKUP_DATE_FORMAT)
previous_lookup_time = previous_lookup_created_at.astimezone(eastern).strftime(
L10N.REPEAT_LOOKUP_TIME_FORMAT)
red_light_camera_violations_string = (
f'{new_red_light_camera_violations} | Red Light Camera Violations\n'
if new_red_light_camera_violations > 0 else '')
speed_camera_violations_string = (
f'{new_speed_camera_violations} | Speed Safety Camera Violations\n'
if new_speed_camera_violations > 0 else '')
reckless_driver_summary_string = (
f'{vehicle_hashtag} was originally '
f'queried on {previous_lookup_date} '
f'at {previous_lookup_time}')
# assume we can't link
can_link_tweet = False
# Where did this come from?
if previous_lookup.message_source == lookup_sources.LookupSource.STATUS.value:
# Determine if tweet is still visible:
if tweet_detection_service.tweet_exists(id=previous_lookup.message_id,
username=previous_lookup.username):
can_link_tweet = True
if can_link_tweet:
reckless_driver_summary_string += L10N.PREVIOUS_LOOKUP_STATUS_STRING.format(
previous_lookup.username,
previous_lookup.username,
previous_lookup.message_id)
else:
reckless_driver_summary_string += '.'
if use_only_visible_tweets and not can_link_tweet:
# If we're only displaying tweets we can quote tweet,
# skip this one since we can'tt.
continue
reckless_driver_update_string = (
f'From {camera_streak_data_before_query.min_streak_date} to '
f'{camera_streak_data_before_query.max_streak_date}, this vehicle '
f'received {camera_streak_data_before_query.max_streak} camera '
f'violations. Over the past 12 months, this vehicle '
f'received {new_boot_eligible_violations} new camera violation'
f"{'' if new_boot_eligible_violations == 1 else 's'}: \n\n"
f'{red_light_camera_violations_string}'
f'{speed_camera_violations_string}')
messages: list[str] = [
reckless_driver_summary_string,
reckless_driver_update_string]
if not is_dry_run:
success: bool = tweeter.send_status(
message_parts=messages,
on_error_message=(
f'Error printing reckless driver update. '
f'Tagging @bdhowald.'))
if success:
LOG.debug('Reckless driver retrospective job '
'ran successfully.')
else:
print(reckless_driver_update_string)
class TestOpenDataService(unittest.TestCase):
def setUp(self):
self.open_data_service = OpenDataService()
def test_find_max_camera_streak(self):
list_of_camera_times = [
datetime(2015, 9, 18, 0, 0),
datetime(2015, 10, 16, 0, 0),
datetime(2015, 11, 2, 0, 0),
datetime(2015, 11, 5, 0, 0),
datetime(2015, 11, 12, 0, 0),
datetime(2016, 2, 2, 0, 0),
datetime(2016, 2, 25, 0, 0),
datetime(2016, 5, 31, 0, 0),
datetime(2016, 9, 8, 0, 0),
datetime(2016, 10, 17, 0, 0),
datetime(2016, 10, 24, 0, 0),
datetime(2016, 10, 26, 0, 0),
datetime(2016, 11, 21, 0, 0),
datetime(2016, 12, 18, 0, 0),
datetime(2016, 12, 22, 0, 0),
datetime(2017, 1, 5, 0, 0),
datetime(2017, 2, 13, 0, 0),
datetime(2017, 5, 10, 0, 0),
datetime(2017, 5, 24, 0, 0),
datetime(2017, 6, 27, 0, 0),
datetime(2017, 6, 27, 0, 0),
datetime(2017, 9, 14, 0, 0),
datetime(2017, 11, 6, 0, 0),
datetime(2018, 1, 28, 0, 0)
]
result = CameraStreakData(
min_streak_date='September 8, 2016',
max_streak=13,
max_streak_date='June 27, 2017')
self.assertEqual(self.open_data_service._find_max_camera_violations_streak(
list_of_camera_times), result)
@ddt.data(
{
'plate': 'ABC1234'
},
{
'medallion_query_result': [
{'dmv_license_plate_number': '8A23B'}
],
'plate': '8A23',
'state': 'NY',
},
)
@ddt.unpack
@mock.patch(
f'traffic_violations.services.apis.open_data_service.'
f'OpenDataService._perform_query')
def test_look_up_vehicle_with_violations(self,
mocked_perform_query,
plate,
state=None,
medallion_query_result=None):
random_state_index = random.randint(
0, len(regexp_constants.STATE_ABBREVIATIONS) - 1)
random_plate_type_index = random.randint(
0, len(regexp_constants.PLATE_TYPES) - 1)
final_plate = plate if medallion_query_result is None else medallion_query_result[0][
'dmv_license_plate_number']
plate_type = regexp_constants.PLATE_TYPES[random_plate_type_index]
state = state if state else regexp_constants.STATE_ABBREVIATIONS[random_state_index]
all_borough_codes = [code for borough_list in list(
BOROUGH_CODES.values()) for code in borough_list]
all_precincts = [precinct for sublist in list(
PRECINCTS_BY_BOROUGH.values()) for precinct in sublist]
summons_numbers = set()
fiscal_year_databases_violations = []
for year, _ in FISCAL_YEAR_DATABASE_ENDPOINTS.items():
fy_violations = []
for _ in range(random.randint(10, 20)):
date = datetime(
year,
random.randint(1, 12),
random.randint(1, 28))
random_borough_code_index = random.randint(
0, len(all_borough_codes) - 1)
random_precinct_index = random.randint(
0, len(all_precincts) - 1)
random_violation_string_index = random.randint(
0, len(HUMANIZED_NAMES_FOR_FISCAL_YEAR_DATABASE_VIOLATIONS.keys())-1)
summons_number = random.randint(
1000000000,
9999999999)
while summons_number in summons_numbers:
summons_number = random.randint(
1000000000,
9999999999)
summons_numbers.add(summons_number)
fy_violations.append({
'date_first_observed': date.strftime('%Y-%m-%dT%H:%M:%S.%f'),
'feet_from_curb': '0 ft',
'from_hours_in_effect': '0800',
'house_number': '123',
'intersecting_street': '',
'issue_date': date.strftime('%Y-%m-%dT%H:%M:%S.%f'),
'issuer_code': '43',
'issuer_command': 'C',
'issuer_precinct': all_precincts[random_precinct_index],
'issuing_agency': 'TRAFFIC',
'law_section': '123 ABC',
'plate_id': plate,
'plate_type': plate_type,
'registration_state': state,
'street_code1': '123',
'street_code2': '456',
'street_code3': '789',
'street_name': 'Fake Street',
'sub_division': '23',
'summons_image': '',
'summons_number': summons_number,
'to_hours_in_effect': '2200',
'vehicle_body_type': 'SUV',
'vehicle_color': 'BLACK',
'vehicle_expiration_date': date + timedelta(days=180),
'vehicle_make': 'FORD',
'vehicle_year': '2017',
'violation_code': list(HUMANIZED_NAMES_FOR_FISCAL_YEAR_DATABASE_VIOLATIONS.keys())[random_violation_string_index],
'violation_county': all_borough_codes[random_borough_code_index],
'violation_in_front_of_or_opposite': '',
'violation_legal_code': '4-08',
'violation_location': '',
'violation_post_code': '12345',
'violation_precinct': all_precincts[random_precinct_index]})
fiscal_year_databases_violations.append(fy_violations)
open_parking_and_camera_violations = []
for _ in range(random.randint(10, 20)):
fine_amount = random.randint(10, 200)
interest_amount = round(float(random.randint(
0, math.floor(fine_amount/2)) + random.random()), 2)
penalty_amount = round(float(random.randint(
0, math.floor(fine_amount/2)) + random.random()), 2)
reduction_amount = round(float(random.randint(
0, math.floor(fine_amount/2)) + random.random()), 2)
payment_amount = round(float(random.randint(0, math.floor(
fine_amount + interest_amount + penalty_amount - reduction_amount))), 2)
amount_due = fine_amount + interest_amount + \
penalty_amount - reduction_amount - payment_amount
date = datetime(
random.randint(1999, 2020),
random.randint(1, 12),
random.randint(1, 28))
random_borough_code_index = random.randint(
0, len(all_borough_codes) - 1)
random_precinct_index = random.randint(
0, len(all_precincts) - 1)
random_violation_string_index = random.randint(
0, len(HUMANIZED_NAMES_FOR_OPEN_PARKING_AND_CAMERA_VIOLATIONS.keys())-1)
summons_number = random.randint(1000000000, 9999999999)
summons_numbers.add(summons_number)
open_parking_and_camera_violations.append({
'amount_due': amount_due,
'county': all_borough_codes[random_borough_code_index],
'fine_amount': fine_amount,
'interest_amount': interest_amount,
'issue_date': date.strftime('%m/%d/%Y'),
'issuing_agency': 'TRAFFIC',
'license_type': plate_type,
'payment_amount': payment_amount,
'penalty_amount': penalty_amount,
'plate': plate,
'precinct': all_precincts[random_precinct_index],
'reduction_amount': reduction_amount,
'state': state,
'summons': (f'http://nycserv.nyc.gov/NYCServWeb/'
f'ShowImage?searchID=VDBSVmQwNVVaekZ'
f'OZWxreFQxRTlQUT09&locationName='
f'_____________________'),
'summons_image_description': 'View Summons',
'summons_number': summons_number,
'violation': list(
HUMANIZED_NAMES_FOR_OPEN_PARKING_AND_CAMERA_VIOLATIONS.keys())[
random_violation_string_index],
'violation_status': 'HEARING HELD-GUILTY',
'violation_time': f'0{random.randint(1,9)}:{random.randint(10,59)}P'
})
side_effects = []
if medallion_query_result:
side_effects.append({'data': medallion_query_result})
side_effects.append({'data': copy.deepcopy(
open_parking_and_camera_violations)})
for violations_list in fiscal_year_databases_violations:
side_effects.append({'data': copy.deepcopy(violations_list)})
mocked_perform_query.side_effect = side_effects
open_parking_and_camera_violations_dict = {}
for summons in open_parking_and_camera_violations:
summons['borough'] = [boro for boro, precincts in PRECINCTS_BY_BOROUGH.items(
) if int(summons['precinct']) in precincts][0]
summons['has_date'] = True
summons['issue_date'] = datetime.strptime(
summons['issue_date'], '%m/%d/%Y').strftime('%Y-%m-%dT%H:%M:%S.%f')
summons['violation'] = HUMANIZED_NAMES_FOR_OPEN_PARKING_AND_CAMERA_VIOLATIONS[summons['violation']]
open_parking_and_camera_violations_dict[summons['summons_number']] = summons
fiscal_year_databases_violations_dict = {}
for violation_list in fiscal_year_databases_violations:
for summons in violation_list:
summons['borough'] = [boro for boro, precincts in PRECINCTS_BY_BOROUGH.items(
) if int(summons['violation_precinct']) in precincts][0]
summons['has_date'] = True
violation_code_definition = HUMANIZED_NAMES_FOR_FISCAL_YEAR_DATABASE_VIOLATIONS.get(summons['violation_code'])
humanized_description = None
if violation_code_definition:
if isinstance(violation_code_definition, list) and summons['issue_date']:
# Some violation codes have applied to multiple violations.
for possible_description in violation_code_definition:
if (datetime.strptime(possible_description['start_date'], '%Y-%m-%d').date() <
datetime.strptime(summons['issue_date'], '%Y-%m-%dT%H:%M:%S.%f').date()):
humanized_description = possible_description['description']
else:
humanized_description = violation_code_definition
summons['violation'] = humanized_description
fiscal_year_databases_violations_dict[summons['summons_number']] = summons
merged_violations = {**open_parking_and_camera_violations_dict,
**fiscal_year_databases_violations_dict}
for key, value in merged_violations.items():
if key in open_parking_and_camera_violations_dict and key in fiscal_year_databases_violations_dict:
merged_violations[key] = {**open_parking_and_camera_violations_dict[key], **fiscal_year_databases_violations_dict[key]}
fines_dict = {}
for prefix in ['fine', 'interest', 'penalty', 'reduction', 'payment']:
fines_dict[prefix] = sum(v[f'{prefix}_amount'] for v in merged_violations.values(
) if v.get(f'{prefix}_amount'))
fined = round(sum([fines_dict.get('fine', 0), fines_dict.get(
'interest', 0), fines_dict.get('penalty', 0)]), 2)
reduced = round(fines_dict.get('reduction', 0), 2)
paid = round(fines_dict.get('payment'), 2)
amount_due = round(fined - reduced - paid, 2)
fines: FineData = FineData(
fined=fined,
reduced=reduced,
paid=paid,
outstanding=amount_due)
tickets: list[Tuple[str, int]] = Counter([v['violation'].title() for v in merged_violations.values(
) if v.get('violation')]).most_common()
years: list[Tuple[str, int]] = Counter([datetime.strptime(v['issue_date'], '%Y-%m-%dT%H:%M:%S.%f').strftime('%Y') if v.get(
'has_date') else 'No Year Available' for v in merged_violations.values()]).most_common()
boroughs = Counter([v['borough'].title()
for v in merged_violations.values()]).most_common()
camera_streak_data = {
'Failure to Stop at Red Light': None,
'Mixed': None,
'School Zone Speed Camera Violation': None,
}
for violation_type in camera_streak_data:
if violation_type == 'Failure to Stop at Red Light':
violation_times = sorted(
[datetime.strptime(v['issue_date'], '%Y-%m-%dT%H:%M:%S.%f') for v in merged_violations.values(
) if v.get('violation') == 'Failure to Stop at Red Light'])
elif violation_type == 'Mixed':
violation_times = sorted(
[datetime.strptime(v['issue_date'], '%Y-%m-%dT%H:%M:%S.%f') for v in merged_violations.values(
) if v.get('violation') and v['violation'] in ['Failure to Stop at Red Light',
'School Zone Speed Camera Violation']])
elif violation_type == 'School Zone Speed Camera Violation':
violation_times = sorted(
[datetime.strptime(v['issue_date'], '%Y-%m-%dT%H:%M:%S.%f') for v in merged_violations.values(
) if v.get('violation') == 'School Zone Speed Camera Violation'])
if violation_times:
max_streak = 0
min_streak_date = None
max_streak_date = None
for date in violation_times:
year_later = date + \
(datetime(date.year + 1, 1, 1) - datetime(date.year, 1, 1))
year_long_tickets = [
comp_date for comp_date in violation_times if date <= comp_date < year_later]
this_streak = len(year_long_tickets)
if this_streak > max_streak:
max_streak = this_streak
min_streak_date = year_long_tickets[0]
max_streak_date = year_long_tickets[-1]
camera_streak_data[violation_type] = CameraStreakData(
min_streak_date=min_streak_date.strftime('%B %-d, %Y'),
max_streak=max_streak,
max_streak_date=max_streak_date.strftime('%B %-d, %Y'))
plate_query = PlateQuery(
created_at='Tue Dec 31 19:28:12 -0500 2019',
message_id=random.randint(
1000000000000000000,
2000000000000000000),
message_source='status',
plate=plate,
plate_types=plate_type,
state=state,
username='******')
result = OpenDataServiceResponse(
data=OpenDataServicePlateLookup(
boroughs=[{'count': v, 'title': k.title()}
for k, v in boroughs],
camera_streak_data=camera_streak_data,
fines=fines,
num_violations=len(merged_violations),
plate=final_plate,
plate_types=plate_query.plate_types,
state=plate_query.state,
violations=[{'count': v, 'title': k.title()}
for k, v in tickets],
years=sorted([{'count': v, 'title': k.title()}
for k, v in years], key=lambda k: k['title'])),
success=True)
self.assertEqual(self.open_data_service.look_up_vehicle(plate_query),
result)
@mock.patch(
f'traffic_violations.services.apis.open_data_service.'
f'OpenDataService._perform_query')
def test_look_up_vehicle_with_no_violations(self,
mocked_perform_query):
plate = 'ABC1234'
plate_types = 'PAS'
state = 'NY'
mocked_perform_query.side_effects = [{'data': {}} for _ in range(0, 8)]
plate_query = PlateQuery(
created_at='Tue Dec 31 19:28:12 -0500 2019',
message_id=random.randint(
1000000000000000000,
2000000000000000000),
message_source='status',
plate=plate,
plate_types=plate_types,
state=state,
username='******')
result = OpenDataServiceResponse(
data=OpenDataServicePlateLookup(
boroughs=[],
camera_streak_data={
'Failure to Stop at Red Light': None,
'Mixed': None,
'School Zone Speed Camera Violation': None},
fines=FineData(fined=0, outstanding=0, paid=0, reduced=0),
num_violations=0,
plate=plate,
plate_types=plate_types,
state=state,
violations=[],
years=[]),
success=True)
self.assertEqual(self.open_data_service.look_up_vehicle(plate_query),
result)
def _perform_plate_lookup(
self, campaigns: list[Campaign], plate_query: PlateQuery,
unique_identifier: str) -> OpenDataServiceResponse:
LOG.debug('Performing lookup for plate.')
nyc_open_data_service: OpenDataService = OpenDataService()
open_data_response: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query)
LOG.debug(f'Violation data: {open_data_response}')
if open_data_response.success:
open_data_plate_lookup: OpenDataServicePlateLookup = open_data_response.data
bus_lane_camera_violations = 0
red_light_camera_violations = 0
speed_camera_violations = 0
for violation_type_summary in open_data_plate_lookup.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary['title'] == 'Bus Lane Violation':
bus_lane_camera_violations = violation_count
if violation_type_summary[
'title'] == 'Failure To Stop At Red Light':
red_light_camera_violations = violation_count
elif violation_type_summary[
'title'] == 'School Zone Speed Camera Violation':
speed_camera_violations = violation_count
camera_streak_data: CameraStreakData = open_data_plate_lookup.camera_streak_data
# If this came from message, add it to the plate_lookups table.
if plate_query.message_source and plate_query.message_id and plate_query.created_at:
new_lookup = PlateLookup(
boot_eligible_under_dvaa_threshold=
(camera_streak_data['Failure to Stop at Red Light'].
max_streak >= thresholds.
DANGEROUS_VEHICLE_ABATEMENT_ACT_RED_LIGHT_CAMERA_THRESHOLD
if camera_streak_data['Failure to Stop at Red Light'] else
False or
camera_streak_data['School Zone Speed Camera Violation'].
max_streak >= thresholds.
DANGEROUS_VEHICLE_ABATEMENT_ACT_SCHOOL_ZONE_SPEED_CAMERA_THRESHOLD
if
camera_streak_data['School Zone Speed Camera Violation']
else False),
boot_eligible_under_rdaa_threshold=(
camera_streak_data['Mixed'].max_streak >=
thresholds.RECKLESS_DRIVER_ACCOUNTABILITY_ACT_THRESHOLD
if camera_streak_data['Mixed'] else False),
bus_lane_camera_violations=bus_lane_camera_violations,
created_at=plate_query.created_at,
message_id=plate_query.message_id,
message_source=plate_query.message_source,
num_tickets=open_data_plate_lookup.num_violations,
plate=plate_query.plate,
plate_types=plate_query.plate_types,
red_light_camera_violations=red_light_camera_violations,
responded_to=True,
speed_camera_violations=speed_camera_violations,
state=plate_query.state,
unique_identifier=unique_identifier,
username=plate_query.username)
# Iterate through included campaigns to tie lookup to each
for campaign in campaigns:
# insert join record for campaign lookup
new_lookup.campaigns.append(campaign)
# Insert plate lookup
PlateLookup.query.session.add(new_lookup)
PlateLookup.query.session.commit()
else:
LOG.info('open data plate lookup failed')
return open_data_response
def perform(self, *args, **kwargs):
is_dry_run: bool = kwargs.get('is_dry_run') or False
days_in_period = 22.0
days_in_year = 366.0
periods_in_year = days_in_year / days_in_period
eastern = pytz.timezone('US/Eastern')
start_date = datetime(2020, 3, 10)
end_date = start_date + timedelta(days=days_in_period, seconds=-1)
tweeter = TrafficViolationsTweeter()
nyc_open_data_service: OpenDataService = OpenDataService()
covid_19_camera_offender_raw_data: List[Dict[
str,
str]] = nyc_open_data_service.lookup_covid_19_camera_violations()
for vehicle in covid_19_camera_offender_raw_data:
plate = vehicle['plate']
state = vehicle['state']
offender: Optional[
Covid19CameraOffender] = Covid19CameraOffender.get_by(
plate_id=plate, state=state)
if offender:
LOG.debug(
f'COVID-19 speeder - {L10N.VEHICLE_HASHTAG.format(state, plate)} '
f"with {vehicle['count']} camera violations has been seen before."
)
continue
LOG.debug(
f'COVID-19 speeder - {L10N.VEHICLE_HASHTAG.format(state, plate)} '
f"with {vehicle['count']} camera violations has not been seen before."
)
plate_query: PlateQuery = PlateQuery(
created_at=datetime.now(),
message_source=LookupSource.API,
plate=plate,
plate_types=None,
state=state)
response: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query, since=start_date, until=end_date)
plate_lookup: OpenDataServicePlateLookup = response.data
red_light_camera_violations = 0
speed_camera_violations = 0
for violation_type_summary in plate_lookup.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary[
'title'] == self.RED_LIGHT_CAMERA_VIOLATION_DESCRIPTION:
red_light_camera_violations = violation_count
if violation_type_summary[
'title'] == self.SPEED_CAMERA_VIOLATION_DESCRIPTION:
speed_camera_violations = violation_count
total_camera_violations = (red_light_camera_violations +
speed_camera_violations)
vehicle_hashtag = L10N.VEHICLE_HASHTAG.format(state, plate)
red_light_camera_violations_string = (
f'{red_light_camera_violations} | Red Light Camera Violations\n'
if red_light_camera_violations > 0 else '')
speed_camera_violations_string = (
f'{speed_camera_violations} | Speed Safety Camera Violations\n'
if speed_camera_violations > 0 else '')
covid_19_reckless_driver_string = (
f"From {start_date.strftime('%B %-d, %Y')} to "
f"{end_date.strftime('%B %-d, %Y')}, {vehicle_hashtag} "
f'received {total_camera_violations} camera '
f'violations:\n\n'
f'{red_light_camera_violations_string}'
f'{speed_camera_violations_string}')
dval_string = (
'At this rate, this vehicle will receive '
f'{round(periods_in_year * total_camera_violations)} '
'speed safety camera violations over '
'a year, qualifying it for towing or booting under '
'@bradlander\'s Dangerous Vehicle Abatement Law and '
'requiring its driver to take a course on the consequences '
'of reckless driving.')
speeding_string = (
'With such little traffic, many drivers are speeding '
'regularly, putting New Yorkers at increased risk of '
'ending up in a hospital at a time our hospitals are '
'stretched to their limits. It\'s also hard to practice '
'social distancing when walking on our narrow sidewalks.')
open_streets_string = (
'Other cities are eating our lunch, @NYCMayor:\n\n'
f'{random.choice(self.COVID_19_OPEN_STREETS_TWEETS)}')
messages: List[str] = [
covid_19_reckless_driver_string, dval_string,
[speeding_string, open_streets_string]
]
if not is_dry_run:
success: bool = tweeter.send_status(
message_parts=messages,
on_error_message=(
f'Error printing COVID-19 reckless driver update. '
f'Tagging @bdhowald.'))
if success:
offender = Covid19CameraOffender(
plate_id=plate,
state=state,
red_light_camera_violations=red_light_camera_violations,
speed_camera_violations=speed_camera_violations)
Covid19CameraOffender.query.session.add(offender)
try:
Covid19CameraOffender.query.session.commit()
LOG.debug('COVID-19 Reckless driver retrospective job '
'ran successfully.')
except:
tweeter.send_status(message_parts=[(
f'Error printing COVID-19 reckless driver update. '
f'Tagging @bdhowald.')])
# Only do one at a time.
break
else:
print(covid_19_reckless_driver_string)
print(dval_string)
print(speeding_string)
print(open_streets_string)
break
def update_lookups(self, lookups: list[PlateLookup]):
for previous_lookup in lookups:
plate_query: PlateQuery = PlateQuery(
created_at=previous_lookup.created_at,
message_source=previous_lookup.message_source,
plate=previous_lookup.plate,
plate_types=previous_lookup.plate_types,
state=previous_lookup.state)
nyc_open_data_service: OpenDataService = OpenDataService()
open_data_response: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query, until=previous_lookup.created_at)
open_data_plate_lookup: OpenDataServicePlateLookup = open_data_response.data
for violation_type_summary in open_data_plate_lookup.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary['title'] == 'Bus Lane Violation':
previous_lookup.bus_lane_camera_violations = violation_count
if violation_type_summary[
'title'] == 'Failure To Stop At Red Light':
previous_lookup.red_light_camera_violations = violation_count
elif violation_type_summary[
'title'] == 'School Zone Speed Camera Violation':
previous_lookup.speed_camera_violations = violation_count
if not previous_lookup.bus_lane_camera_violations:
previous_lookup.bus_lane_camera_violations = 0
if not previous_lookup.red_light_camera_violations:
previous_lookup.red_light_camera_violations = 0
if not previous_lookup.speed_camera_violations:
previous_lookup.speed_camera_violations = 0
if not previous_lookup.boot_eligible_under_dvaa_threshold or True:
camera_streak_data = open_data_plate_lookup.camera_streak_data
red_light_camera_streak_data = camera_streak_data.get(
'Failure to Stop at Red Light')
speed_camera_streak_data = camera_streak_data.get(
'School Zone Speed Camera Violation')
eligible_to_be_booted_for_red_light_violations_under_dvaa = (
red_light_camera_streak_data is not None
and red_light_camera_streak_data.max_streak >= thresholds.
DANGEROUS_VEHICLE_ABATEMENT_ACT_RED_LIGHT_CAMERA_THRESHOLD)
eligible_to_be_booted_for_speed_camera_violations_under_dvaa = (
speed_camera_streak_data is not None
and speed_camera_streak_data.max_streak >= thresholds.
DANGEROUS_VEHICLE_ABATEMENT_ACT_SCHOOL_ZONE_SPEED_CAMERA_THRESHOLD
)
previous_lookup.boot_eligible_under_dvaa_threshold = (
eligible_to_be_booted_for_red_light_violations_under_dvaa
or
eligible_to_be_booted_for_speed_camera_violations_under_dvaa
)
LOG.debug(f'updating lookup {previous_lookup.id}')
print(f'updating lookup {previous_lookup.id}')
def perform(self, *args, **kwargs):
is_dry_run: bool = kwargs.get('is_dry_run') or False
start_date = datetime.datetime(2020, 3, 10, 0, 0)
end_date = datetime.datetime(2021, 11, 26, 23, 59, 59)
days_in_period = (end_date - start_date).days
num_years = days_in_period / 365.0
tweeter = TrafficViolationsTweeter()
nyc_open_data_service: OpenDataService = OpenDataService()
covid_19_camera_offender_raw_data: list[dict[str, str]] = nyc_open_data_service.lookup_covid_19_camera_violations()
for vehicle in covid_19_camera_offender_raw_data:
plate = vehicle['plate']
state = vehicle['state']
offender: Optional[Covid19CameraOffender] = Covid19CameraOffender.get_by(
plate_id=plate,
state=state,
count_as_queried=True)
if offender:
LOG.debug(f'COVID-19 speeder - {L10N.VEHICLE_HASHTAG.format(state, plate)} '
f"with {vehicle['total_camera_violations']} camera violations has been seen before.")
continue
LOG.debug(f'COVID-19 speeder - {L10N.VEHICLE_HASHTAG.format(state, plate)} '
f"with {vehicle['total_camera_violations']} camera violations has not been seen before.")
plate_query: PlateQuery = PlateQuery(created_at=datetime.datetime.now(),
message_source=LookupSource.API,
plate=plate,
plate_types=None,
state=state)
response: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query,
since=start_date,
until=end_date)
plate_lookup: OpenDataServicePlateLookup = response.data
red_light_camera_violations = 0
speed_camera_violations = 0
for violation_type_summary in plate_lookup.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary['title'] == self.RED_LIGHT_CAMERA_VIOLATION_DESCRIPTION:
red_light_camera_violations = violation_count
if violation_type_summary['title'] == self.SPEED_CAMERA_VIOLATION_DESCRIPTION:
speed_camera_violations = violation_count
speed_camera_violations_per_year = speed_camera_violations / num_years
# If this driver doesn't meet the threshold, continue
if speed_camera_violations_per_year < self.DVAL_SPEED_CAMERA_THRESHOLD:
continue
total_camera_violations = speed_camera_violations + red_light_camera_violations
vehicle_hashtag = L10N.VEHICLE_HASHTAG.format(
state, plate)
red_light_camera_violations_string = (
f'{red_light_camera_violations} | Red Light Camera Violations\n'
if red_light_camera_violations > 0 else '')
speed_camera_violations_string = (
f'{speed_camera_violations} | Speed Safety Camera Violations\n'
if speed_camera_violations > 0 else '')
covid_19_reckless_driver_string = (
f"From {start_date.strftime('%B %-d, %Y')} to "
f"{end_date.strftime('%B %-d, %Y')}, {vehicle_hashtag} "
f'received {total_camera_violations} camera '
f'violations:\n\n'
f'{red_light_camera_violations_string}'
f'{speed_camera_violations_string}')
dval_string = (
'This vehicle has received an average of '
f'{round(speed_camera_violations / num_years, 1)} '
'speed safety camera violations per year, '
'qualifying it for towing or booting under '
'@bradlander\'s Dangerous Vehicle Abatement Law and '
'requiring its driver to take a course on the consequences '
'of reckless driving.')
messages: list[str] = [covid_19_reckless_driver_string, dval_string]
if not is_dry_run:
success: bool = tweeter.send_status(
message_parts=messages,
on_error_message=(
f'Error printing COVID-19 reckless driver update. '
f'Tagging @bdhowald.'))
if success:
offender = Covid19CameraOffender(plate_id=plate,
state=state,
red_light_camera_violations=red_light_camera_violations,
speed_camera_violations=speed_camera_violations)
Covid19CameraOffender.query.session.add(offender)
try:
Covid19CameraOffender.query.session.commit()
LOG.debug('COVID-19 Reckless driver retrospective job '
'ran successfully.')
except:
tweeter.send_status(message_parts=[(
f'Error printing COVID-19 reckless driver update. '
f'Tagging @bdhowald.')])
# Only do one at a time.
break
else:
print(covid_19_reckless_driver_string)
print(dval_string)
break