def __init__(self):
self._app_api = None
self._client_api = None
# Create reply argument_builder
self.reply_argument_builder = ReplyArgumentBuilder(self._get_twitter_application_api())
# Create new aggregator
self.aggregator = TrafficViolationsAggregator()
# Need to find if unanswered statuses have been deleted
self.tweet_detection_service = TweetDetectionService()
# Log how many times we've called the apis
self._direct_messages_iteration = 0
self._events_iteration = 0
self._statuses_iteration = 0
self._lookup_threads = []
# Initialize cached values to None
self._follower_ids: Optional[list[int]] = None
self._follower_ids_last_fetched: Optional[datetime] = None
def __init__(self):
self.tweet_detection_service = TweetDetectionService()
self.eastern = pytz.timezone('US/Eastern')
self.utc = pytz.timezone('UTC')
class TrafficViolationsAggregator:
CAMERA_VIOLATIONS = [
'Bus Lane Violation', 'Failure To Stop At Red Light',
'School Zone Speed Camera Violation'
]
MYSQL_TIME_FORMAT: str = '%Y-%m-%d %H:%M:%S'
UNIQUE_IDENTIFIER_STRING_LENGTH = 8
def __init__(self):
self.tweet_detection_service = TweetDetectionService()
self.eastern = pytz.timezone('US/Eastern')
self.utc = pytz.timezone('UTC')
def initiate_reply(self, lookup_request: Type[BaseLookupRequest]):
LOG.info('Calling initiate_reply')
if lookup_request.requires_response():
return self._create_response(lookup_request)
def _create_repeat_lookup_string(
self,
new_violations: int,
plate: str,
state: str,
username: str,
previous_lookup: Optional[Dict[str, Any]] = None):
violations_string = ''
if new_violations > 0:
# assume we can't link
can_link_tweet = False
# Where did this come from?
if previous_lookup.message_source == lookup_sources.LookupSource.STATUS.value:
# Determine if tweet is still visible:
if self.tweet_detection_service.tweet_exists(
id=previous_lookup.message_id, username=username):
can_link_tweet = True
# Determine when the last lookup was...
previous_time = previous_lookup.created_at
now = datetime.now()
adjusted_time = self.utc.localize(previous_time)
adjusted_now = self.utc.localize(now)
# If at least five minutes have passed...
if adjusted_now - timedelta(minutes=5) > adjusted_time:
# Add the new ticket info and previous lookup time to the string.
violations_string += L10N.LAST_QUERIED_STRING.format(
adjusted_time.astimezone(self.eastern).strftime(
L10N.REPEAT_LOOKUP_DATE_FORMAT),
adjusted_time.astimezone(self.eastern).strftime(
L10N.REPEAT_LOOKUP_TIME_FORMAT))
if can_link_tweet:
violations_string += L10N.PREVIOUS_LOOKUP_STATUS_STRING.format(
previous_lookup.username, previous_lookup.username,
previous_lookup.message_id)
else:
violations_string += '.'
violations_string += L10N.REPEAT_LOOKUP_STRING.format(
L10N.VEHICLE_HASHTAG.format(state, plate), new_violations,
L10N.pluralize(new_violations))
return violations_string
def _create_response(self, request_object: Type[BaseLookupRequest]):
LOG.info('\n')
LOG.info('Calling create_response')
# Grab tweet details for reply.
LOG.debug(f'request_object: {request_object}')
# Collect response parts here.
response_parts = []
successful_lookup = False
error_on_lookup = False
# Wrap in try/catch block
try:
# Find potential plates
potential_vehicles: List[Vehicle] = self._find_potential_vehicles(
request_object.string_tokens())
LOG.debug(f'potential_vehicles: {potential_vehicles}')
potential_vehicles += self._find_potential_vehicles_using_legacy_logic(
request_object.legacy_string_tokens())
LOG.debug(f'potential_vehicles: {potential_vehicles}')
potential_vehicles = self._ensure_unique_plates(
vehicles=potential_vehicles)
# Find included campaign hashtags
included_campaigns: List[Campaign] = self._detect_campaigns(
request_object.string_tokens())
LOG.debug(f'included_campaigns: {included_campaigns}')
# for each vehicle, we need to determine if the supplied information amounts to a valid plate
# then we need to look up each valid plate
# then we need to respond in a single thread in order with the responses
summary: TrafficViolationsAggregatorResponse = TrafficViolationsAggregatorResponse(
)
for potential_vehicle in potential_vehicles:
if potential_vehicle.valid_plate:
plate_query: PlateQuery = self._get_plate_query(
vehicle=potential_vehicle,
request_object=request_object)
# do we have a previous lookup
previous_lookup: Optional[
PlateLookup] = self._query_for_previous_lookup(
plate_query=plate_query)
LOG.debug(
f'Previous lookup for this vehicle: {previous_lookup}')
# Obtain a unique identifier for the lookup
unique_identifier: str = self._get_unique_identifier()
# Do the real work!
open_data_response: OpenDataServiceResponse = self._perform_plate_lookup(
campaigns=included_campaigns,
plate_query=plate_query,
unique_identifier=unique_identifier)
if open_data_response.success:
# Record successful lookup.
successful_lookup = True
plate_lookup: OpenDataServicePlateLookup = open_data_response.data
# how many times have we searched for this plate from a tweet
current_frequency: int = self._query_for_lookup_frequency(
plate_query)
# self._proceess_lookup_results()
# Add lookup to summary
summary.plate_lookups.append(plate_lookup)
if plate_lookup.violations:
plate_lookup_response_parts: List[
Any] = self._form_plate_lookup_response_parts(
borough_data=plate_lookup.boroughs,
camera_streak_data=plate_lookup.
camera_streak_data,
fine_data=plate_lookup.fines,
frequency=current_frequency,
plate=plate_lookup.plate,
plate_types=plate_lookup.plate_types,
previous_lookup=previous_lookup,
state=plate_lookup.state,
username=request_object.username(),
unique_identifier=unique_identifier,
violations=plate_lookup.violations,
year_data=plate_lookup.years)
response_parts.append(plate_lookup_response_parts)
# [[campaign_stuff], tickets_0, tickets_1, etc.]
else:
# Let user know we didn't find anything.
plate_types_string = (
f' (types: {plate_query.plate_types})'
) if plate_lookup.plate_types else ''
response_parts.append(
L10N.NO_TICKETS_FOUND_STRING.format(
plate_query.state, plate_lookup.plate,
plate_types_string))
else:
# Record lookup error.
error_on_lookup = True
response_parts.append([
f"Sorry, I received an error when looking up "
f"{plate_query.state}:{plate_query.plate}"
f"{(' (types: ' + plate_query.plate_types + ')') if plate_query.plate_types else ''}. "
f"Please try again."
])
else:
# Record the failed lookup.
new_failed_lookup = FailedPlateLookup(
message_id=request_object.external_id(),
username=request_object.username())
# Insert plate lookup
FailedPlateLookup.query.session.add(new_failed_lookup)
FailedPlateLookup.query.session.commit()
# Legacy data where state is not a valid abbreviation.
if potential_vehicle.state:
LOG.debug("We have a state, but it's invalid.")
response_parts.append([
f"The state should be two characters, but you supplied '{potential_vehicle.state}'. "
f"Please try again."
])
# '<state>:<plate>' format, but no valid state could be detected.
elif potential_vehicle.original_string:
LOG.debug(
"We don't have a state, but we have an attempted lookup with the new format."
)
response_parts.append([
f"Sorry, a plate and state could not be inferred from "
f"{potential_vehicle.original_string}."
])
# If we have a plate, but no state.
elif potential_vehicle.plate:
LOG.debug("We have a plate, but no state")
response_parts.append(
["Sorry, the state appears to be blank."])
# If we have multiple vehicles, prepend a summary.
if len(summary.plate_lookups) > 1:
summary_string: Optional[str] = self._form_summary_string(
summary)
if summary_string:
response_parts.insert(0, summary_string)
# Look up campaign hashtags after doing the plate lookups and then
# prepend to response.
if included_campaigns:
campaign_lookups: List[Tuple[
str, int,
int]] = self._perform_campaign_lookup(included_campaigns)
response_parts.insert(
0,
self._form_campaign_lookup_response_parts(
campaign_lookups, request_object.username()))
successful_lookup = True
# If we don't look up a single plate successfully,
# figure out how we can help the user.
if not successful_lookup and not error_on_lookup:
# Record the failed lookup.
new_failed_lookup = FailedPlateLookup(
message_id=request_object.external_id(),
username=request_object.username())
# Insert plate lookup
FailedPlateLookup.query.session.add(new_failed_lookup)
FailedPlateLookup.query.session.commit()
LOG.debug('The data seems to be in the wrong format.')
state_matches = [
regexp_constants.STATE_ABBREVIATIONS_PATTERN.search(
s.upper()) != None
for s in request_object.string_tokens()
]
number_matches = [
regexp_constants.NUMBER_PATTERN.search(s.upper()) != None
for s in list(
filter(
lambda part: re.sub(r'\.|@', '', part.lower())
not in set(request_object.mentioned_users),
request_object.string_tokens()))
]
# We have what appears to be a plate and a state abbreviation.
if all([any(state_matches), any(number_matches)]):
LOG.debug(
'There is both plate and state information in this message.'
)
# Let user know plate format
response_parts.append([
"I’d be happy to look that up for you!\n\nJust a reminder, the format is <state|province|territory>:<plate>, e.g. NY:abc1234"
])
# Maybe we have plate or state. Let's find out.
else:
LOG.debug(
'The tweet is missing either state or plate or both.')
state_minus_words_matches = [
regexp_constants.STATE_MINUS_WORDS_PATTERN.search(
s.upper()) != None
for s in request_object.string_tokens()
]
number_matches = [
regexp_constants.NUMBER_PATTERN.search(s.upper()) !=
None for s in list(
filter(
lambda part: re.sub(r'\.|@', '', part.lower())
not in set(request_object.mentioned_users),
request_object.string_tokens()))
]
# We have either plate or state.
if any(state_minus_words_matches) or any(number_matches):
# Let user know plate format
response_parts.append([
"I think you're trying to look up a plate, but can't be sure.\n\nJust a reminder, the format is <state|province|territory>:<plate>, e.g. NY:abc1234"
])
# We have neither plate nor state. Do nothing.
else:
LOG.debug(
'ignoring message since no plate or state information to respond to.'
)
except Exception as e:
# Set response data
error_on_lookup = True
response_parts.append(
["Sorry, I encountered an error. Tagging @bdhowald."])
# Log error
LOG.error('Missing necessary information to continue')
LOG.error(e)
LOG.error(str(e))
LOG.error(e.args)
logging.exception("stack trace")
# Indicate successful response processing.
return {
'error_on_lookup': error_on_lookup,
'request_object': request_object,
'response_parts': response_parts,
'success': True,
'successful_lookup': successful_lookup,
'username': request_object.username()
}
def _detect_campaigns(self, string_tokens) -> List[Campaign]:
""" Look for campaign hashtags in the message's text
and return matching campaigns.
"""
return Campaign.get_all_in(hashtag=tuple([
regexp_constants.HASHTAG_PATTERN.sub('', string)
for string in string_tokens
]))
def _detect_plate_types(self, plate_types_input) -> bool:
if ',' in plate_types_input:
parts = plate_types_input.upper().split(',')
return any([
regexp_constants.PLATE_TYPES_PATTERN.search(part) != None
for part in parts
])
else:
return regexp_constants.PLATE_TYPES_PATTERN.search(
plate_types_input.upper()) != None
def _detect_state(self, state_input) -> bool:
# or state_full_pattern.search(state_input.upper()) != None
""" Does this input constitute a valid state abbreviation """
if state_input is not None:
return regexp_constants.STATE_ABBREVIATIONS_PATTERN.search(
state_input.upper()) != None
return False
def _ensure_unique_plates(self, vehicles: List[Vehicle]) -> List[Vehicle]:
vehicle_dict: Dict[str, Vehicle] = {}
unique_vehicles: List[Vehicle] = []
for vehicle in vehicles:
lookup_string = (
f'{vehicle.state}:'
f'{vehicle.plate}'
f"{(':' + vehicle.plate_types) if vehicle.plate_types else ''}"
)
if vehicle_dict.get(lookup_string) is None:
vehicle_dict[lookup_string] = vehicle
unique_vehicles.append(vehicle)
return unique_vehicles
def _find_potential_vehicles(self,
list_of_strings: List[str]) -> List[Vehicle]:
# Use new logic of '<state>:<plate>'
plate_tuples: Union[Tuple[str, str, str], Tuple[str, str]] = [
[part.strip() for part in match.split(':')]
for match in re.findall(regexp_constants.PLATE_FORMAT_REGEX,
' '.join(list_of_strings))
if all(substr not in match.lower()
for substr in ['://', 'state:', 'plate:'])
]
return self._infer_plate_and_state_data(plate_tuples)
def _find_potential_vehicles_using_legacy_logic(
self, list_of_strings: List[str]) -> List[Vehicle]:
# Find potential plates
# Use old logic of 'state:<state> plate:<plate>'
potential_vehicles: List[Vehicle] = []
legacy_plate_data: Tuple = dict([[
piece.strip() for piece in match.split(':')
] for match in [
part.lower() for part in list_of_strings
if (('state:' in part.lower() or 'plate:' in part.lower()
or 'types:' in part.lower()) and '://' not in part.lower())
]])
if legacy_plate_data:
if self._detect_state(legacy_plate_data.get(
'state')) and legacy_plate_data.get('plate'):
legacy_plate_data['valid_plate'] = True
else:
legacy_plate_data['valid_plate'] = False
vehicle: Vehicle = Vehicle(
plate=legacy_plate_data.get('plate'),
plate_types=legacy_plate_data.get('types'),
state=legacy_plate_data.get('state'),
valid_plate=legacy_plate_data['valid_plate'])
potential_vehicles.append(vehicle)
return potential_vehicles
def _form_campaign_lookup_response_parts(
self, campaign_summaries: List[Tuple[str, int,
int]], username: str):
campaign_chunks: List[str] = []
campaign_string = ""
for campaign in campaign_summaries:
campaign_name = campaign[0]
campaign_vehicles = campaign[1]
campaign_tickets = campaign[2]
next_string_part = (
f"{'{:,}'.format(campaign_vehicles)} {'vehicle with' if campaign_vehicles == 1 else 'vehicles with a total of'} "
f"{'{:,}'.format(campaign_tickets)} ticket{L10N.pluralize(campaign_tickets)} {'has' if campaign_vehicles == 1 else 'have'} "
f"been tagged with {campaign_name}.\n\n")
# how long would it be
potential_response_length = len(username + ' ' + campaign_string +
next_string_part)
if (potential_response_length <=
twitter_constants.MAX_TWITTER_STATUS_LENGTH):
campaign_string += next_string_part
else:
campaign_chunks.append(campaign_string)
campaign_string = next_string_part
# Get any part of string left over
campaign_chunks.append(campaign_string)
return campaign_chunks
def _form_plate_lookup_response_parts(
self,
borough_data: List[Tuple[str, int]],
frequency: int,
fine_data: FineData,
plate: str,
plate_types: List[str],
state: str,
unique_identifier: str,
username: str,
violations: List[Tuple[str, int]],
year_data: List[Tuple[str, int]],
camera_streak_data: Optional[CameraStreakData] = None,
previous_lookup: Optional[PlateLookup] = None):
# response_chunks holds tweet-length-sized parts of the response
# to be tweeted out or appended into a single direct message.
response_chunks: List[str] = []
violations_string: str = ""
# Get total violations
total_violations: int = sum([s['count'] for s in violations])
LOG.debug(f'total_violations: {total_violations}')
# Append to initially blank string to build tweet.
violations_string += L10N.LOOKUP_SUMMARY_STRING.format(
L10N.VEHICLE_HASHTAG.format(state, plate),
L10N.get_plate_types_string(plate_types), frequency,
L10N.pluralize(int(frequency)))
# If this vehicle has been queried before...
if previous_lookup:
previous_num_violations: int = previous_lookup.num_tickets
violations_string += self._create_repeat_lookup_string(
new_violations=(total_violations - previous_num_violations),
plate=plate,
previous_lookup=previous_lookup,
state=state,
username=username)
response_chunks += self._handle_response_part_formation(
collection=violations,
continued_format_string=L10N.LOOKUP_TICKETS_STRING_CONTD.format(
L10N.VEHICLE_HASHTAG.format(state, plate)),
count='count',
cur_string=violations_string,
description='title',
default_description='No Year Available',
prefix_format_string=L10N.LOOKUP_TICKETS_STRING.format(
total_violations),
result_format_string=L10N.LOOKUP_RESULTS_DETAIL_STRING,
username=username)
if year_data:
response_chunks += self._handle_response_part_formation(
collection=year_data,
continued_format_string=L10N.LOOKUP_YEAR_STRING_CONTD.format(
L10N.VEHICLE_HASHTAG.format(state, plate)),
count='count',
description='title',
default_description='No Year Available',
prefix_format_string=L10N.LOOKUP_YEAR_STRING.format(
L10N.VEHICLE_HASHTAG.format(state, plate)),
result_format_string=L10N.LOOKUP_RESULTS_DETAIL_STRING,
username=username)
if borough_data:
response_chunks += self._handle_response_part_formation(
collection=borough_data,
continued_format_string=L10N.LOOKUP_BOROUGH_STRING_CONTD.
format(L10N.VEHICLE_HASHTAG.format(state, plate)),
count='count',
description='title',
default_description='No Borough Available',
prefix_format_string=L10N.LOOKUP_BOROUGH_STRING.format(
L10N.VEHICLE_HASHTAG.format(state, plate)),
result_format_string=L10N.LOOKUP_RESULTS_DETAIL_STRING,
username=username)
if fine_data and fine_data.fines_assessed():
cur_string = f"Known fines for {L10N.VEHICLE_HASHTAG.format(state, plate)}:\n\n"
max_count_length = len('${:,.2f}'.format(fine_data.max_amount()))
spaces_needed = (max_count_length * 2) + 1
for fine_type, amount in fine_data:
currency_string = '${:,.2f}'.format(amount)
count_length = len(str(currency_string))
# e.g., if spaces_needed is 5, and count_length is 2, we need
# to pad to 3.
left_justify_amount = spaces_needed - count_length
# formulate next string part
next_part = (f"{currency_string.ljust(left_justify_amount)}| "
f"{fine_type.replace('_', ' ').title()}\n")
# determine current string length if necessary
potential_response_length = len(username + ' ' + cur_string +
next_part)
# If username, space, violation string so far and new part are less or
# equal than 280 characters, append to existing tweet string.
if (potential_response_length <=
twitter_constants.MAX_TWITTER_STATUS_LENGTH):
cur_string += next_part
else:
response_chunks.append(cur_string)
cur_string = "Known fines for #{}_{}, cont'd:\n\n"
cur_string += next_part
# add to container
response_chunks.append(cur_string)
if camera_streak_data:
if camera_streak_data.max_streak and camera_streak_data.max_streak >= 5:
# formulate streak string
streak_string = (
f"Under @bradlander's proposed legislation, "
f"this vehicle could have been booted or impounded "
f"due to its {camera_streak_data.max_streak} camera violations "
f"(>= 5/year) from {camera_streak_data.min_streak_date}"
f" to {camera_streak_data.max_streak_date}.\n")
# add to container
response_chunks.append(streak_string)
unique_link: str = self._get_website_plate_lookup_link(
unique_identifier)
website_link_string = f'View more details at {unique_link}.'
response_chunks.append(website_link_string)
# Send it back!
return response_chunks
def _form_summary_string(
self,
summary: TrafficViolationsAggregatorResponse) -> Optional[str]:
num_vehicles = len(summary.plate_lookups)
vehicle_tickets = [
len(lookup.violations) for lookup in summary.plate_lookups
]
total_tickets = sum(vehicle_tickets)
fines_by_vehicle: List[FineData] = [
lookup.fines for lookup in summary.plate_lookups
]
vehicles_with_fines: int = len([
fine_data for fine_data in fines_by_vehicle if fine_data.fined > 0
])
aggregate_fines: FineData = FineData(
**{
field: sum(
getattr(lookup.fines, field)
for lookup in summary.plate_lookups)
for field in FineData.FINE_FIELDS
})
if aggregate_fines.fined > 0:
return [
f"You queried {num_vehicles} vehicles, of which "
f"{vehicles_with_fines} vehicle{L10N.pluralize(vehicles_with_fines)} "
f"{'has' if vehicles_with_fines == 1 else 'have collectively'} received {total_tickets} ticket{L10N.pluralize(total_tickets)} "
f"with at least {'${:,.2f}'.format(aggregate_fines.fined - aggregate_fines.reduced)} "
f"in fines, of which {'${:,.2f}'.format(aggregate_fines.paid)} has been paid.\n\n"
]
def _generate_unique_identifier(self):
return ''.join(random.SystemRandom().choice(string.ascii_lowercase +
string.digits)
for _ in range(self.UNIQUE_IDENTIFIER_STRING_LENGTH))
def _get_unique_identifier(self):
unique_identifier = self._generate_unique_identifier()
while PlateLookup.query.filter(
PlateLookup.unique_identifier == unique_identifier).all():
unique_identifier = self._generate_unique_identifier()
return unique_identifier
def _get_plate_query(self, request_object: Type[BaseLookupRequest],
vehicle: Vehicle) -> PlateQuery:
"""Transform a request object into plate query"""
created_at: str = datetime.strptime(
request_object.created_at,
twitter_constants.TWITTER_TIME_FORMAT).strftime(
self.MYSQL_TIME_FORMAT)
message_id: Optional[str] = request_object.external_id()
message_source: str = request_object.message_source
plate: str = regexp_constants.PLATE_PATTERN.sub(
'',
vehicle.plate.strip().upper())
plate_types: Optional[str] = None
if vehicle.plate_types is not None:
plate_types = ','.join(
sorted([
type.strip()
for type in vehicle.plate_types.upper().split(',')
]))
state: str = vehicle.state.upper()
username: Optional[str] = request_object.username()
plate_query: PlateQuery = PlateQuery(created_at=created_at,
message_id=message_id,
message_source=message_source,
plate=plate,
plate_types=plate_types,
state=state,
username=username)
LOG.debug(f'plate_query: {plate_query}')
return plate_query
def _get_website_plate_lookup_link(self, unique_identifier: str) -> str:
return f'{endpoints.HOWS_MY_DRIVING_NY_WEBSITE}/{unique_identifier}'
def _handle_response_part_formation(self,
count: str,
collection: Dict[str, Any],
continued_format_string: str,
description: str,
default_description: str,
prefix_format_string: str,
result_format_string: str,
username: str,
cur_string: str = None):
# collect the responses
response_container = []
cur_string = cur_string if cur_string else ''
if prefix_format_string:
cur_string += prefix_format_string
max_count_length = len(str(max(item[count] for item in collection)))
spaces_needed = (max_count_length * 2) + 1
# Grab item
for item in collection:
# Titleize for readability.
violation_description = item[description].title()
# Use a default description if need be
if len(violation_description) == 0:
violation_description = default_description
violation_count = item[count]
count_length = len(str(violation_count))
# e.g., if spaces_needed is 5, and count_length is 2, we need to
# pad to 3.
left_justify_amount = spaces_needed - count_length
# formulate next string part
next_part = result_format_string.format(
str(violation_count).ljust(left_justify_amount),
violation_description)
# determine current string length
potential_response_length = len(
f'{username} {cur_string}{next_part}')
# If username, space, violation string so far and new part are less or
# equal than 280 characters, append to existing tweet string.
if (potential_response_length <=
twitter_constants.MAX_TWITTER_STATUS_LENGTH):
cur_string += next_part
else:
response_container.append(cur_string)
if continued_format_string:
cur_string = continued_format_string
else:
cur_string = ''
cur_string += next_part
# If we finish the list with a non-empty string,
# append that string to response parts
if len(cur_string) != 0:
# Append ready string into parts for response.
response_container.append(cur_string)
# Return parts
return response_container
def _infer_plate_and_state_data(
self, list_of_vehicle_tuples: Union[Tuple[str, str, str], Tuple[str,
str]]
) -> List[Vehicle]:
potential_vehicles: List[Vehicle] = []
for vehicle_tuple in list_of_vehicle_tuples:
original_string = ':'.join(vehicle_tuple)
plate = None
plate_types = None
state = None
valid_plate = False
if len(vehicle_tuple) in range(2, 4):
state_bools: List[bool] = [
self._detect_state(part) for part in vehicle_tuple
]
try:
state_index = state_bools.index(True)
except ValueError:
state_index = None
plate_types_bools: List[bool] = [
self._detect_plate_types(part) for part in vehicle_tuple
]
try:
plate_types_index = plate_types_bools.index(True)
except ValueError:
plate_types_index = None
have_valid_plate: bool = (
len(vehicle_tuple) == 2 and state_index is not None) or (
len(vehicle_tuple) == 3
and None not in [plate_types_index, state_index])
if have_valid_plate:
non_state_plate_types_parts = [
x for x in list(range(0, len(vehicle_tuple)))
if x not in [plate_types_index, state_index]
]
plate_index = None
# We have a tuple with state and plate, and possibly plate
# types
if non_state_plate_types_parts:
plate_index = non_state_plate_types_parts[0]
# We don't seem to have a plate, which means the plate
# types might be the plate
elif plate_types_index is not None:
alphanumeric_only = re.match(
'^[\w-]+$',
vehicle_tuple[plate_types_index]) is not None
if alphanumeric_only:
plate_index = plate_types_index
plate_types_index = None
# Put plate data together
if plate_index is not None and vehicle_tuple[
plate_index] != '':
plate = vehicle_tuple[plate_index]
state = vehicle_tuple[state_index]
if plate_types_index is not None:
plate_types = vehicle_tuple[plate_types_index]
valid_plate = True
vehicle: Vehicle = Vehicle(original_string=original_string,
plate=plate,
plate_types=plate_types,
state=state,
valid_plate=valid_plate)
potential_vehicles.append(vehicle)
return potential_vehicles
def _perform_campaign_lookup(
self,
included_campaigns: List[Campaign]) -> List[Tuple[str, int, int]]:
LOG.debug('Performing lookup for campaigns.')
result: List[Tuple[str, int, int]] = []
for campaign in included_campaigns:
subquery = campaign.plate_lookups.session.query(
PlateLookup.plate,
PlateLookup.state,
func.max(PlateLookup.created_at).label(
'most_recent_campaign_lookup'),
).group_by(PlateLookup.plate, PlateLookup.state).filter(
and_(PlateLookup.campaigns.any(Campaign.id.in_([campaign.id])),
PlateLookup.count_towards_frequency == True)).subquery(
'subquery')
full_query = PlateLookup.query.join(
subquery, (PlateLookup.plate == subquery.c.plate) &
(PlateLookup.state == subquery.c.state) &
(PlateLookup.created_at
== subquery.c.most_recent_campaign_lookup)).order_by(
subquery.c.most_recent_campaign_lookup.desc(),
PlateLookup.created_at.desc())
campaign_lookups = full_query.all()
campaign_vehicles: int = len(campaign_lookups)
campaign_tickets: int = sum(
[lookup.num_tickets for lookup in campaign_lookups])
result.append(
(campaign.hashtag, campaign_vehicles, campaign_tickets))
return result
def _perform_plate_lookup(
self, campaigns: List[Campaign], plate_query: PlateQuery,
unique_identifier: str) -> OpenDataServiceResponse:
LOG.debug('Performing lookup for plate.')
nyc_open_data_service: OpenDataService = OpenDataService()
open_data_response: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query)
LOG.debug(f'Violation data: {open_data_response}')
if open_data_response.success:
open_data_plate_lookup: OpenDataServicePlateLookup = open_data_response.data
bus_lane_camera_violations = 0
red_light_camera_violations = 0
speed_camera_violations = 0
for violation_type_summary in open_data_plate_lookup.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary['title'] == 'Bus Lane Violation':
bus_lane_camera_violations = violation_count
if violation_type_summary[
'title'] == 'Failure To Stop At Red Light':
red_light_camera_violations = violation_count
elif violation_type_summary[
'title'] == 'School Zone Speed Camera Violation':
speed_camera_violations = violation_count
camera_streak_data: CameraStreakData = open_data_plate_lookup.camera_streak_data
# If this came from message, add it to the plate_lookups table.
if plate_query.message_source and plate_query.message_id and plate_query.created_at:
new_lookup = PlateLookup(
boot_eligible=camera_streak_data.max_streak >= 5
if camera_streak_data else False,
bus_lane_camera_violations=bus_lane_camera_violations,
created_at=plate_query.created_at,
message_id=plate_query.message_id,
message_source=plate_query.message_source,
num_tickets=open_data_plate_lookup.num_violations,
plate=plate_query.plate,
plate_types=plate_query.plate_types,
red_light_camera_violations=red_light_camera_violations,
speed_camera_violations=speed_camera_violations,
state=plate_query.state,
unique_identifier=unique_identifier,
username=plate_query.username)
# Iterate through included campaigns to tie lookup to each
for campaign in campaigns:
# insert join record for campaign lookup
new_lookup.campaigns.append(campaign)
# Insert plate lookup
PlateLookup.query.session.add(new_lookup)
PlateLookup.query.session.commit()
else:
LOG.info(f'open data plate lookup failed')
return open_data_response
def _proceess_lookup_results(self, plate_query: PlateLookup):
pass
def _query_for_lookup_frequency(self, plate_query: PlateQuery) -> int:
return len(
PlateLookup.get_all_by(plate=plate_query.plate,
plate_types=plate_query.plate_types,
state=plate_query.state,
count_towards_frequency=True))
def _query_for_previous_lookup(
self, plate_query: PlateQuery) -> Optional[PlateLookup]:
""" See if we've seen this vehicle before. """
lookups_for_vehicle: List[PlateLookup] = PlateLookup.get_all_by(
plate=plate_query.plate,
state=plate_query.state,
plate_types=plate_query.plate_types,
count_towards_frequency=True)
if lookups_for_vehicle:
lookups_for_vehicle.sort(key=lambda x: x.created_at, reverse=True)
return lookups_for_vehicle[0]
else:
return None
def perform(self, *args, **kwargs):
is_dry_run: bool = kwargs.get('is_dry_run') or False
use_dvaa_thresholds: bool = kwargs.get('use_dvaa_thresholds') or False
use_only_visible_tweets: bool = kwargs.get('use_only_visible_tweets') or False
tweet_detection_service = TweetDetectionService()
tweeter = TrafficViolationsTweeter()
eastern = pytz.timezone('US/Eastern')
utc = pytz.timezone('UTC')
now = datetime.now()
# If today is leap day, there were no lookups a year ago today
if (now.day == self.LEAP_DAY_DATE and
now.month == self.LEAP_DAY_MONTH):
return
# If today is March 1, and last year was a leap year, show lookups
# from the previous February 29.
one_year_after_leap_day = True if (
now.day == self.POST_LEAP_DAY_DATE and
now.month == self.POST_LEAP_DAY_MONTH and
self._is_leap_year(now.year - 1)) else False
top_of_the_hour_last_year = now.replace(
microsecond=0,
minute=0,
second=0) - relativedelta(years=1)
top_of_the_next_hour_last_year = (
top_of_the_hour_last_year + relativedelta(hours=1))
if use_dvaa_thresholds:
threshold_attribute: str = 'boot_eligible_under_dvaa_threshold'
else:
threshold_attribute: str = 'boot_eligible_under_rdaa_threshold'
base_query: list[Tuple[int]] = PlateLookup.query.session.query(
func.max(PlateLookup.id).label('most_recent_vehicle_lookup')
).filter(
or_(
and_(PlateLookup.created_at >= top_of_the_hour_last_year,
PlateLookup.created_at < top_of_the_next_hour_last_year,
getattr(PlateLookup, threshold_attribute) == True,
PlateLookup.count_towards_frequency == True),
and_(one_year_after_leap_day,
PlateLookup.created_at >= (top_of_the_hour_last_year - relativedelta(days=1)),
PlateLookup.created_at < (top_of_the_next_hour_last_year - relativedelta(days=1)),
getattr(PlateLookup, threshold_attribute) == True,
PlateLookup.count_towards_frequency == True)
)
)
if use_only_visible_tweets:
base_query = base_query.filter(
PlateLookup.message_source == lookup_sources.LookupSource.STATUS.value)
recent_plate_lookup_ids = base_query.group_by(
PlateLookup.plate,
PlateLookup.state
).all()
lookup_ids_to_update: list[int] = [id[0] for id in recent_plate_lookup_ids]
lookups_to_update: list[PlateLookup] = PlateLookup.get_all_in(
id=lookup_ids_to_update)
if not lookups_to_update:
LOG.debug(f'No vehicles for which to perform retrospective job '
f'between {top_of_the_hour_last_year} and '
f'and {top_of_the_next_hour_last_year}.')
for previous_lookup in lookups_to_update:
LOG.debug(f'Performing retrospective job for '
f'{L10N.VEHICLE_HASHTAG.format(previous_lookup.state, previous_lookup.plate)} ')
plate_query: PlateQuery = PlateQuery(created_at=now,
message_source=previous_lookup.message_source,
plate=previous_lookup.plate,
plate_types=previous_lookup.plate_types,
state=previous_lookup.state)
nyc_open_data_service: OpenDataService = OpenDataService()
data_before_query: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query,
until=previous_lookup.created_at)
lookup_before_query: OpenDataServicePlateLookup = data_before_query.data
camera_streak_data_before_query: CameraStreakData = lookup_before_query.camera_streak_data['Mixed']
data_after_query: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
plate_query=plate_query,
since=previous_lookup.created_at,
until=previous_lookup.created_at + relativedelta(years=1))
lookup_after_query: OpenDataServicePlateLookup = data_after_query.data
new_bus_lane_camera_violations: Optional[int] = None
new_speed_camera_violations: Optional[int] = None
new_red_light_camera_violations: Optional[int] = None
for violation_type_summary in lookup_after_query.violations:
if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
violation_count = violation_type_summary['count']
if violation_type_summary['title'] == 'Bus Lane Violation':
new_bus_lane_camera_violations = violation_count
if violation_type_summary['title'] == 'Failure To Stop At Red Light':
new_red_light_camera_violations = violation_count
if violation_type_summary['title'] == 'School Zone Speed Camera Violation':
new_speed_camera_violations = violation_count
if new_bus_lane_camera_violations is None:
new_bus_lane_camera_violations = 0
if new_red_light_camera_violations is None:
new_red_light_camera_violations = 0
if new_speed_camera_violations is None:
new_speed_camera_violations = 0
new_boot_eligible_violations = (new_red_light_camera_violations +
new_speed_camera_violations)
if new_boot_eligible_violations > 0:
vehicle_hashtag = L10N.VEHICLE_HASHTAG.format(
previous_lookup.state, previous_lookup.plate)
previous_lookup_created_at = utc.localize(
previous_lookup.created_at)
previous_lookup_date = previous_lookup_created_at.astimezone(eastern).strftime(
L10N.REPEAT_LOOKUP_DATE_FORMAT)
previous_lookup_time = previous_lookup_created_at.astimezone(eastern).strftime(
L10N.REPEAT_LOOKUP_TIME_FORMAT)
red_light_camera_violations_string = (
f'{new_red_light_camera_violations} | Red Light Camera Violations\n'
if new_red_light_camera_violations > 0 else '')
speed_camera_violations_string = (
f'{new_speed_camera_violations} | Speed Safety Camera Violations\n'
if new_speed_camera_violations > 0 else '')
reckless_driver_summary_string = (
f'{vehicle_hashtag} was originally '
f'queried on {previous_lookup_date} '
f'at {previous_lookup_time}')
# assume we can't link
can_link_tweet = False
# Where did this come from?
if previous_lookup.message_source == lookup_sources.LookupSource.STATUS.value:
# Determine if tweet is still visible:
if tweet_detection_service.tweet_exists(id=previous_lookup.message_id,
username=previous_lookup.username):
can_link_tweet = True
if can_link_tweet:
reckless_driver_summary_string += L10N.PREVIOUS_LOOKUP_STATUS_STRING.format(
previous_lookup.username,
previous_lookup.username,
previous_lookup.message_id)
else:
reckless_driver_summary_string += '.'
if use_only_visible_tweets and not can_link_tweet:
# If we're only displaying tweets we can quote tweet,
# skip this one since we can'tt.
continue
reckless_driver_update_string = (
f'From {camera_streak_data_before_query.min_streak_date} to '
f'{camera_streak_data_before_query.max_streak_date}, this vehicle '
f'received {camera_streak_data_before_query.max_streak} camera '
f'violations. Over the past 12 months, this vehicle '
f'received {new_boot_eligible_violations} new camera violation'
f"{'' if new_boot_eligible_violations == 1 else 's'}: \n\n"
f'{red_light_camera_violations_string}'
f'{speed_camera_violations_string}')
messages: list[str] = [
reckless_driver_summary_string,
reckless_driver_update_string]
if not is_dry_run:
success: bool = tweeter.send_status(
message_parts=messages,
on_error_message=(
f'Error printing reckless driver update. '
f'Tagging @bdhowald.'))
if success:
LOG.debug('Reckless driver retrospective job '
'ran successfully.')
else:
print(reckless_driver_update_string)
class TrafficViolationsTweeter:
PRODUCTION_APP_RATE_LIMITING_INTERVAL_IN_SECONDS = 3.0
DEVELOPMENT_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS = 3000.0
PRODUCTION_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS = 300.0
FOLLOWERS_RATE_LIMITING_INTERVAL_IN_SECONDS = 15 * SECONDS_PER_MINUTE
MAX_DIRECT_MESSAGES_RETURNED = 50
def __init__(self):
self._app_api = None
self._client_api = None
# Create reply argument_builder
self.reply_argument_builder = ReplyArgumentBuilder(self._get_twitter_application_api())
# Create new aggregator
self.aggregator = TrafficViolationsAggregator()
# Need to find if unanswered statuses have been deleted
self.tweet_detection_service = TweetDetectionService()
# Log how many times we've called the apis
self._direct_messages_iteration = 0
self._events_iteration = 0
self._statuses_iteration = 0
self._lookup_threads = []
# Initialize cached values to None
self._follower_ids: Optional[list[int]] = None
self._follower_ids_last_fetched: Optional[datetime] = None
def find_and_respond_to_requests(self) -> None:
"""Convenience method to collect the different ways TwitterEvent
objects are created, found, and responded to and begin the process
of calling these methods at process start.
"""
self._find_and_respond_to_missed_direct_messages()
self._find_and_respond_to_missed_statuses()
self._find_and_respond_to_twitter_events()
def send_status(self,
message_parts: Union[list[any], list[str]],
on_error_message: str) -> bool:
"""Send statuses from @HowsMyDrivingNY"""
try:
self._recursively_process_status_updates(message_parts)
return True
except Exception as e:
LOG.error(e)
self._recursively_process_status_updates(on_error_message)
return False
def terminate_lookups(self) -> None:
"""Stop looking for twitter events, statuses, or direct messages to respond to."""
for thread in self._lookup_threads:
thread.cancel()
def _add_twitter_events_for_missed_direct_messages(self, messages: list[tweepy.models.Status]) -> None:
"""Creates TwitterEvent objects when the Account Activity API fails to send us
direct message events via webhooks to have them created by the HowsMyDrivingNY API.
:param messages: list[tweepy.DirectMessage]: The direct messages returned via the
Twitter Search API via Tweepy.
"""
undetected_messages = 0
sender_ids = set(int(message.message_create['sender_id']) for message in messages)
sender_objects = self._get_twitter_client_api().lookup_users(user_id=sender_ids)
senders = {sender.id_str:sender for sender in sender_objects}
for message in messages:
existing_event: Optional[TwitterEvent] = TwitterEvent.query.filter(TwitterEvent.event_id == message.id).first()
if not existing_event and int(message.message_create['sender_id']) != HMDNY_TWITTER_USER_ID:
undetected_messages += 1
sender = senders[message.message_create['sender_id']]
event: TwitterEvent = TwitterEvent(
event_type=TwitterMessageType.DIRECT_MESSAGE.value,
event_id=message.id,
user_handle=sender.screen_name,
user_id=sender.id,
event_text=message.message_create['message_data']['text'],
created_at=message.created_timestamp,
in_reply_to_message_id=None,
location=None,
user_mention_ids=','.join([user['id_str'] for user in message.message_create['message_data']['entities']['user_mentions']]),
user_mentions=' '.join([user['screen_name'] for user in message.message_create['message_data']['entities']['user_mentions']]),
detected_via_account_activity_api=False)
TwitterEvent.query.session.add(event)
TwitterEvent.query.session.commit()
LOG.debug(
f"Found {undetected_messages} direct message{'' if undetected_messages == 1 else 's'} that "
f"{'was' if undetected_messages == 1 else 'were'} previously undetected.")
def _add_twitter_events_for_missed_statuses(self, messages: list[tweepy.models.Status]) -> None:
"""Creates TwitterEvent objects when the Account Activity API fails to send us
status events via webhooks to have them created by the HowsMyDrivingNY API.
:param messages: list[tweepy.models.Status]: The statuses returned via the Twitter
Search API via Tweepy.
"""
undetected_messages = 0
for message in messages:
existing_event: Optional[TwitterEvent] = TwitterEvent.query.filter(TwitterEvent.event_id == message.id).first()
if not existing_event and message.user.id != HMDNY_TWITTER_USER_ID:
undetected_messages += 1
event: TwitterEvent = TwitterEvent(
event_type=TwitterMessageType.STATUS.value,
event_id=message.id,
user_handle=message.user.screen_name,
user_id=message.user.id,
event_text=message.full_text,
created_at=message.created_at.replace(tzinfo=pytz.timezone('UTC')).timestamp() * MILLISECONDS_PER_SECOND,
in_reply_to_message_id=message.in_reply_to_status_id,
location=message.place and message.place.full_name,
user_mention_ids=','.join([user['id_str'] for user in message.entities['user_mentions']]),
user_mentions=' '.join([user['screen_name'] for user in message.entities['user_mentions']]),
detected_via_account_activity_api=False)
TwitterEvent.query.session.add(event)
TwitterEvent.query.session.commit()
LOG.debug(
f"Found {undetected_messages} status{'' if undetected_messages == 1 else 'es'} that "
f"{'was' if undetected_messages == 1 else 'were'} previously undetected.")
def _filter_failed_twitter_events(self, failed_events: list[TwitterEvent]) -> list[TwitterEvent]:
failed_events_that_need_response: list[TwitterEvent] = []
for failed_event in failed_events:
# If event is a tweet, but can no longer be found, there's nothing we can do.
if (failed_event.event_type == TwitterMessageType.STATUS.value and
not self.tweet_detection_service.tweet_exists(id=failed_event.event_id,
username=failed_event.user_handle)):
failed_event.error_on_lookup = False
failed_event.num_times_failed = 0
failed_event.last_failed_at_time = None
failed_event.query.session.commit()
continue
if failed_event.num_times_failed == 0:
failed_events_that_need_response.append(failed_event)
elif failed_event.num_times_failed == 1:
time_to_retry = failed_event.last_failed_at_time + timedelta(minutes=5)
if time_to_retry <= datetime.utcnow():
failed_events_that_need_response.append(failed_event)
elif failed_event.num_times_failed == 2:
time_to_retry = failed_event.last_failed_at_time + timedelta(hours=1)
if time_to_retry <= datetime.utcnow():
failed_events_that_need_response.append(failed_event)
elif failed_event.num_times_failed == 3:
time_to_retry = failed_event.last_failed_at_time + timedelta(hours=3)
if time_to_retry <= datetime.utcnow():
failed_events_that_need_response.append(failed_event)
elif failed_event.num_times_failed == 4:
time_to_retry = failed_event.last_failed_at_time + timedelta(days=1)
if time_to_retry <= datetime.utcnow():
failed_events_that_need_response.append(failed_event)
else:
LOG.debug(f'Event response cannot be retried automatically.')
LOG.debug(f'failed events to retry: {failed_events_that_need_response}')
return failed_events_that_need_response
def _find_and_respond_to_missed_direct_messages(self) -> None:
"""Uses Tweepy to call the Twitter Search API to find direct messages to/from
HowsMyDrivingNY. It then passes this data to a function that creates
TwitterEvent objects when those direct message events have not already been recorded.
"""
interval = (self.PRODUCTION_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS if self._is_production()
else self.DEVELOPMENT_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS)
self._direct_messages_iteration += 1
LOG.debug(
f'Looking up missed direct messages on iteration {self._direct_messages_iteration}')
# set up timer
direct_message_thread = threading.Timer(
interval, self._find_and_respond_to_missed_direct_messages)
self._lookup_threads.append(direct_message_thread)
# start timer
direct_message_thread.start()
try:
# most_recent_undetected_twitter_event = TwitterEvent.query.filter(
# and_(TwitterEvent.detected_via_account_activity_api == False,
# TwitterEvent.event_type == TwitterMessageType.DIRECT_MESSAGE.value)
# ).order_by(TwitterEvent.event_id.desc()).first()
# Tweepy bug with cursors prevents us from searching for more than 50 events
# at a time until 3.9, so it'll have to do.
direct_messages_since_last_twitter_event = self._get_twitter_client_api(
).get_direct_messages(
count=self.MAX_DIRECT_MESSAGES_RETURNED)
received_messages = [message for message in direct_messages_since_last_twitter_event if
int(message.message_create['sender_id']) != HMDNY_TWITTER_USER_ID]
self._add_twitter_events_for_missed_direct_messages(messages=received_messages)
except Exception as e:
LOG.error(e)
LOG.error(str(e))
LOG.error(e.args)
logging.exception("stack trace")
finally:
TwitterEvent.query.session.close()
def _find_and_respond_to_missed_statuses(self) -> None:
"""Uses Tweepy to call the Twitter Search API to find statuses mentioning
HowsMyDrivingNY. It then passes this data to a function that creates
TwitterEvent objects when those status events have not already been recorded.
"""
interval = (self.PRODUCTION_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS if self._is_production()
else self.DEVELOPMENT_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS)
self._statuses_iteration += 1
LOG.debug(
f'Looking up missed statuses on iteration {self._statuses_iteration}')
# set up timer
statuses_thread = threading.Timer(
interval, self._find_and_respond_to_missed_statuses)
self._lookup_threads.append(statuses_thread)
# start timer
statuses_thread.start()
try:
# Find most recent undetected twitter status event, and then
# search for recent events until we can find no more.
most_recent_undetected_twitter_event = TwitterEvent.query.filter(
and_(TwitterEvent.detected_via_account_activity_api == False,
TwitterEvent.event_type == TwitterMessageType.STATUS.value)
).order_by(TwitterEvent.event_id.desc()).first()
if most_recent_undetected_twitter_event:
statuses_since_last_twitter_event: list[tweepy.models.Status] = []
max_status_id: Optional[int] = None
while max_status_id is None or statuses_since_last_twitter_event:
statuses_since_last_twitter_event = self._get_twitter_client_api(
).mentions_timeline(
max_id=max_status_id,
since_id=most_recent_undetected_twitter_event.event_id,
tweet_mode='extended')
if statuses_since_last_twitter_event:
self._add_twitter_events_for_missed_statuses(statuses_since_last_twitter_event)
max_status_id = statuses_since_last_twitter_event[-1].id - 1
else:
max_status_id = most_recent_undetected_twitter_event.event_id - 1
except Exception as e:
LOG.error(e)
LOG.error(str(e))
LOG.error(e.args)
logging.exception("stack trace")
finally:
TwitterEvent.query.session.close()
def _find_and_respond_to_twitter_events(self) -> None:
"""Looks for TwitterEvent objects that have not yet been responded to and
begins the process of creating a response. Additionally, failed events are
rerun to provide a correct response, particularly useful in cases where
external apis are down for maintenance.
"""
interval = (
self.PRODUCTION_APP_RATE_LIMITING_INTERVAL_IN_SECONDS if self._is_production()
else self.DEVELOPMENT_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS)
self._events_iteration += 1
LOG.debug(
f'Looking up twitter events on iteration {self._events_iteration}')
# set up timer
twitter_events_thread = threading.Timer(
interval, self._find_and_respond_to_twitter_events)
self._lookup_threads.append(twitter_events_thread)
# start timer
twitter_events_thread.start()
try:
new_events: list[TwitterEvent] = TwitterEvent.get_all_by(
is_duplicate=False,
responded_to=False,
response_in_progress=False)
LOG.debug(f'new events: {new_events}')
failed_events: list[TwitterEvent] = TwitterEvent.get_all_by(
is_duplicate=False,
error_on_lookup=True,
responded_to=True,
response_in_progress=False)
failed_events_that_need_response: list[TwitterEvent] = self._filter_failed_twitter_events(failed_events)
events_to_respond_to: [list[TwitterEvent]] = new_events + failed_events_that_need_response
LOG.debug(f'events to respond to: {events_to_respond_to}')
for event in events_to_respond_to:
self._process_twitter_event(event=event)
except Exception as e:
LOG.error(e)
LOG.error(str(e))
LOG.error(e.args)
logging.exception("stack trace")
finally:
TwitterEvent.query.session.close()
def _get_twitter_application_api(self) -> twitter_api_wrapper.TwitterApplicationApiWrapper:
"""Set the application (non-client) api connection for this instance"""
if not self._app_api:
self._app_api = twitter_api_wrapper.TwitterApplicationApiWrapper().get_connection()
return self._app_api
def _get_twitter_client_api(self) -> twitter_api_wrapper.TwitterClientApiWrapper:
"""Set the client api connection for this instance"""
if not self._client_api:
self._client_api = twitter_api_wrapper.TwitterClientApiWrapper().get_connection()
return self._client_api
def _get_follower_ids(self):
"""Get list of followers from Twitter every 15 minutes.
This is used to determine who should be prompted to like
the reply tweet in order to trigger a response.
If the cached value is older than 15 minutes, refetch.
"""
now = datetime.utcnow()
# If cache is empty or stale, refetch.
if not self._follower_ids_last_fetched or (
((now - self._follower_ids_last_fetched).seconds
> self.FOLLOWERS_RATE_LIMITING_INTERVAL_IN_SECONDS)):
follower_ids = []
next_cursor: int = -1
while next_cursor:
results, cursors = self._get_twitter_application_api().get_follower_ids(cursor=next_cursor)
next_cursor = cursors[1]
follower_ids += results
# set follower_ids
self._follower_ids = follower_ids
# cache current time
self._follower_ids_last_fetched = now
# Return cached or fetched value.
return self._follower_ids
def _is_production(self):
"""Determines if we are running in production to see if we can create
direct message and status responses.
TODO: Come up with a better way to determine production environment.
"""
return os.getenv('ENV') == 'production'
def _process_response(self,
request_object: Type[BaseLookupRequest],
response_parts: list[Any],
successful_lookup: bool = False) -> Optional[int]:
"""Directs the response to a Twitter message, depending on whether
or not the event is a direct message or a status. Statuses that mention
HowsMyDrivingNY earn a favorite/like.
"""
message_source = request_object.message_source if request_object else None
message_id = request_object.external_id() if request_object else None
# Respond to user
if message_source == LookupSource.DIRECT_MESSAGE.value:
LOG.debug('responding as direct message')
combined_message = self._recursively_compile_direct_messages(
response_parts)
LOG.debug(f'combined_message: {combined_message}')
return self._send_direct_message(message=combined_message,
recipient_id=request_object.user_id)
elif message_source == LookupSource.STATUS.value:
# If we have at least one successful lookup, favorite the status
if successful_lookup:
# Favorite every look-up from a status
try:
self._is_production() and self._get_twitter_application_api(
).create_favorite(message_id)
# But don't crash on error
except tweepy.error.TweepError as te:
# There's no easy way to know if this status has already
# been favorited
pass
LOG.debug('responding as status update')
user_mention_ids = (request_object.mentioned_user_ids if
request_object.mentioned_user_ids else None)
return self._recursively_process_status_updates(
response_parts=response_parts,
message_id=message_id,
user_mention_ids=user_mention_ids)
else:
LOG.error('Unkown message source. Cannot respond.')
def _process_twitter_event(self, event: TwitterEvent):
LOG.debug(f'Beginning response for event: {event.id}')
# search for duplicates
is_event_duplicate: bool = TwitterEvent.query.filter_by(
event_type=event.event_type,
event_id=event.event_id,
user_handle=event.user_handle,
responded_to=True
).filter(
TwitterEvent.id != event.id).count() > 0
if is_event_duplicate:
event.is_duplicate = True
TwitterEvent.query.session.commit()
LOG.info(f'Event {event.id} is a duplicate, skipping.')
else:
event.response_in_progress = True
TwitterEvent.query.session.commit()
try:
message_source: str = LookupSource(event.event_type)
# build request
lookup_request: Type[BaseLookupRequest] = self.reply_argument_builder.build_reply_data(
message=event,
message_source=message_source)
user_is_follower: bool = lookup_request.requesting_user_is_follower(
follower_ids=self._get_follower_ids())
perform_lookup_for_user: bool = (user_is_follower or
event.user_favorited_non_follower_reply)
if self.aggregator.lookup_has_valid_plates(
lookup_request=lookup_request) and not perform_lookup_for_user:
response_parts: list[Any]
if lookup_request.is_direct_message():
response_parts = [L10N.NON_FOLLOWER_DIRECT_MESSAGE_REPLY_STRING]
elif lookup_request.is_status():
response_parts = [L10N.NON_FOLLOWER_TWEET_REPLY_STRING]
try:
reply_message_id = self._process_response(
request_object=lookup_request,
response_parts=response_parts)
# Save the reply id, so that when the user favorites it,
# we can trigger the search.
non_follower_reply = NonFollowerReply(
created_at=(int(datetime.utcnow().timestamp() *
MILLISECONDS_PER_SECOND)),
event_type=event.event_type,
event_id=reply_message_id,
in_reply_to_message_id=event.event_id,
user_handle=event.user_handle,
user_id=event.user_id)
NonFollowerReply.query.session.add(
non_follower_reply)
NonFollowerReply.query.session.commit()
except tweepy.error.TweepError as e:
event.error_on_lookup = True
event.num_times_failed += 1
event.last_failed_at_time = datetime.utcnow()
else:
# Reply to the event.
reply_to_event = self.aggregator.initiate_reply(
lookup_request=lookup_request)
success = reply_to_event['success']
if success:
# There's no need to tell people that
# there was an error more than once.
if not (reply_to_event[
'error_on_lookup'] and event.error_on_lookup):
try:
self._process_response(
request_object=reply_to_event['request_object'],
response_parts=reply_to_event['response_parts'],
successful_lookup=reply_to_event.get('successful_lookup'))
except tweepy.error.TweepError as e:
reply_to_event['error_on_lookup'] = True
# Update error status
if reply_to_event['error_on_lookup']:
event.error_on_lookup = True
event.num_times_failed += 1
event.last_failed_at_time = datetime.utcnow()
else:
event.error_on_lookup = False
event.num_times_failed = 0
event.last_failed_at_time = None
# We've responded!
event.response_in_progress = False
event.responded_to = True
TwitterEvent.query.session.commit()
except ValueError as e:
LOG.error(
f'Encountered unknown event type. '
f'Response is not possible.')
def _recursively_compile_direct_messages(self, response_parts):
"""Direct message responses from the aggregator return lists
of chunked information (by violation type, by borough, by year, etc.).
Data look like:
[
[data_type_1_part_1, data_type_1_part_2, data_type_1_part_3],
[data_type_2_part_1, data_type_2_part_2],
[data_type_3_part_1, data_type_3_part_2, data_type_3_part_3],
]
This ensures that the data is grouped with like data. Using recursion,
the final message is built into one large message.
"""
return_message = []
# Iterate through all response parts
for part in response_parts:
if isinstance(part, list):
return_message.append(
self._recursively_compile_direct_messages(part))
else:
return_message.append(part)
return '\n'.join(return_message)
def _recursively_process_status_updates(self,
response_parts: Union[list[any], list[str]],
message_id: Optional[int] = None,
user_mention_ids: Optional[list[str]] = None) -> Optional[int]:
"""Status responses from the aggregator return lists
of chunked information (by violation type, by borough, by year, etc.).
Data look like:
[
[data_type_1_part_1, data_type_1_part_2, data_type_1_part_3],
[data_type_2_part_1, data_type_2_part_2],
[data_type_3_part_1, data_type_3_part_2, data_type_3_part_3],
]
This ensures that the data is grouped with like data. Using recursion,
response statuses are created, then their message ids are saved to be
used as the in_reply_to_status_id for the next status.
"""
# Iterate through all response parts
for part in response_parts:
# Some may be lists themselves
if isinstance(part, list):
message_id = self._recursively_process_status_updates(
response_parts=part,
message_id=message_id,
user_mention_ids=user_mention_ids)
else:
if self._is_production():
new_message = self._get_twitter_application_api(
).update_status(
status=part,
in_reply_to_status_id=message_id,
exclude_reply_user_ids=user_mention_ids)
message_id = new_message.id
LOG.debug(f'message_id: {message_id}')
else:
LOG.debug(
"This is where 'self._get_twitter_application_api()"
".update_status(status=part, in_reply_to_status_id=message_id, "
"exclude_reply_user_ids=user_mention_ids)' "
"would be called in production.")
return None
return message_id
def _send_direct_message(self, message: str, recipient_id: int) -> Optional[int]:
"""Send a direct message to a Twitter user."""
if self._is_production():
new_message = self._get_twitter_application_api().send_direct_message(
recipient_id=recipient_id,
text=message)
return new_message.id
else:
LOG.debug(
"This is where 'self._get_twitter_application_api()"
".send_direct_message(recipient_id=recipient_id, "
"text=message)' would be called in production.")
return None