コード例 #1
0
    def __init__(self):

        self._app_api = None
        self._client_api = None

        # Create reply argument_builder
        self.reply_argument_builder = ReplyArgumentBuilder(self._get_twitter_application_api())

        # Create new aggregator
        self.aggregator = TrafficViolationsAggregator()

        # Need to find if unanswered statuses have been deleted
        self.tweet_detection_service = TweetDetectionService()

        # Log how many times we've called the apis
        self._direct_messages_iteration = 0
        self._events_iteration = 0
        self._statuses_iteration = 0

        self._lookup_threads = []

        # Initialize cached values to None
        self._follower_ids: Optional[list[int]] = None
        self._follower_ids_last_fetched: Optional[datetime] = None
コード例 #2
0
    def __init__(self):
        self.tweet_detection_service = TweetDetectionService()

        self.eastern = pytz.timezone('US/Eastern')
        self.utc = pytz.timezone('UTC')
コード例 #3
0
class TrafficViolationsAggregator:

    CAMERA_VIOLATIONS = [
        'Bus Lane Violation', 'Failure To Stop At Red Light',
        'School Zone Speed Camera Violation'
    ]

    MYSQL_TIME_FORMAT: str = '%Y-%m-%d %H:%M:%S'

    UNIQUE_IDENTIFIER_STRING_LENGTH = 8

    def __init__(self):
        self.tweet_detection_service = TweetDetectionService()

        self.eastern = pytz.timezone('US/Eastern')
        self.utc = pytz.timezone('UTC')

    def initiate_reply(self, lookup_request: Type[BaseLookupRequest]):
        LOG.info('Calling initiate_reply')

        if lookup_request.requires_response():
            return self._create_response(lookup_request)

    def _create_repeat_lookup_string(
            self,
            new_violations: int,
            plate: str,
            state: str,
            username: str,
            previous_lookup: Optional[Dict[str, Any]] = None):

        violations_string = ''

        if new_violations > 0:

            # assume we can't link
            can_link_tweet = False

            # Where did this come from?
            if previous_lookup.message_source == lookup_sources.LookupSource.STATUS.value:
                # Determine if tweet is still visible:
                if self.tweet_detection_service.tweet_exists(
                        id=previous_lookup.message_id, username=username):
                    can_link_tweet = True

            # Determine when the last lookup was...
            previous_time = previous_lookup.created_at
            now = datetime.now()

            adjusted_time = self.utc.localize(previous_time)
            adjusted_now = self.utc.localize(now)

            # If at least five minutes have passed...
            if adjusted_now - timedelta(minutes=5) > adjusted_time:

                # Add the new ticket info and previous lookup time to the string.
                violations_string += L10N.LAST_QUERIED_STRING.format(
                    adjusted_time.astimezone(self.eastern).strftime(
                        L10N.REPEAT_LOOKUP_DATE_FORMAT),
                    adjusted_time.astimezone(self.eastern).strftime(
                        L10N.REPEAT_LOOKUP_TIME_FORMAT))

                if can_link_tweet:
                    violations_string += L10N.PREVIOUS_LOOKUP_STATUS_STRING.format(
                        previous_lookup.username, previous_lookup.username,
                        previous_lookup.message_id)
                else:
                    violations_string += '.'

                violations_string += L10N.REPEAT_LOOKUP_STRING.format(
                    L10N.VEHICLE_HASHTAG.format(state, plate), new_violations,
                    L10N.pluralize(new_violations))

        return violations_string

    def _create_response(self, request_object: Type[BaseLookupRequest]):

        LOG.info('\n')
        LOG.info('Calling create_response')

        # Grab tweet details for reply.
        LOG.debug(f'request_object: {request_object}')

        # Collect response parts here.
        response_parts = []
        successful_lookup = False
        error_on_lookup = False

        # Wrap in try/catch block
        try:

            # Find potential plates
            potential_vehicles: List[Vehicle] = self._find_potential_vehicles(
                request_object.string_tokens())
            LOG.debug(f'potential_vehicles: {potential_vehicles}')

            potential_vehicles += self._find_potential_vehicles_using_legacy_logic(
                request_object.legacy_string_tokens())
            LOG.debug(f'potential_vehicles: {potential_vehicles}')

            potential_vehicles = self._ensure_unique_plates(
                vehicles=potential_vehicles)

            # Find included campaign hashtags
            included_campaigns: List[Campaign] = self._detect_campaigns(
                request_object.string_tokens())
            LOG.debug(f'included_campaigns: {included_campaigns}')

            # for each vehicle, we need to determine if the supplied information amounts to a valid plate
            # then we need to look up each valid plate
            # then we need to respond in a single thread in order with the responses

            summary: TrafficViolationsAggregatorResponse = TrafficViolationsAggregatorResponse(
            )

            for potential_vehicle in potential_vehicles:

                if potential_vehicle.valid_plate:

                    plate_query: PlateQuery = self._get_plate_query(
                        vehicle=potential_vehicle,
                        request_object=request_object)

                    # do we have a previous lookup
                    previous_lookup: Optional[
                        PlateLookup] = self._query_for_previous_lookup(
                            plate_query=plate_query)
                    LOG.debug(
                        f'Previous lookup for this vehicle: {previous_lookup}')

                    # Obtain a unique identifier for the lookup
                    unique_identifier: str = self._get_unique_identifier()

                    # Do the real work!
                    open_data_response: OpenDataServiceResponse = self._perform_plate_lookup(
                        campaigns=included_campaigns,
                        plate_query=plate_query,
                        unique_identifier=unique_identifier)

                    if open_data_response.success:

                        # Record successful lookup.
                        successful_lookup = True

                        plate_lookup: OpenDataServicePlateLookup = open_data_response.data

                        # how many times have we searched for this plate from a tweet
                        current_frequency: int = self._query_for_lookup_frequency(
                            plate_query)

                        # self._proceess_lookup_results()

                        # Add lookup to summary
                        summary.plate_lookups.append(plate_lookup)

                        if plate_lookup.violations:

                            plate_lookup_response_parts: List[
                                Any] = self._form_plate_lookup_response_parts(
                                    borough_data=plate_lookup.boroughs,
                                    camera_streak_data=plate_lookup.
                                    camera_streak_data,
                                    fine_data=plate_lookup.fines,
                                    frequency=current_frequency,
                                    plate=plate_lookup.plate,
                                    plate_types=plate_lookup.plate_types,
                                    previous_lookup=previous_lookup,
                                    state=plate_lookup.state,
                                    username=request_object.username(),
                                    unique_identifier=unique_identifier,
                                    violations=plate_lookup.violations,
                                    year_data=plate_lookup.years)

                            response_parts.append(plate_lookup_response_parts)
                            # [[campaign_stuff], tickets_0, tickets_1, etc.]

                        else:
                            # Let user know we didn't find anything.
                            plate_types_string = (
                                f' (types: {plate_query.plate_types})'
                            ) if plate_lookup.plate_types else ''
                            response_parts.append(
                                L10N.NO_TICKETS_FOUND_STRING.format(
                                    plate_query.state, plate_lookup.plate,
                                    plate_types_string))

                    else:

                        # Record lookup error.
                        error_on_lookup = True

                        response_parts.append([
                            f"Sorry, I received an error when looking up "
                            f"{plate_query.state}:{plate_query.plate}"
                            f"{(' (types: ' + plate_query.plate_types + ')') if plate_query.plate_types else ''}. "
                            f"Please try again."
                        ])

                else:

                    # Record the failed lookup.
                    new_failed_lookup = FailedPlateLookup(
                        message_id=request_object.external_id(),
                        username=request_object.username())

                    # Insert plate lookup
                    FailedPlateLookup.query.session.add(new_failed_lookup)
                    FailedPlateLookup.query.session.commit()

                    # Legacy data where state is not a valid abbreviation.
                    if potential_vehicle.state:
                        LOG.debug("We have a state, but it's invalid.")

                        response_parts.append([
                            f"The state should be two characters, but you supplied '{potential_vehicle.state}'. "
                            f"Please try again."
                        ])

                    # '<state>:<plate>' format, but no valid state could be detected.
                    elif potential_vehicle.original_string:
                        LOG.debug(
                            "We don't have a state, but we have an attempted lookup with the new format."
                        )

                        response_parts.append([
                            f"Sorry, a plate and state could not be inferred from "
                            f"{potential_vehicle.original_string}."
                        ])

                    # If we have a plate, but no state.
                    elif potential_vehicle.plate:
                        LOG.debug("We have a plate, but no state")

                        response_parts.append(
                            ["Sorry, the state appears to be blank."])

            # If we have multiple vehicles, prepend a summary.
            if len(summary.plate_lookups) > 1:

                summary_string: Optional[str] = self._form_summary_string(
                    summary)

                if summary_string:
                    response_parts.insert(0, summary_string)

            # Look up campaign hashtags after doing the plate lookups and then
            # prepend to response.
            if included_campaigns:
                campaign_lookups: List[Tuple[
                    str, int,
                    int]] = self._perform_campaign_lookup(included_campaigns)
                response_parts.insert(
                    0,
                    self._form_campaign_lookup_response_parts(
                        campaign_lookups, request_object.username()))

                successful_lookup = True

            # If we don't look up a single plate successfully,
            # figure out how we can help the user.
            if not successful_lookup and not error_on_lookup:

                # Record the failed lookup.
                new_failed_lookup = FailedPlateLookup(
                    message_id=request_object.external_id(),
                    username=request_object.username())

                # Insert plate lookup
                FailedPlateLookup.query.session.add(new_failed_lookup)
                FailedPlateLookup.query.session.commit()

                LOG.debug('The data seems to be in the wrong format.')

                state_matches = [
                    regexp_constants.STATE_ABBREVIATIONS_PATTERN.search(
                        s.upper()) != None
                    for s in request_object.string_tokens()
                ]
                number_matches = [
                    regexp_constants.NUMBER_PATTERN.search(s.upper()) != None
                    for s in list(
                        filter(
                            lambda part: re.sub(r'\.|@', '', part.lower())
                            not in set(request_object.mentioned_users),
                            request_object.string_tokens()))
                ]

                # We have what appears to be a plate and a state abbreviation.
                if all([any(state_matches), any(number_matches)]):
                    LOG.debug(
                        'There is both plate and state information in this message.'
                    )

                    # Let user know plate format
                    response_parts.append([
                        "I’d be happy to look that up for you!\n\nJust a reminder, the format is <state|province|territory>:<plate>, e.g. NY:abc1234"
                    ])

                # Maybe we have plate or state. Let's find out.
                else:
                    LOG.debug(
                        'The tweet is missing either state or plate or both.')

                    state_minus_words_matches = [
                        regexp_constants.STATE_MINUS_WORDS_PATTERN.search(
                            s.upper()) != None
                        for s in request_object.string_tokens()
                    ]

                    number_matches = [
                        regexp_constants.NUMBER_PATTERN.search(s.upper()) !=
                        None for s in list(
                            filter(
                                lambda part: re.sub(r'\.|@', '', part.lower())
                                not in set(request_object.mentioned_users),
                                request_object.string_tokens()))
                    ]

                    # We have either plate or state.
                    if any(state_minus_words_matches) or any(number_matches):

                        # Let user know plate format
                        response_parts.append([
                            "I think you're trying to look up a plate, but can't be sure.\n\nJust a reminder, the format is <state|province|territory>:<plate>, e.g. NY:abc1234"
                        ])

                    # We have neither plate nor state. Do nothing.
                    else:
                        LOG.debug(
                            'ignoring message since no plate or state information to respond to.'
                        )

        except Exception as e:
            # Set response data
            error_on_lookup = True
            response_parts.append(
                ["Sorry, I encountered an error. Tagging @bdhowald."])

            # Log error
            LOG.error('Missing necessary information to continue')
            LOG.error(e)
            LOG.error(str(e))
            LOG.error(e.args)
            logging.exception("stack trace")

        # Indicate successful response processing.
        return {
            'error_on_lookup': error_on_lookup,
            'request_object': request_object,
            'response_parts': response_parts,
            'success': True,
            'successful_lookup': successful_lookup,
            'username': request_object.username()
        }

    def _detect_campaigns(self, string_tokens) -> List[Campaign]:
        """ Look for campaign hashtags in the message's text
        and return matching campaigns.

        """

        return Campaign.get_all_in(hashtag=tuple([
            regexp_constants.HASHTAG_PATTERN.sub('', string)
            for string in string_tokens
        ]))

    def _detect_plate_types(self, plate_types_input) -> bool:
        if ',' in plate_types_input:
            parts = plate_types_input.upper().split(',')
            return any([
                regexp_constants.PLATE_TYPES_PATTERN.search(part) != None
                for part in parts
            ])
        else:
            return regexp_constants.PLATE_TYPES_PATTERN.search(
                plate_types_input.upper()) != None

    def _detect_state(self, state_input) -> bool:
        # or state_full_pattern.search(state_input.upper()) != None
        """ Does this input constitute a valid state abbreviation """
        if state_input is not None:
            return regexp_constants.STATE_ABBREVIATIONS_PATTERN.search(
                state_input.upper()) != None

        return False

    def _ensure_unique_plates(self, vehicles: List[Vehicle]) -> List[Vehicle]:

        vehicle_dict: Dict[str, Vehicle] = {}
        unique_vehicles: List[Vehicle] = []

        for vehicle in vehicles:
            lookup_string = (
                f'{vehicle.state}:'
                f'{vehicle.plate}'
                f"{(':' + vehicle.plate_types) if vehicle.plate_types else ''}"
            )

            if vehicle_dict.get(lookup_string) is None:
                vehicle_dict[lookup_string] = vehicle
                unique_vehicles.append(vehicle)

        return unique_vehicles

    def _find_potential_vehicles(self,
                                 list_of_strings: List[str]) -> List[Vehicle]:

        # Use new logic of '<state>:<plate>'
        plate_tuples: Union[Tuple[str, str, str], Tuple[str, str]] = [
            [part.strip() for part in match.split(':')]
            for match in re.findall(regexp_constants.PLATE_FORMAT_REGEX,
                                    ' '.join(list_of_strings))
            if all(substr not in match.lower()
                   for substr in ['://', 'state:', 'plate:'])
        ]

        return self._infer_plate_and_state_data(plate_tuples)

    def _find_potential_vehicles_using_legacy_logic(
            self, list_of_strings: List[str]) -> List[Vehicle]:

        # Find potential plates

        # Use old logic of 'state:<state> plate:<plate>'
        potential_vehicles: List[Vehicle] = []
        legacy_plate_data: Tuple = dict([[
            piece.strip() for piece in match.split(':')
        ] for match in [
            part.lower() for part in list_of_strings
            if (('state:' in part.lower() or 'plate:' in part.lower()
                 or 'types:' in part.lower()) and '://' not in part.lower())
        ]])

        if legacy_plate_data:
            if self._detect_state(legacy_plate_data.get(
                    'state')) and legacy_plate_data.get('plate'):
                legacy_plate_data['valid_plate'] = True
            else:
                legacy_plate_data['valid_plate'] = False

            vehicle: Vehicle = Vehicle(
                plate=legacy_plate_data.get('plate'),
                plate_types=legacy_plate_data.get('types'),
                state=legacy_plate_data.get('state'),
                valid_plate=legacy_plate_data['valid_plate'])

            potential_vehicles.append(vehicle)

        return potential_vehicles

    def _form_campaign_lookup_response_parts(
            self, campaign_summaries: List[Tuple[str, int,
                                                 int]], username: str):

        campaign_chunks: List[str] = []
        campaign_string = ""

        for campaign in campaign_summaries:
            campaign_name = campaign[0]
            campaign_vehicles = campaign[1]
            campaign_tickets = campaign[2]

            next_string_part = (
                f"{'{:,}'.format(campaign_vehicles)} {'vehicle with' if campaign_vehicles == 1 else 'vehicles with a total of'} "
                f"{'{:,}'.format(campaign_tickets)} ticket{L10N.pluralize(campaign_tickets)} {'has' if campaign_vehicles == 1 else 'have'} "
                f"been tagged with {campaign_name}.\n\n")

            # how long would it be
            potential_response_length = len(username + ' ' + campaign_string +
                                            next_string_part)

            if (potential_response_length <=
                    twitter_constants.MAX_TWITTER_STATUS_LENGTH):
                campaign_string += next_string_part
            else:
                campaign_chunks.append(campaign_string)
                campaign_string = next_string_part

        # Get any part of string left over
        campaign_chunks.append(campaign_string)

        return campaign_chunks

    def _form_plate_lookup_response_parts(
            self,
            borough_data: List[Tuple[str, int]],
            frequency: int,
            fine_data: FineData,
            plate: str,
            plate_types: List[str],
            state: str,
            unique_identifier: str,
            username: str,
            violations: List[Tuple[str, int]],
            year_data: List[Tuple[str, int]],
            camera_streak_data: Optional[CameraStreakData] = None,
            previous_lookup: Optional[PlateLookup] = None):

        # response_chunks holds tweet-length-sized parts of the response
        # to be tweeted out or appended into a single direct message.
        response_chunks: List[str] = []
        violations_string: str = ""

        # Get total violations
        total_violations: int = sum([s['count'] for s in violations])
        LOG.debug(f'total_violations: {total_violations}')

        # Append to initially blank string to build tweet.
        violations_string += L10N.LOOKUP_SUMMARY_STRING.format(
            L10N.VEHICLE_HASHTAG.format(state, plate),
            L10N.get_plate_types_string(plate_types), frequency,
            L10N.pluralize(int(frequency)))

        # If this vehicle has been queried before...
        if previous_lookup:
            previous_num_violations: int = previous_lookup.num_tickets

            violations_string += self._create_repeat_lookup_string(
                new_violations=(total_violations - previous_num_violations),
                plate=plate,
                previous_lookup=previous_lookup,
                state=state,
                username=username)

        response_chunks += self._handle_response_part_formation(
            collection=violations,
            continued_format_string=L10N.LOOKUP_TICKETS_STRING_CONTD.format(
                L10N.VEHICLE_HASHTAG.format(state, plate)),
            count='count',
            cur_string=violations_string,
            description='title',
            default_description='No Year Available',
            prefix_format_string=L10N.LOOKUP_TICKETS_STRING.format(
                total_violations),
            result_format_string=L10N.LOOKUP_RESULTS_DETAIL_STRING,
            username=username)

        if year_data:
            response_chunks += self._handle_response_part_formation(
                collection=year_data,
                continued_format_string=L10N.LOOKUP_YEAR_STRING_CONTD.format(
                    L10N.VEHICLE_HASHTAG.format(state, plate)),
                count='count',
                description='title',
                default_description='No Year Available',
                prefix_format_string=L10N.LOOKUP_YEAR_STRING.format(
                    L10N.VEHICLE_HASHTAG.format(state, plate)),
                result_format_string=L10N.LOOKUP_RESULTS_DETAIL_STRING,
                username=username)

        if borough_data:
            response_chunks += self._handle_response_part_formation(
                collection=borough_data,
                continued_format_string=L10N.LOOKUP_BOROUGH_STRING_CONTD.
                format(L10N.VEHICLE_HASHTAG.format(state, plate)),
                count='count',
                description='title',
                default_description='No Borough Available',
                prefix_format_string=L10N.LOOKUP_BOROUGH_STRING.format(
                    L10N.VEHICLE_HASHTAG.format(state, plate)),
                result_format_string=L10N.LOOKUP_RESULTS_DETAIL_STRING,
                username=username)

        if fine_data and fine_data.fines_assessed():

            cur_string = f"Known fines for {L10N.VEHICLE_HASHTAG.format(state, plate)}:\n\n"

            max_count_length = len('${:,.2f}'.format(fine_data.max_amount()))
            spaces_needed = (max_count_length * 2) + 1

            for fine_type, amount in fine_data:

                currency_string = '${:,.2f}'.format(amount)
                count_length = len(str(currency_string))

                # e.g., if spaces_needed is 5, and count_length is 2, we need
                # to pad to 3.
                left_justify_amount = spaces_needed - count_length

                # formulate next string part
                next_part = (f"{currency_string.ljust(left_justify_amount)}| "
                             f"{fine_type.replace('_', ' ').title()}\n")

                # determine current string length if necessary
                potential_response_length = len(username + ' ' + cur_string +
                                                next_part)

                # If username, space, violation string so far and new part are less or
                # equal than 280 characters, append to existing tweet string.
                if (potential_response_length <=
                        twitter_constants.MAX_TWITTER_STATUS_LENGTH):
                    cur_string += next_part
                else:
                    response_chunks.append(cur_string)

                    cur_string = "Known fines for #{}_{}, cont'd:\n\n"
                    cur_string += next_part

            # add to container
            response_chunks.append(cur_string)

        if camera_streak_data:

            if camera_streak_data.max_streak and camera_streak_data.max_streak >= 5:

                # formulate streak string
                streak_string = (
                    f"Under @bradlander's proposed legislation, "
                    f"this vehicle could have been booted or impounded "
                    f"due to its {camera_streak_data.max_streak} camera violations "
                    f"(>= 5/year) from {camera_streak_data.min_streak_date}"
                    f" to {camera_streak_data.max_streak_date}.\n")

                # add to container
                response_chunks.append(streak_string)

        unique_link: str = self._get_website_plate_lookup_link(
            unique_identifier)

        website_link_string = f'View more details at {unique_link}.'
        response_chunks.append(website_link_string)

        # Send it back!
        return response_chunks

    def _form_summary_string(
            self,
            summary: TrafficViolationsAggregatorResponse) -> Optional[str]:

        num_vehicles = len(summary.plate_lookups)
        vehicle_tickets = [
            len(lookup.violations) for lookup in summary.plate_lookups
        ]
        total_tickets = sum(vehicle_tickets)

        fines_by_vehicle: List[FineData] = [
            lookup.fines for lookup in summary.plate_lookups
        ]

        vehicles_with_fines: int = len([
            fine_data for fine_data in fines_by_vehicle if fine_data.fined > 0
        ])

        aggregate_fines: FineData = FineData(
            **{
                field: sum(
                    getattr(lookup.fines, field)
                    for lookup in summary.plate_lookups)
                for field in FineData.FINE_FIELDS
            })

        if aggregate_fines.fined > 0:
            return [
                f"You queried {num_vehicles} vehicles, of which "
                f"{vehicles_with_fines} vehicle{L10N.pluralize(vehicles_with_fines)} "
                f"{'has' if vehicles_with_fines == 1 else 'have collectively'} received {total_tickets} ticket{L10N.pluralize(total_tickets)} "
                f"with at least {'${:,.2f}'.format(aggregate_fines.fined - aggregate_fines.reduced)} "
                f"in fines, of which {'${:,.2f}'.format(aggregate_fines.paid)} has been paid.\n\n"
            ]

    def _generate_unique_identifier(self):
        return ''.join(random.SystemRandom().choice(string.ascii_lowercase +
                                                    string.digits)
                       for _ in range(self.UNIQUE_IDENTIFIER_STRING_LENGTH))

    def _get_unique_identifier(self):
        unique_identifier = self._generate_unique_identifier()
        while PlateLookup.query.filter(
                PlateLookup.unique_identifier == unique_identifier).all():
            unique_identifier = self._generate_unique_identifier()
        return unique_identifier

    def _get_plate_query(self, request_object: Type[BaseLookupRequest],
                         vehicle: Vehicle) -> PlateQuery:
        """Transform a request object into plate query"""

        created_at: str = datetime.strptime(
            request_object.created_at,
            twitter_constants.TWITTER_TIME_FORMAT).strftime(
                self.MYSQL_TIME_FORMAT)

        message_id: Optional[str] = request_object.external_id()
        message_source: str = request_object.message_source

        plate: str = regexp_constants.PLATE_PATTERN.sub(
            '',
            vehicle.plate.strip().upper())

        plate_types: Optional[str] = None
        if vehicle.plate_types is not None:
            plate_types = ','.join(
                sorted([
                    type.strip()
                    for type in vehicle.plate_types.upper().split(',')
                ]))

        state: str = vehicle.state.upper()

        username: Optional[str] = request_object.username()

        plate_query: PlateQuery = PlateQuery(created_at=created_at,
                                             message_id=message_id,
                                             message_source=message_source,
                                             plate=plate,
                                             plate_types=plate_types,
                                             state=state,
                                             username=username)

        LOG.debug(f'plate_query: {plate_query}')

        return plate_query

    def _get_website_plate_lookup_link(self, unique_identifier: str) -> str:
        return f'{endpoints.HOWS_MY_DRIVING_NY_WEBSITE}/{unique_identifier}'

    def _handle_response_part_formation(self,
                                        count: str,
                                        collection: Dict[str, Any],
                                        continued_format_string: str,
                                        description: str,
                                        default_description: str,
                                        prefix_format_string: str,
                                        result_format_string: str,
                                        username: str,
                                        cur_string: str = None):

        # collect the responses
        response_container = []

        cur_string = cur_string if cur_string else ''

        if prefix_format_string:
            cur_string += prefix_format_string

        max_count_length = len(str(max(item[count] for item in collection)))
        spaces_needed = (max_count_length * 2) + 1

        # Grab item
        for item in collection:

            # Titleize for readability.
            violation_description = item[description].title()

            # Use a default description if need be
            if len(violation_description) == 0:
                violation_description = default_description

            violation_count = item[count]
            count_length = len(str(violation_count))

            # e.g., if spaces_needed is 5, and count_length is 2, we need to
            # pad to 3.
            left_justify_amount = spaces_needed - count_length

            # formulate next string part
            next_part = result_format_string.format(
                str(violation_count).ljust(left_justify_amount),
                violation_description)

            # determine current string length
            potential_response_length = len(
                f'{username} {cur_string}{next_part}')

            # If username, space, violation string so far and new part are less or
            # equal than 280 characters, append to existing tweet string.
            if (potential_response_length <=
                    twitter_constants.MAX_TWITTER_STATUS_LENGTH):
                cur_string += next_part
            else:
                response_container.append(cur_string)
                if continued_format_string:
                    cur_string = continued_format_string
                else:
                    cur_string = ''

                cur_string += next_part

        # If we finish the list with a non-empty string,
        # append that string to response parts
        if len(cur_string) != 0:
            # Append ready string into parts for response.
            response_container.append(cur_string)

        # Return parts
        return response_container

    def _infer_plate_and_state_data(
        self, list_of_vehicle_tuples: Union[Tuple[str, str, str], Tuple[str,
                                                                        str]]
    ) -> List[Vehicle]:

        potential_vehicles: List[Vehicle] = []

        for vehicle_tuple in list_of_vehicle_tuples:

            original_string = ':'.join(vehicle_tuple)
            plate = None
            plate_types = None
            state = None
            valid_plate = False

            if len(vehicle_tuple) in range(2, 4):
                state_bools: List[bool] = [
                    self._detect_state(part) for part in vehicle_tuple
                ]
                try:
                    state_index = state_bools.index(True)
                except ValueError:
                    state_index = None

                plate_types_bools: List[bool] = [
                    self._detect_plate_types(part) for part in vehicle_tuple
                ]
                try:
                    plate_types_index = plate_types_bools.index(True)
                except ValueError:
                    plate_types_index = None

                have_valid_plate: bool = (
                    len(vehicle_tuple) == 2 and state_index is not None) or (
                        len(vehicle_tuple) == 3
                        and None not in [plate_types_index, state_index])

                if have_valid_plate:
                    non_state_plate_types_parts = [
                        x for x in list(range(0, len(vehicle_tuple)))
                        if x not in [plate_types_index, state_index]
                    ]

                    plate_index = None

                    # We have a tuple with state and plate, and possibly plate
                    # types
                    if non_state_plate_types_parts:

                        plate_index = non_state_plate_types_parts[0]

                    # We don't seem to have a plate, which means the plate
                    # types might be the plate
                    elif plate_types_index is not None:

                        alphanumeric_only = re.match(
                            '^[\w-]+$',
                            vehicle_tuple[plate_types_index]) is not None

                        if alphanumeric_only:
                            plate_index = plate_types_index
                            plate_types_index = None

                    # Put plate data together
                    if plate_index is not None and vehicle_tuple[
                            plate_index] != '':
                        plate = vehicle_tuple[plate_index]
                        state = vehicle_tuple[state_index]

                        if plate_types_index is not None:
                            plate_types = vehicle_tuple[plate_types_index]

                        valid_plate = True

            vehicle: Vehicle = Vehicle(original_string=original_string,
                                       plate=plate,
                                       plate_types=plate_types,
                                       state=state,
                                       valid_plate=valid_plate)

            potential_vehicles.append(vehicle)

        return potential_vehicles

    def _perform_campaign_lookup(
            self,
            included_campaigns: List[Campaign]) -> List[Tuple[str, int, int]]:

        LOG.debug('Performing lookup for campaigns.')

        result: List[Tuple[str, int, int]] = []

        for campaign in included_campaigns:

            subquery = campaign.plate_lookups.session.query(
                PlateLookup.plate,
                PlateLookup.state,
                func.max(PlateLookup.created_at).label(
                    'most_recent_campaign_lookup'),
            ).group_by(PlateLookup.plate, PlateLookup.state).filter(
                and_(PlateLookup.campaigns.any(Campaign.id.in_([campaign.id])),
                     PlateLookup.count_towards_frequency == True)).subquery(
                         'subquery')

            full_query = PlateLookup.query.join(
                subquery, (PlateLookup.plate == subquery.c.plate) &
                (PlateLookup.state == subquery.c.state) &
                (PlateLookup.created_at
                 == subquery.c.most_recent_campaign_lookup)).order_by(
                     subquery.c.most_recent_campaign_lookup.desc(),
                     PlateLookup.created_at.desc())

            campaign_lookups = full_query.all()

            campaign_vehicles: int = len(campaign_lookups)
            campaign_tickets: int = sum(
                [lookup.num_tickets for lookup in campaign_lookups])

            result.append(
                (campaign.hashtag, campaign_vehicles, campaign_tickets))

        return result

    def _perform_plate_lookup(
            self, campaigns: List[Campaign], plate_query: PlateQuery,
            unique_identifier: str) -> OpenDataServiceResponse:

        LOG.debug('Performing lookup for plate.')

        nyc_open_data_service: OpenDataService = OpenDataService()
        open_data_response: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
            plate_query=plate_query)

        LOG.debug(f'Violation data: {open_data_response}')

        if open_data_response.success:

            open_data_plate_lookup: OpenDataServicePlateLookup = open_data_response.data

            bus_lane_camera_violations = 0
            red_light_camera_violations = 0
            speed_camera_violations = 0

            for violation_type_summary in open_data_plate_lookup.violations:
                if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
                    violation_count = violation_type_summary['count']

                    if violation_type_summary['title'] == 'Bus Lane Violation':
                        bus_lane_camera_violations = violation_count
                    if violation_type_summary[
                            'title'] == 'Failure To Stop At Red Light':
                        red_light_camera_violations = violation_count
                    elif violation_type_summary[
                            'title'] == 'School Zone Speed Camera Violation':
                        speed_camera_violations = violation_count

            camera_streak_data: CameraStreakData = open_data_plate_lookup.camera_streak_data

            # If this came from message, add it to the plate_lookups table.
            if plate_query.message_source and plate_query.message_id and plate_query.created_at:
                new_lookup = PlateLookup(
                    boot_eligible=camera_streak_data.max_streak >= 5
                    if camera_streak_data else False,
                    bus_lane_camera_violations=bus_lane_camera_violations,
                    created_at=plate_query.created_at,
                    message_id=plate_query.message_id,
                    message_source=plate_query.message_source,
                    num_tickets=open_data_plate_lookup.num_violations,
                    plate=plate_query.plate,
                    plate_types=plate_query.plate_types,
                    red_light_camera_violations=red_light_camera_violations,
                    speed_camera_violations=speed_camera_violations,
                    state=plate_query.state,
                    unique_identifier=unique_identifier,
                    username=plate_query.username)

                # Iterate through included campaigns to tie lookup to each
                for campaign in campaigns:
                    # insert join record for campaign lookup
                    new_lookup.campaigns.append(campaign)

                # Insert plate lookup
                PlateLookup.query.session.add(new_lookup)
                PlateLookup.query.session.commit()

        else:
            LOG.info(f'open data plate lookup failed')

        return open_data_response

    def _proceess_lookup_results(self, plate_query: PlateLookup):
        pass

    def _query_for_lookup_frequency(self, plate_query: PlateQuery) -> int:
        return len(
            PlateLookup.get_all_by(plate=plate_query.plate,
                                   plate_types=plate_query.plate_types,
                                   state=plate_query.state,
                                   count_towards_frequency=True))

    def _query_for_previous_lookup(
            self, plate_query: PlateQuery) -> Optional[PlateLookup]:
        """ See if we've seen this vehicle before. """

        lookups_for_vehicle: List[PlateLookup] = PlateLookup.get_all_by(
            plate=plate_query.plate,
            state=plate_query.state,
            plate_types=plate_query.plate_types,
            count_towards_frequency=True)

        if lookups_for_vehicle:
            lookups_for_vehicle.sort(key=lambda x: x.created_at, reverse=True)

            return lookups_for_vehicle[0]

        else:
            return None
コード例 #4
0
    def perform(self, *args, **kwargs):
        is_dry_run: bool = kwargs.get('is_dry_run') or False
        use_dvaa_thresholds: bool = kwargs.get('use_dvaa_thresholds') or False
        use_only_visible_tweets: bool = kwargs.get('use_only_visible_tweets') or False

        tweet_detection_service = TweetDetectionService()
        tweeter = TrafficViolationsTweeter()

        eastern = pytz.timezone('US/Eastern')
        utc = pytz.timezone('UTC')

        now = datetime.now()

        # If today is leap day, there were no lookups a year ago today
        if (now.day == self.LEAP_DAY_DATE and
            now.month == self.LEAP_DAY_MONTH):
            return

        # If today is March 1, and last year was a leap year, show lookups
        # from the previous February 29.
        one_year_after_leap_day = True if (
            now.day == self.POST_LEAP_DAY_DATE and
            now.month == self.POST_LEAP_DAY_MONTH and
            self._is_leap_year(now.year - 1)) else False

        top_of_the_hour_last_year = now.replace(
            microsecond=0,
            minute=0,
            second=0) - relativedelta(years=1)

        top_of_the_next_hour_last_year = (
            top_of_the_hour_last_year + relativedelta(hours=1))

        if use_dvaa_thresholds:
            threshold_attribute: str = 'boot_eligible_under_dvaa_threshold'
        else:
            threshold_attribute: str = 'boot_eligible_under_rdaa_threshold'

        base_query: list[Tuple[int]] = PlateLookup.query.session.query(
            func.max(PlateLookup.id).label('most_recent_vehicle_lookup')
        ).filter(
            or_(
                  and_(PlateLookup.created_at >= top_of_the_hour_last_year,
                       PlateLookup.created_at < top_of_the_next_hour_last_year,
                       getattr(PlateLookup, threshold_attribute) == True,
                       PlateLookup.count_towards_frequency == True),
                  and_(one_year_after_leap_day,
                       PlateLookup.created_at >= (top_of_the_hour_last_year - relativedelta(days=1)),
                       PlateLookup.created_at < (top_of_the_next_hour_last_year - relativedelta(days=1)),
                       getattr(PlateLookup, threshold_attribute) == True,
                       PlateLookup.count_towards_frequency == True)
                )
        )

        if use_only_visible_tweets:
            base_query = base_query.filter(
                PlateLookup.message_source == lookup_sources.LookupSource.STATUS.value)

        recent_plate_lookup_ids = base_query.group_by(
            PlateLookup.plate,
            PlateLookup.state
        ).all()

        lookup_ids_to_update: list[int] = [id[0] for id in recent_plate_lookup_ids]

        lookups_to_update: list[PlateLookup] = PlateLookup.get_all_in(
            id=lookup_ids_to_update)

        if not lookups_to_update:
            LOG.debug(f'No vehicles for which to perform retrospective job '
                      f'between {top_of_the_hour_last_year} and '
                      f'and {top_of_the_next_hour_last_year}.')

        for previous_lookup in lookups_to_update:

            LOG.debug(f'Performing retrospective job for '
                      f'{L10N.VEHICLE_HASHTAG.format(previous_lookup.state, previous_lookup.plate)} ')

            plate_query: PlateQuery = PlateQuery(created_at=now,
                                                 message_source=previous_lookup.message_source,
                                                 plate=previous_lookup.plate,
                                                 plate_types=previous_lookup.plate_types,
                                                 state=previous_lookup.state)

            nyc_open_data_service: OpenDataService = OpenDataService()
            data_before_query: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
                plate_query=plate_query,
                until=previous_lookup.created_at)

            lookup_before_query: OpenDataServicePlateLookup = data_before_query.data
            camera_streak_data_before_query: CameraStreakData = lookup_before_query.camera_streak_data['Mixed']

            data_after_query: OpenDataServiceResponse = nyc_open_data_service.look_up_vehicle(
                plate_query=plate_query,
                since=previous_lookup.created_at,
                until=previous_lookup.created_at + relativedelta(years=1))


            lookup_after_query: OpenDataServicePlateLookup = data_after_query.data

            new_bus_lane_camera_violations: Optional[int] = None
            new_speed_camera_violations: Optional[int] = None
            new_red_light_camera_violations: Optional[int] = None

            for violation_type_summary in lookup_after_query.violations:
                if violation_type_summary['title'] in self.CAMERA_VIOLATIONS:
                    violation_count = violation_type_summary['count']

                    if violation_type_summary['title'] == 'Bus Lane Violation':
                        new_bus_lane_camera_violations = violation_count
                    if violation_type_summary['title'] == 'Failure To Stop At Red Light':
                        new_red_light_camera_violations = violation_count
                    if violation_type_summary['title'] == 'School Zone Speed Camera Violation':
                        new_speed_camera_violations = violation_count

            if new_bus_lane_camera_violations is None:
                new_bus_lane_camera_violations = 0

            if new_red_light_camera_violations is None:
                new_red_light_camera_violations = 0

            if new_speed_camera_violations is None:
                new_speed_camera_violations = 0

            new_boot_eligible_violations = (new_red_light_camera_violations +
                                            new_speed_camera_violations)

            if new_boot_eligible_violations > 0:
                vehicle_hashtag = L10N.VEHICLE_HASHTAG.format(
                    previous_lookup.state, previous_lookup.plate)
                previous_lookup_created_at = utc.localize(
                    previous_lookup.created_at)
                previous_lookup_date = previous_lookup_created_at.astimezone(eastern).strftime(
                    L10N.REPEAT_LOOKUP_DATE_FORMAT)
                previous_lookup_time = previous_lookup_created_at.astimezone(eastern).strftime(
                    L10N.REPEAT_LOOKUP_TIME_FORMAT)

                red_light_camera_violations_string = (
                    f'{new_red_light_camera_violations} | Red Light Camera Violations\n'
                    if new_red_light_camera_violations > 0 else '')

                speed_camera_violations_string = (
                    f'{new_speed_camera_violations} | Speed Safety Camera Violations\n'
                    if new_speed_camera_violations > 0 else '')

                reckless_driver_summary_string = (
                    f'{vehicle_hashtag} was originally '
                    f'queried on {previous_lookup_date} '
                    f'at {previous_lookup_time}')

                # assume we can't link
                can_link_tweet = False

                # Where did this come from?
                if previous_lookup.message_source == lookup_sources.LookupSource.STATUS.value:
                    # Determine if tweet is still visible:
                    if tweet_detection_service.tweet_exists(id=previous_lookup.message_id,
                                                            username=previous_lookup.username):
                        can_link_tweet = True

                if can_link_tweet:
                    reckless_driver_summary_string += L10N.PREVIOUS_LOOKUP_STATUS_STRING.format(
                        previous_lookup.username,
                        previous_lookup.username,
                        previous_lookup.message_id)
                else:
                    reckless_driver_summary_string += '.'

                if use_only_visible_tweets and not can_link_tweet:
                    # If we're only displaying tweets we can quote tweet,
                    # skip this one since we can'tt.
                    continue

                reckless_driver_update_string = (
                    f'From {camera_streak_data_before_query.min_streak_date} to '
                    f'{camera_streak_data_before_query.max_streak_date}, this vehicle '
                    f'received {camera_streak_data_before_query.max_streak} camera '
                    f'violations. Over the past 12 months, this vehicle '
                    f'received {new_boot_eligible_violations} new camera violation'
                    f"{'' if new_boot_eligible_violations == 1 else 's'}: \n\n"
                    f'{red_light_camera_violations_string}'
                    f'{speed_camera_violations_string}')

                messages: list[str] = [
                    reckless_driver_summary_string,
                    reckless_driver_update_string]

                if not is_dry_run:
                    success: bool = tweeter.send_status(
                        message_parts=messages,
                        on_error_message=(
                            f'Error printing reckless driver update. '
                            f'Tagging @bdhowald.'))

                    if success:
                        LOG.debug('Reckless driver retrospective job '
                                  'ran successfully.')
                else:
                    print(reckless_driver_update_string)
コード例 #5
0
class TrafficViolationsTweeter:

    PRODUCTION_APP_RATE_LIMITING_INTERVAL_IN_SECONDS = 3.0

    DEVELOPMENT_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS = 3000.0
    PRODUCTION_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS = 300.0

    FOLLOWERS_RATE_LIMITING_INTERVAL_IN_SECONDS = 15 * SECONDS_PER_MINUTE

    MAX_DIRECT_MESSAGES_RETURNED = 50


    def __init__(self):

        self._app_api = None
        self._client_api = None

        # Create reply argument_builder
        self.reply_argument_builder = ReplyArgumentBuilder(self._get_twitter_application_api())

        # Create new aggregator
        self.aggregator = TrafficViolationsAggregator()

        # Need to find if unanswered statuses have been deleted
        self.tweet_detection_service = TweetDetectionService()

        # Log how many times we've called the apis
        self._direct_messages_iteration = 0
        self._events_iteration = 0
        self._statuses_iteration = 0

        self._lookup_threads = []

        # Initialize cached values to None
        self._follower_ids: Optional[list[int]] = None
        self._follower_ids_last_fetched: Optional[datetime] = None


    def find_and_respond_to_requests(self) -> None:
        """Convenience method to collect the different ways TwitterEvent
        objects are created, found, and responded to and begin the process
        of calling these methods at process start.
        """
        self._find_and_respond_to_missed_direct_messages()
        self._find_and_respond_to_missed_statuses()
        self._find_and_respond_to_twitter_events()

    def send_status(self,
                    message_parts: Union[list[any], list[str]],
                    on_error_message: str) -> bool:
        """Send statuses from @HowsMyDrivingNY"""
        try:
            self._recursively_process_status_updates(message_parts)

            return True
        except Exception as e:
            LOG.error(e)
            self._recursively_process_status_updates(on_error_message)

            return False

    def terminate_lookups(self) -> None:
        """Stop looking for twitter events, statuses, or direct messages to respond to."""
        for thread in self._lookup_threads:
            thread.cancel()

    def _add_twitter_events_for_missed_direct_messages(self, messages: list[tweepy.models.Status]) -> None:
        """Creates TwitterEvent objects when the Account Activity API fails to send us
        direct message events via webhooks to have them created by the HowsMyDrivingNY API.

        :param messages: list[tweepy.DirectMessage]: The direct messages returned via the
                                                     Twitter Search API via Tweepy.
        """

        undetected_messages = 0

        sender_ids = set(int(message.message_create['sender_id']) for message in messages)

        sender_objects = self._get_twitter_client_api().lookup_users(user_id=sender_ids)
        senders = {sender.id_str:sender for sender in sender_objects}

        for message in messages:

            existing_event: Optional[TwitterEvent] = TwitterEvent.query.filter(TwitterEvent.event_id == message.id).first()

            if not existing_event and int(message.message_create['sender_id']) != HMDNY_TWITTER_USER_ID:
                undetected_messages += 1

                sender = senders[message.message_create['sender_id']]

                event: TwitterEvent = TwitterEvent(
                    event_type=TwitterMessageType.DIRECT_MESSAGE.value,
                    event_id=message.id,
                    user_handle=sender.screen_name,
                    user_id=sender.id,
                    event_text=message.message_create['message_data']['text'],
                    created_at=message.created_timestamp,
                    in_reply_to_message_id=None,
                    location=None,
                    user_mention_ids=','.join([user['id_str'] for user in message.message_create['message_data']['entities']['user_mentions']]),
                    user_mentions=' '.join([user['screen_name'] for user in message.message_create['message_data']['entities']['user_mentions']]),
                    detected_via_account_activity_api=False)

                TwitterEvent.query.session.add(event)

        TwitterEvent.query.session.commit()

        LOG.debug(
            f"Found {undetected_messages} direct message{'' if undetected_messages == 1 else 's'} that "
            f"{'was' if undetected_messages == 1 else 'were'} previously undetected.")


    def _add_twitter_events_for_missed_statuses(self, messages: list[tweepy.models.Status]) -> None:
        """Creates TwitterEvent objects when the Account Activity API fails to send us
        status events via webhooks to have them created by the HowsMyDrivingNY API.

        :param messages: list[tweepy.models.Status]: The statuses returned via the Twitter
                                                     Search API via Tweepy.
        """

        undetected_messages = 0

        for message in messages:
            existing_event: Optional[TwitterEvent] = TwitterEvent.query.filter(TwitterEvent.event_id == message.id).first()

            if not existing_event and message.user.id != HMDNY_TWITTER_USER_ID:
                undetected_messages += 1

                event: TwitterEvent = TwitterEvent(
                    event_type=TwitterMessageType.STATUS.value,
                    event_id=message.id,
                    user_handle=message.user.screen_name,
                    user_id=message.user.id,
                    event_text=message.full_text,
                    created_at=message.created_at.replace(tzinfo=pytz.timezone('UTC')).timestamp() * MILLISECONDS_PER_SECOND,
                    in_reply_to_message_id=message.in_reply_to_status_id,
                    location=message.place and message.place.full_name,
                    user_mention_ids=','.join([user['id_str'] for user in message.entities['user_mentions']]),
                    user_mentions=' '.join([user['screen_name'] for user in message.entities['user_mentions']]),
                    detected_via_account_activity_api=False)

                TwitterEvent.query.session.add(event)

        TwitterEvent.query.session.commit()

        LOG.debug(
            f"Found {undetected_messages} status{'' if undetected_messages == 1 else 'es'} that "
            f"{'was' if undetected_messages == 1 else 'were'} previously undetected.")

    def _filter_failed_twitter_events(self, failed_events: list[TwitterEvent]) -> list[TwitterEvent]:
        failed_events_that_need_response: list[TwitterEvent] = []

        for failed_event in failed_events:
            # If event is a tweet, but can no longer be found, there's nothing we can do.
            if (failed_event.event_type == TwitterMessageType.STATUS.value and
                not self.tweet_detection_service.tweet_exists(id=failed_event.event_id,
                                                          username=failed_event.user_handle)):

                failed_event.error_on_lookup = False
                failed_event.num_times_failed = 0
                failed_event.last_failed_at_time = None

                failed_event.query.session.commit()

                continue

            if failed_event.num_times_failed == 0:
                failed_events_that_need_response.append(failed_event)

            elif failed_event.num_times_failed == 1:
                time_to_retry = failed_event.last_failed_at_time + timedelta(minutes=5)
                if time_to_retry <= datetime.utcnow():
                    failed_events_that_need_response.append(failed_event)

            elif failed_event.num_times_failed == 2:
                time_to_retry = failed_event.last_failed_at_time + timedelta(hours=1)
                if time_to_retry <= datetime.utcnow():
                    failed_events_that_need_response.append(failed_event)

            elif failed_event.num_times_failed == 3:
                time_to_retry = failed_event.last_failed_at_time + timedelta(hours=3)
                if time_to_retry <= datetime.utcnow():
                    failed_events_that_need_response.append(failed_event)

            elif failed_event.num_times_failed == 4:
                time_to_retry = failed_event.last_failed_at_time + timedelta(days=1)
                if time_to_retry <= datetime.utcnow():
                    failed_events_that_need_response.append(failed_event)

            else:
                LOG.debug(f'Event response cannot be retried automatically.')


        LOG.debug(f'failed events to retry: {failed_events_that_need_response}')

        return failed_events_that_need_response


    def _find_and_respond_to_missed_direct_messages(self) -> None:
        """Uses Tweepy to call the Twitter Search API to find direct messages to/from
        HowsMyDrivingNY. It then passes this data to a function that creates
        TwitterEvent objects when those direct message events have not already been recorded.
        """

        interval = (self.PRODUCTION_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS if self._is_production()
            else self.DEVELOPMENT_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS)

        self._direct_messages_iteration += 1
        LOG.debug(
            f'Looking up missed direct messages on iteration {self._direct_messages_iteration}')

        # set up timer
        direct_message_thread = threading.Timer(
            interval, self._find_and_respond_to_missed_direct_messages)
        self._lookup_threads.append(direct_message_thread)

        # start timer
        direct_message_thread.start()

        try:
            # most_recent_undetected_twitter_event = TwitterEvent.query.filter(
            #     and_(TwitterEvent.detected_via_account_activity_api == False,
            #          TwitterEvent.event_type == TwitterMessageType.DIRECT_MESSAGE.value)
            # ).order_by(TwitterEvent.event_id.desc()).first()

            # Tweepy bug with cursors prevents us from searching for more than 50 events
            # at a time until 3.9, so it'll have to do.

            direct_messages_since_last_twitter_event = self._get_twitter_client_api(
                ).get_direct_messages(
                    count=self.MAX_DIRECT_MESSAGES_RETURNED)

            received_messages = [message for message in direct_messages_since_last_twitter_event if
                int(message.message_create['sender_id']) != HMDNY_TWITTER_USER_ID]

            self._add_twitter_events_for_missed_direct_messages(messages=received_messages)

        except Exception as e:
            LOG.error(e)
            LOG.error(str(e))
            LOG.error(e.args)
            logging.exception("stack trace")

        finally:
            TwitterEvent.query.session.close()

    def _find_and_respond_to_missed_statuses(self) -> None:
        """Uses Tweepy to call the Twitter Search API to find statuses mentioning
        HowsMyDrivingNY. It then passes this data to a function that creates
        TwitterEvent objects when those status events have not already been recorded.
        """

        interval = (self.PRODUCTION_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS if self._is_production()
            else self.DEVELOPMENT_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS)

        self._statuses_iteration += 1
        LOG.debug(
            f'Looking up missed statuses on iteration {self._statuses_iteration}')

        # set up timer
        statuses_thread = threading.Timer(
            interval, self._find_and_respond_to_missed_statuses)
        self._lookup_threads.append(statuses_thread)

        # start timer
        statuses_thread.start()

        try:
            # Find most recent undetected twitter status event, and then
            # search for recent events until we can find no more.
            most_recent_undetected_twitter_event = TwitterEvent.query.filter(
                and_(TwitterEvent.detected_via_account_activity_api == False,
                     TwitterEvent.event_type == TwitterMessageType.STATUS.value)
            ).order_by(TwitterEvent.event_id.desc()).first()

            if most_recent_undetected_twitter_event:

                statuses_since_last_twitter_event: list[tweepy.models.Status] = []
                max_status_id: Optional[int] = None

                while max_status_id is None or statuses_since_last_twitter_event:
                    statuses_since_last_twitter_event = self._get_twitter_client_api(
                        ).mentions_timeline(
                            max_id=max_status_id,
                            since_id=most_recent_undetected_twitter_event.event_id,
                            tweet_mode='extended')

                    if statuses_since_last_twitter_event:
                        self._add_twitter_events_for_missed_statuses(statuses_since_last_twitter_event)
                        max_status_id = statuses_since_last_twitter_event[-1].id - 1
                    else:
                        max_status_id = most_recent_undetected_twitter_event.event_id - 1

        except Exception as e:
            LOG.error(e)
            LOG.error(str(e))
            LOG.error(e.args)
            logging.exception("stack trace")

        finally:
            TwitterEvent.query.session.close()

    def _find_and_respond_to_twitter_events(self) -> None:
        """Looks for TwitterEvent objects that have not yet been responded to and
        begins the process of creating a response. Additionally, failed events are
        rerun to provide a correct response, particularly useful in cases where
        external apis are down for maintenance.
        """

        interval = (
            self.PRODUCTION_APP_RATE_LIMITING_INTERVAL_IN_SECONDS if self._is_production()
            else self.DEVELOPMENT_CLIENT_RATE_LIMITING_INTERVAL_IN_SECONDS)

        self._events_iteration += 1
        LOG.debug(
            f'Looking up twitter events on iteration {self._events_iteration}')

        # set up timer
        twitter_events_thread = threading.Timer(
            interval, self._find_and_respond_to_twitter_events)
        self._lookup_threads.append(twitter_events_thread)

        # start timer
        twitter_events_thread.start()

        try:
            new_events: list[TwitterEvent] = TwitterEvent.get_all_by(
                is_duplicate=False,
                responded_to=False,
                response_in_progress=False)

            LOG.debug(f'new events: {new_events}')

            failed_events: list[TwitterEvent] = TwitterEvent.get_all_by(
                is_duplicate=False,
                error_on_lookup=True,
                responded_to=True,
                response_in_progress=False)

            failed_events_that_need_response: list[TwitterEvent] = self._filter_failed_twitter_events(failed_events)

            events_to_respond_to: [list[TwitterEvent]] = new_events + failed_events_that_need_response

            LOG.debug(f'events to respond to: {events_to_respond_to}')

            for event in events_to_respond_to:
                self._process_twitter_event(event=event)

        except Exception as e:

            LOG.error(e)
            LOG.error(str(e))
            LOG.error(e.args)
            logging.exception("stack trace")

        finally:
            TwitterEvent.query.session.close()

    def _get_twitter_application_api(self) -> twitter_api_wrapper.TwitterApplicationApiWrapper:
        """Set the application (non-client) api connection for this instance"""

        if not self._app_api:
            self._app_api = twitter_api_wrapper.TwitterApplicationApiWrapper().get_connection()

        return self._app_api

    def _get_twitter_client_api(self) -> twitter_api_wrapper.TwitterClientApiWrapper:
        """Set the client api connection for this instance"""

        if not self._client_api:
            self._client_api = twitter_api_wrapper.TwitterClientApiWrapper().get_connection()

        return self._client_api


    def _get_follower_ids(self):
        """Get list of followers from Twitter every 15 minutes.

        This is used to determine who should be prompted to like
        the reply tweet in order to trigger a response.

        If the cached value is older than 15 minutes, refetch.
        """
        now = datetime.utcnow()

        # If cache is empty or stale, refetch.
        if not self._follower_ids_last_fetched or (
            ((now - self._follower_ids_last_fetched).seconds
                > self.FOLLOWERS_RATE_LIMITING_INTERVAL_IN_SECONDS)):

            follower_ids = []
            next_cursor: int = -1

            while next_cursor:
                results, cursors = self._get_twitter_application_api().get_follower_ids(cursor=next_cursor)
                next_cursor = cursors[1]
                follower_ids += results

            # set follower_ids
            self._follower_ids = follower_ids

            # cache current time
            self._follower_ids_last_fetched = now

        # Return cached or fetched value.
        return self._follower_ids

    def _is_production(self):
        """Determines if we are running in production to see if we can create
        direct message and status responses.

        TODO: Come up with a better way to determine production environment.
        """
        return os.getenv('ENV') == 'production'

    def _process_response(self,
        request_object: Type[BaseLookupRequest],
        response_parts: list[Any],
        successful_lookup: bool = False) -> Optional[int]:

        """Directs the response to a Twitter message, depending on whether
        or not the event is a direct message or a status. Statuses that mention
        HowsMyDrivingNY earn a favorite/like.
        """

        message_source = request_object.message_source if request_object else None
        message_id = request_object.external_id() if request_object else None

        # Respond to user
        if message_source == LookupSource.DIRECT_MESSAGE.value:

            LOG.debug('responding as direct message')

            combined_message = self._recursively_compile_direct_messages(
                response_parts)

            LOG.debug(f'combined_message: {combined_message}')

            return self._send_direct_message(message=combined_message,
                                             recipient_id=request_object.user_id)

        elif message_source == LookupSource.STATUS.value:
            # If we have at least one successful lookup, favorite the status
            if successful_lookup:

                # Favorite every look-up from a status
                try:
                    self._is_production() and self._get_twitter_application_api(
                        ).create_favorite(message_id)

                # But don't crash on error
                except tweepy.error.TweepError as te:
                    # There's no easy way to know if this status has already
                    # been favorited
                    pass

            LOG.debug('responding as status update')

            user_mention_ids = (request_object.mentioned_user_ids if
                request_object.mentioned_user_ids else None)

            return self._recursively_process_status_updates(
                response_parts=response_parts,
                message_id=message_id,
                user_mention_ids=user_mention_ids)

        else:
            LOG.error('Unkown message source. Cannot respond.')

    def _process_twitter_event(self, event: TwitterEvent):
        LOG.debug(f'Beginning response for event: {event.id}')

        # search for duplicates
        is_event_duplicate: bool = TwitterEvent.query.filter_by(
            event_type=event.event_type,
            event_id=event.event_id,
            user_handle=event.user_handle,
            responded_to=True
        ).filter(
            TwitterEvent.id != event.id).count() > 0

        if is_event_duplicate:
            event.is_duplicate = True
            TwitterEvent.query.session.commit()

            LOG.info(f'Event {event.id} is a duplicate, skipping.')

        else:

            event.response_in_progress = True
            TwitterEvent.query.session.commit()

            try:
                message_source: str = LookupSource(event.event_type)

                # build request
                lookup_request: Type[BaseLookupRequest] = self.reply_argument_builder.build_reply_data(
                    message=event,
                    message_source=message_source)

                user_is_follower: bool = lookup_request.requesting_user_is_follower(
                    follower_ids=self._get_follower_ids())

                perform_lookup_for_user: bool = (user_is_follower or
                    event.user_favorited_non_follower_reply)

                if self.aggregator.lookup_has_valid_plates(
                    lookup_request=lookup_request) and not perform_lookup_for_user:

                    response_parts: list[Any]

                    if lookup_request.is_direct_message():
                        response_parts = [L10N.NON_FOLLOWER_DIRECT_MESSAGE_REPLY_STRING]
                    elif lookup_request.is_status():
                        response_parts = [L10N.NON_FOLLOWER_TWEET_REPLY_STRING]

                    try:
                        reply_message_id = self._process_response(
                            request_object=lookup_request,
                            response_parts=response_parts)

                        # Save the reply id, so that when the user favorites it,
                        # we can trigger the search.
                        non_follower_reply = NonFollowerReply(
                            created_at=(int(datetime.utcnow().timestamp() *
                                MILLISECONDS_PER_SECOND)),
                            event_type=event.event_type,
                            event_id=reply_message_id,
                            in_reply_to_message_id=event.event_id,
                            user_handle=event.user_handle,
                            user_id=event.user_id)

                        NonFollowerReply.query.session.add(
                            non_follower_reply)
                        NonFollowerReply.query.session.commit()

                    except tweepy.error.TweepError as e:
                        event.error_on_lookup = True
                        event.num_times_failed += 1
                        event.last_failed_at_time = datetime.utcnow()

                else:
                    # Reply to the event.
                    reply_to_event = self.aggregator.initiate_reply(
                        lookup_request=lookup_request)

                    success = reply_to_event['success']

                    if success:
                        # There's no need to tell people that
                        # there was an error more than once.
                        if not (reply_to_event[
                                'error_on_lookup'] and event.error_on_lookup):

                            try:
                                self._process_response(
                                    request_object=reply_to_event['request_object'],
                                    response_parts=reply_to_event['response_parts'],
                                    successful_lookup=reply_to_event.get('successful_lookup'))
                            except tweepy.error.TweepError as e:
                                reply_to_event['error_on_lookup'] = True

                    # Update error status
                    if reply_to_event['error_on_lookup']:
                        event.error_on_lookup = True
                        event.num_times_failed += 1
                        event.last_failed_at_time = datetime.utcnow()
                    else:
                        event.error_on_lookup = False
                        event.num_times_failed = 0
                        event.last_failed_at_time = None

                # We've responded!
                event.response_in_progress = False
                event.responded_to = True

                TwitterEvent.query.session.commit()

            except ValueError as e:
                LOG.error(
                    f'Encountered unknown event type. '
                    f'Response is not possible.')

    def _recursively_compile_direct_messages(self, response_parts):
        """Direct message responses from the aggregator return lists
        of chunked information (by violation type, by borough, by year, etc.).
        Data look like:

        [
          [data_type_1_part_1, data_type_1_part_2, data_type_1_part_3],
          [data_type_2_part_1, data_type_2_part_2],
          [data_type_3_part_1, data_type_3_part_2, data_type_3_part_3],
        ]

        This ensures that the data is grouped with like data. Using recursion,
        the final message is built into one large message.
        """

        return_message = []

        # Iterate through all response parts
        for part in response_parts:
            if isinstance(part, list):
                return_message.append(
                    self._recursively_compile_direct_messages(part))
            else:
                return_message.append(part)

        return '\n'.join(return_message)

    def _recursively_process_status_updates(self,
                                            response_parts: Union[list[any], list[str]],
                                            message_id: Optional[int] = None,
                                            user_mention_ids: Optional[list[str]] = None) -> Optional[int]:

        """Status responses from the aggregator return lists
        of chunked information (by violation type, by borough, by year, etc.).
        Data look like:

        [
          [data_type_1_part_1, data_type_1_part_2, data_type_1_part_3],
          [data_type_2_part_1, data_type_2_part_2],
          [data_type_3_part_1, data_type_3_part_2, data_type_3_part_3],
        ]

        This ensures that the data is grouped with like data. Using recursion,
        response statuses are created, then their message ids are saved to be
        used as the in_reply_to_status_id for the next status.
        """

        # Iterate through all response parts
        for part in response_parts:
            # Some may be lists themselves
            if isinstance(part, list):
                message_id = self._recursively_process_status_updates(
                    response_parts=part,
                    message_id=message_id,
                    user_mention_ids=user_mention_ids)
            else:
                if self._is_production():
                    new_message = self._get_twitter_application_api(
                        ).update_status(
                            status=part,
                            in_reply_to_status_id=message_id,
                            exclude_reply_user_ids=user_mention_ids)

                    message_id = new_message.id

                    LOG.debug(f'message_id: {message_id}')
                else:
                    LOG.debug(
                        "This is where 'self._get_twitter_application_api()"
                        ".update_status(status=part, in_reply_to_status_id=message_id, "
                        "exclude_reply_user_ids=user_mention_ids)' "
                        "would be called in production.")
                    return None

        return message_id

    def _send_direct_message(self, message: str, recipient_id: int) -> Optional[int]:
        """Send a direct message to a Twitter user."""

        if self._is_production():
            new_message = self._get_twitter_application_api().send_direct_message(
                recipient_id=recipient_id,
                text=message)
            return new_message.id
        else:
            LOG.debug(
                "This is where 'self._get_twitter_application_api()"
                ".send_direct_message(recipient_id=recipient_id, "
                "text=message)' would be called in production.")
            return None