def new_aggregate_event(request: Request, event_variant: GLib.Variant, dbsession: DbSession) -> Optional[AggregateEvent]: event_id = str(UUID(bytes=get_bytes(event_variant.get_child_value(1)))) if event_id in IGNORED_EVENTS: return None user_id = event_variant.get_child_value(0).get_uint32() count = event_variant.get_child_value(2).get_int64() event_relative_timestamp = event_variant.get_child_value(3).get_int64() payload = event_variant.get_child_value(4) event_date = get_event_datetime(request.absolute_timestamp, request.relative_timestamp, event_relative_timestamp) # We don't have any aggregate event yet, therefore it can only be unknown # Mypy complains here, even though this should be fine: # https://github.com/dropbox/sqlalchemy-stubs/issues/97 event = UnknownAggregateEvent( request=request, user_id=user_id, # type: ignore occured_at=event_date, count=count, event_id=event_id, payload=payload) dbsession.add(event) return event
def new_singular_event(request: Request, event_variant: GLib.Variant, dbsession: DbSession) -> Optional[SingularEvent]: event_id = str(UUID(bytes=get_bytes(event_variant.get_child_value(1)))) if event_id in IGNORED_EVENTS: return None user_id = event_variant.get_child_value(0).get_uint32() event_relative_timestamp = event_variant.get_child_value(2).get_int64() payload = event_variant.get_child_value(3) event_date = get_event_datetime(request.absolute_timestamp, request.relative_timestamp, event_relative_timestamp) try: event_model = SINGULAR_EVENT_MODELS[event_id] except KeyError: # Mypy complains here, even though this should be fine: # https://github.com/dropbox/sqlalchemy-stubs/issues/97 event = UnknownSingularEvent( request=request, user_id=user_id, # type: ignore occured_at=event_date, event_id=event_id, payload=payload) dbsession.add(event) return event try: # Mypy complains here, even though this should be fine: # https://github.com/dropbox/sqlalchemy-stubs/issues/97 event = event_model( request=request, user_id=user_id, # type: ignore occured_at=event_date, payload=payload) except Exception as e: if isinstance(e, EmptyPayloadError ) and event_id in IGNORED_EMPTY_PAYLOAD_ERRORS: return None log.exception('An error occured while processing the event:') # Mypy complains here, even though this should be fine: # https://github.com/dropbox/sqlalchemy-stubs/issues/97 event = InvalidSingularEvent( request=request, user_id=user_id, # type: ignore occured_at=event_date, event_id=event_id, payload=payload, error=str(e)) dbsession.add(event) return event
def process(dbsession: DbSession, record: bytes) -> None: log.debug('Processing ping v1 record: %s', record) ping_config_id = PingConfiguration.id_from_serialized(record, dbsession) ping = Ping.from_serialized(record) ping.config_id = ping_config_id dbsession.add(ping) log.debug('Inserting ping record:\n%s', ping)
def process(dbsession: DbSession, record: bytes) -> None: log.debug('Processing metric v2 record: %s', record) request_builder = RequestBuilder.parse_bytes(record) request = request_builder.build_request() dbsession.add(request) for event_variant in request_builder.singulars: singular_event = new_singular_event(request, event_variant, dbsession) if singular_event is not None: log.debug('Inserting singular metric:\n%s', singular_event) for event_variant in request_builder.aggregates: aggregate_event = new_aggregate_event(request, event_variant, dbsession) log.debug('Inserting aggregate metric:\n%s', aggregate_event) for event_variant in request_builder.sequences: sequence_event = new_sequence_event(request, event_variant, dbsession) if sequence_event is not None: log.debug('Inserting sequence event:\n%s', sequence_event) try: dbsession.commit() except IntegrityError as e: # FIXME: This is fragile, can we do better? if "uq_metrics_request_v2_sha512" in str(e): log.debug('Request had already been processed in the past') return # FIXME: Given how the request is built, this shouldn't ever happen; if it does though, we # absolutely need an integration test raise # pragma: no cover
def process(dbsession: DbSession, record: bytes) -> None: log.debug('Processing activation v1 record: %s', record) activation = Activation.from_serialized(record) dbsession.add(activation) log.debug('Inserting activation record:\n%s', activation)
def new_sequence_event( request: Request, sequence_variant: GLib.Variant, dbsession: DbSession ) -> Optional[Union[SequenceEvent, InvalidSequence, UnknownSequence]]: event_id = str(UUID(bytes=get_bytes(sequence_variant.get_child_value(1)))) if event_id in IGNORED_EVENTS: return None user_id = sequence_variant.get_child_value(0).get_uint32() events = sequence_variant.get_child_value(2) num_events = events.n_children() if num_events < 2: error = f'Sequence must have at least 2 elements, but only had {num_events}' # Mypy complains here, even though this should be fine: # https://github.com/dropbox/sqlalchemy-stubs/issues/97 sequence = InvalidSequence( request=request, user_id=user_id, # type: ignore event_id=event_id, payload=events, error=error) dbsession.add(sequence) return sequence start_variant, *_progress_variants, stop_variant = get_child_values(events) # For now, we ignore progress events entirely. We also assume the stop event always has a null # payload. This works for most sequence events we care about in priority. # TODO: Figure this out for the more complex events start_relative_timestamp = start_variant.get_child_value(0).get_int64() payload = start_variant.get_child_value(1) started_at = get_event_datetime(request.absolute_timestamp, request.relative_timestamp, start_relative_timestamp) stop_relative_timestamp = stop_variant.get_child_value(0).get_int64() stopped_at = get_event_datetime(request.absolute_timestamp, request.relative_timestamp, stop_relative_timestamp) try: event_model = SEQUENCE_EVENT_MODELS[event_id] except KeyError: # Mypy complains here, even though this should be fine: # https://github.com/dropbox/sqlalchemy-stubs/issues/97 sequence = UnknownSequence( request=request, user_id=user_id, # type: ignore event_id=event_id, payload=events) dbsession.add(sequence) return sequence try: # Mypy complains here, even though this should be fine: # https://github.com/dropbox/sqlalchemy-stubs/issues/97 sequence = event_model( request=request, user_id=user_id, # type: ignore started_at=started_at, stopped_at=stopped_at, payload=payload) except Exception as e: if isinstance(e, EmptyPayloadError ) and event_id in IGNORED_EMPTY_PAYLOAD_ERRORS: return None log.exception('An error occured while processing the sequence:') # Mypy complains here, even though this should be fine: # https://github.com/dropbox/sqlalchemy-stubs/issues/97 sequence = InvalidSequence( request=request, user_id=user_id, # type: ignore event_id=event_id, payload=events, error=str(e)) dbsession.add(sequence) return sequence