def new_aggregate_event(request: Request, event_variant: GLib.Variant,
dbsession: DbSession) -> Optional[AggregateEvent]:
event_id = str(UUID(bytes=get_bytes(event_variant.get_child_value(1))))
if event_id in IGNORED_EVENTS:
return None
user_id = event_variant.get_child_value(0).get_uint32()
count = event_variant.get_child_value(2).get_int64()
event_relative_timestamp = event_variant.get_child_value(3).get_int64()
payload = event_variant.get_child_value(4)
event_date = get_event_datetime(request.absolute_timestamp,
request.relative_timestamp,
event_relative_timestamp)
# We don't have any aggregate event yet, therefore it can only be unknown
# Mypy complains here, even though this should be fine:
# https://github.com/dropbox/sqlalchemy-stubs/issues/97
event = UnknownAggregateEvent(
request=request,
user_id=user_id, # type: ignore
occured_at=event_date,
count=count,
event_id=event_id,
payload=payload)
dbsession.add(event)
return event
def new_singular_event(request: Request, event_variant: GLib.Variant,
dbsession: DbSession) -> Optional[SingularEvent]:
event_id = str(UUID(bytes=get_bytes(event_variant.get_child_value(1))))
if event_id in IGNORED_EVENTS:
return None
user_id = event_variant.get_child_value(0).get_uint32()
event_relative_timestamp = event_variant.get_child_value(2).get_int64()
payload = event_variant.get_child_value(3)
event_date = get_event_datetime(request.absolute_timestamp,
request.relative_timestamp,
event_relative_timestamp)
try:
event_model = SINGULAR_EVENT_MODELS[event_id]
except KeyError:
# Mypy complains here, even though this should be fine:
# https://github.com/dropbox/sqlalchemy-stubs/issues/97
event = UnknownSingularEvent(
request=request,
user_id=user_id, # type: ignore
occured_at=event_date,
event_id=event_id,
payload=payload)
dbsession.add(event)
return event
try:
# Mypy complains here, even though this should be fine:
# https://github.com/dropbox/sqlalchemy-stubs/issues/97
event = event_model(
request=request,
user_id=user_id, # type: ignore
occured_at=event_date,
payload=payload)
except Exception as e:
if isinstance(e, EmptyPayloadError
) and event_id in IGNORED_EMPTY_PAYLOAD_ERRORS:
return None
log.exception('An error occured while processing the event:')
# Mypy complains here, even though this should be fine:
# https://github.com/dropbox/sqlalchemy-stubs/issues/97
event = InvalidSingularEvent(
request=request,
user_id=user_id, # type: ignore
occured_at=event_date,
event_id=event_id,
payload=payload,
error=str(e))
dbsession.add(event)
return event
def process(dbsession: DbSession, record: bytes) -> None:
log.debug('Processing ping v1 record: %s', record)
ping_config_id = PingConfiguration.id_from_serialized(record, dbsession)
ping = Ping.from_serialized(record)
ping.config_id = ping_config_id
dbsession.add(ping)
log.debug('Inserting ping record:\n%s', ping)
def process(dbsession: DbSession, record: bytes) -> None:
log.debug('Processing metric v2 record: %s', record)
request_builder = RequestBuilder.parse_bytes(record)
request = request_builder.build_request()
dbsession.add(request)
for event_variant in request_builder.singulars:
singular_event = new_singular_event(request, event_variant, dbsession)
if singular_event is not None:
log.debug('Inserting singular metric:\n%s', singular_event)
for event_variant in request_builder.aggregates:
aggregate_event = new_aggregate_event(request, event_variant,
dbsession)
log.debug('Inserting aggregate metric:\n%s', aggregate_event)
for event_variant in request_builder.sequences:
sequence_event = new_sequence_event(request, event_variant, dbsession)
if sequence_event is not None:
log.debug('Inserting sequence event:\n%s', sequence_event)
try:
dbsession.commit()
except IntegrityError as e:
# FIXME: This is fragile, can we do better?
if "uq_metrics_request_v2_sha512" in str(e):
log.debug('Request had already been processed in the past')
return
# FIXME: Given how the request is built, this shouldn't ever happen; if it does though, we
# absolutely need an integration test
raise # pragma: no cover
def process(dbsession: DbSession, record: bytes) -> None:
log.debug('Processing activation v1 record: %s', record)
activation = Activation.from_serialized(record)
dbsession.add(activation)
log.debug('Inserting activation record:\n%s', activation)
def new_sequence_event(
request: Request, sequence_variant: GLib.Variant, dbsession: DbSession
) -> Optional[Union[SequenceEvent, InvalidSequence, UnknownSequence]]:
event_id = str(UUID(bytes=get_bytes(sequence_variant.get_child_value(1))))
if event_id in IGNORED_EVENTS:
return None
user_id = sequence_variant.get_child_value(0).get_uint32()
events = sequence_variant.get_child_value(2)
num_events = events.n_children()
if num_events < 2:
error = f'Sequence must have at least 2 elements, but only had {num_events}'
# Mypy complains here, even though this should be fine:
# https://github.com/dropbox/sqlalchemy-stubs/issues/97
sequence = InvalidSequence(
request=request,
user_id=user_id, # type: ignore
event_id=event_id,
payload=events,
error=error)
dbsession.add(sequence)
return sequence
start_variant, *_progress_variants, stop_variant = get_child_values(events)
# For now, we ignore progress events entirely. We also assume the stop event always has a null
# payload. This works for most sequence events we care about in priority.
# TODO: Figure this out for the more complex events
start_relative_timestamp = start_variant.get_child_value(0).get_int64()
payload = start_variant.get_child_value(1)
started_at = get_event_datetime(request.absolute_timestamp,
request.relative_timestamp,
start_relative_timestamp)
stop_relative_timestamp = stop_variant.get_child_value(0).get_int64()
stopped_at = get_event_datetime(request.absolute_timestamp,
request.relative_timestamp,
stop_relative_timestamp)
try:
event_model = SEQUENCE_EVENT_MODELS[event_id]
except KeyError:
# Mypy complains here, even though this should be fine:
# https://github.com/dropbox/sqlalchemy-stubs/issues/97
sequence = UnknownSequence(
request=request,
user_id=user_id, # type: ignore
event_id=event_id,
payload=events)
dbsession.add(sequence)
return sequence
try:
# Mypy complains here, even though this should be fine:
# https://github.com/dropbox/sqlalchemy-stubs/issues/97
sequence = event_model(
request=request,
user_id=user_id, # type: ignore
started_at=started_at,
stopped_at=stopped_at,
payload=payload)
except Exception as e:
if isinstance(e, EmptyPayloadError
) and event_id in IGNORED_EMPTY_PAYLOAD_ERRORS:
return None
log.exception('An error occured while processing the sequence:')
# Mypy complains here, even though this should be fine:
# https://github.com/dropbox/sqlalchemy-stubs/issues/97
sequence = InvalidSequence(
request=request,
user_id=user_id, # type: ignore
event_id=event_id,
payload=events,
error=str(e))
dbsession.add(sequence)
return sequence
