def get_queryset(self):
queryset = super().get_queryset()
provider_id = getattr(self.request.user, "provider_id", None)
if provider_id:
# Filter for general-purpose policies,
# or the ones written for this provider
queryset = queryset.filter(
Q(providers__isnull=True) | Q(providers=provider_id)
)
# The raw results will expose what other providers this policy applies to
# Is this information leakage?
else:
# Only general-purpose policies to other users
queryset = queryset.filter(providers__isnull=True)
# Filter by date range
start_time = self.request.GET.get("start_time")
if start_time:
start_time = utils.from_mds_timestamp(int(start_time))
else:
start_time = Now()
range_q = Q(end_date__gt=start_time) | Q(end_date__isnull=True)
end_time = self.request.GET.get("end_time")
if end_time:
end_time = utils.from_mds_timestamp(int(end_time))
range_q &= Q(start_date__lte=end_time)
queryset = queryset.filter(range_q)
return queryset
def _create_event_record(status_change):
properties = {"trip_id": status_change.get("associated_trip")}
event_location = status_change["event_location"]
if event_location:
# GeoJSON Point Feature
try:
longitude, latitude, altitude = event_location["geometry"][
"coordinates"]
except ValueError:
longitude, latitude = event_location["geometry"]["coordinates"]
altitude = None
point = geos.Point(longitude, latitude, altitude, srid=4326)
properties["telemetry"] = {
"timestamp": event_location["properties"]["timestamp"],
"gps": {
"lng": longitude,
"lat": latitude
},
# No coordinates, no battery charge saved
"battery_pct": status_change.get("battery_pct"),
}
if altitude:
properties["telemetry"]["gps"]["altitude"] = altitude
else: # Spec violation!
point = None
return models.EventRecord(
device_id=status_change["device_id"],
timestamp=utils.from_mds_timestamp(status_change["event_time"]),
point=point,
event_type=status_change["agency_event_type"],
properties=properties,
)
def _process_status_changes(self, status_changes):
logger.debug("Processing...")
# accept timestamp as a string instead of an integer
status_changes = self._validate_event_times(status_changes)
if not status_changes:
# Data so bad there is no or nothing but invalid event times
logger.exception(
"No valid event_time found in status_changes series: %s",
status_changes)
# How can we prevent from asking them again next time?
if self.provider.last_start_time_polled:
return self.provider.last_start_time_polled + datetime.timedelta(
milliseconds=1)
# The provider really doesn't help!
return timezone.now()
# do not rely on expected order
last_event_time_polled = utils.from_mds_timestamp(
max(status_change["event_time"]
for status_change in status_changes))
status_changes = self._validate_status_changes(status_changes)
if not status_changes:
# None were valid, we won't ask that series again
# (provided status changes are ordered by event_time ascending)
return last_event_time_polled
self._create_missing_providers(status_changes)
self._create_missing_devices(status_changes)
self._create_event_records(status_changes)
return last_event_time_polled
def _create_event_record(status_change):
properties = {"trip_id": status_change.get("associated_trip")}
event_location = status_change["event_location"]
if event_location:
# GeoJSON Point Feature
try:
longitude, latitude, altitude = event_location["geometry"]["coordinates"]
except ValueError:
longitude, latitude = event_location["geometry"]["coordinates"]
altitude = None
point = geos.Point(longitude, latitude, altitude, srid=4326)
properties["telemetry"] = {
"timestamp": event_location["properties"]["timestamp"],
"gps": {"lng": longitude, "lat": latitude},
# No coordinates, no battery charge saved
"battery_pct": status_change.get("battery_pct"),
}
if altitude:
properties["telemetry"]["gps"]["altitude"] = altitude
else: # Spec violation!
point = None
publication_time = None
if status_change.get("publication_time"):
publication_time = utils.from_mds_timestamp(status_change["publication_time"])
# "Aggregation" providers store a recorded field, and we want to keep the same value
# until we get the publication_time everywhere
# Vendor only, 0.3 only, will disappear
recorded = None
if status_change.get("recorded"):
recorded = utils.from_mds_timestamp(status_change["recorded"])
if publication_time and recorded:
difference = abs(publication_time - recorded)
if difference > datetime.timedelta(minutes=10):
logger.warning("publication_time and recorded differ by %s", difference)
return models.EventRecord(
device_id=status_change["device_id"],
timestamp=utils.from_mds_timestamp(status_change["event_time"]),
point=point,
event_type=status_change["agency_event_type"],
event_type_reason=status_change["agency_event_type_reason"],
properties=properties,
publication_time=publication_time or recorded,
)
def events_callback(request, context):
"""Check the query parameters"""
# Start time is were the poller stopped last time
start_time = utils.from_mds_timestamp(int(request.qs["start_time"][0]))
assert almost_equal(start_time, last_event_time_polled)
# End time is after start time but before the lag threshold
end_time = utils.from_mds_timestamp(int(request.qs["end_time"][0]))
assert end_time > start_time
assert almost_equal(end_time,
timezone.now(),
precision=lag_plus_one_second)
context.status_code = 200
return {
"version": "0.4.0",
"data": {
"status_changes": []
},
}
def status_changes(self, request, *args, **kwargs):
start_time = request.query_params.get("start_time")
end_time = request.query_params.get("end_time")
# Only forward events that were first retrieved though providers'
# `status_changes' endpoint
event_types = enums.PROVIDER_REASON_TO_AGENCY_EVENT.values()
events = models.EventRecord.objects.select_related(
"device__provider").filter(event_type__in=event_types)
if start_time:
start_time = utils.from_mds_timestamp(int(start_time))
events = events.filter(timestamp__gte=start_time)
if end_time:
end_time = utils.from_mds_timestamp(int(end_time))
events = events.filter(timestamp__lte=end_time)
paginator = CustomPagination()
page = paginator.paginate_queryset(events.order_by("timestamp"),
request)
data = DeviceStatusChangesSerializer(page, many=True).data
return paginator.get_paginated_response(data)
def events_callback(request, context):
"""Check the query parameters"""
# Start time is were the poller stopped last time
start_time = utils.from_mds_timestamp(int(request.qs["start_time"][0]))
assert almost_equal(start_time, last_event_time_polled)
# End time is after start time but valued "now()" when the poller was running
end_time = utils.from_mds_timestamp(int(request.qs["end_time"][0]))
assert end_time > start_time
assert almost_equal(end_time,
timezone.now(),
precision=datetime.timedelta(seconds=1))
context.status_code = 200
return make_response(
provider,
expected_device,
expected_event,
event_type_reason="service_start",
version="0.4.0",
)
def get_queryset(self):
queryset = super().get_queryset()
provider_id = self.request.GET.get("provider_id")
end_date = self.request.GET.get("end_date")
if end_date:
end_date = utils.from_mds_timestamp(int(end_date))
filters = {}
if provider_id and end_date:
filters["compliances__vehicle__provider__id"] = provider_id
filters["compliances__start_date__lte"] = end_date
filter_prefetch = Prefetch(
"compliances",
queryset=models.Compliance.objects.exclude(
end_date__lte=end_date).filter(
vehicle__provider__id=provider_id,
start_date__lt=end_date),
to_attr="compliances_pref",
)
elif end_date:
filter_prefetch = Prefetch(
"compliances",
queryset=models.Compliance.objects.exclude(
end_date__lte=end_date).filter(start_date__lt=end_date),
to_attr="compliances_pref",
)
filters["compliances__start_date__lte"] = end_date
elif provider_id:
filter_prefetch = Prefetch(
"compliances",
queryset=models.Compliance.objects.filter(
vehicle__provider__id=provider_id),
to_attr="compliances_pref",
)
filters["compliances__vehicle__provider__id"] = provider_id
else:
filter_prefetch = Prefetch(
"compliances",
queryset=models.Compliance.objects.filter(),
to_attr="compliances_pref",
)
if (provider_id and end_date) or end_date:
queryset = (queryset.exclude(
compliances__end_date__lt=end_date).filter(
**filters).prefetch_related(filter_prefetch))
elif provider_id:
queryset = queryset.filter(
**filters).prefetch_related(filter_prefetch)
else:
queryset = queryset.prefetch_related(filter_prefetch)
return queryset
def _create_register_event_record(status_change):
"""
As the goal of the poller is to catch up with the history of a provider,
simulate the registration of a device with a fake register event.
This fake event is flagged in its properties to tell it apart from real ones.
Should the device be unregistered and registered again according to the specs,
don't delete the fake events in the past.
"""
return models.EventRecord(
device_id=status_change["device_id"],
# Another event for the same device with the same timestamp will be rejected
timestamp=utils.from_mds_timestamp(status_change["event_time"]) -
datetime.timedelta(milliseconds=1),
event_type=enums.EVENT_TYPE.register.name,
properties={"created_on_register": True},
source=enums.EVENT_SOURCE.provider_api.name,
)