def _poll_status_changes(self):
next_url = urllib.parse.urljoin(self.provider.base_api_url,
"status_changes")
if self.provider.api_configuration.get("trailing_slash"):
next_url += "/"
params = {}
# Start where we left, it's all based on providers sorting by start_time
# (we wouldn't know if the provider recorded late telemetries earlier)
if self.provider.last_start_time_polled:
params["start_time"] = utils.to_mds_timestamp(
self.provider.last_start_time_polled)
# Otherwise limit polling
elif PROVIDER_POLLER_LIMIT_DAYS:
params["start_time"] = utils.to_mds_timestamp(
timezone.now() -
datetime.timedelta(PROVIDER_POLLER_LIMIT_DAYS))
# Provider-specific params to optimise polling
try:
params.update(
self.provider.api_configuration["status_changes_params"])
except KeyError:
pass
if params:
next_url = "%s?%s" % (next_url, urllib.parse.urlencode(params))
# Pagination
while next_url:
body = self._get_body(next_url)
# Translate older versions of data
translated_data = translate_data(body["data"], body["version"])
status_changes = translated_data["status_changes"]
if not status_changes:
break
# A transaction for each "page" of data
with transaction.atomic():
last_start_time_polled = self._process_status_changes(
status_changes)
self.provider.last_start_time_polled = last_start_time_polled
self.provider.save(update_fields=["last_start_time_polled"])
next_url = body.get("links", {}).get("next")
def test_compliance_list_basic(client, django_assert_num_queries):
device = factories.Device()
policy = factories.Policy(published=True)
# One provider-specific policy
# provider_policy = factories.ComplianceFactory()
# And one general-purpose policy
compliance = factories.ComplianceFactory(
rule=uuid.UUID("81b1bc92-65b7-4434-8ada-2feeb0b7b223"),
geography=uuid.UUID("e0e4a085-7a50-43e0-afa4-6792ca897c5a"),
policy_id=policy.id,
vehicle_id=device.id,
start_date=datetime.datetime(2007,
12,
6,
16,
29,
43,
79043,
tzinfo=pytz.UTC),
end_date=datetime.datetime(2007,
12,
7,
16,
29,
43,
79043,
tzinfo=pytz.UTC),
)
compliance_ongoing = factories.ComplianceFactory(
rule=uuid.UUID("89b5bbb5-ba98-4498-9649-787eb8ddbb8e"),
geography=uuid.UUID("2cfbdd7f-8ba2-4b48-9826-951fe3249981"),
policy_id=factories.Policy().id,
vehicle_id=factories.Device().id,
start_date=datetime.datetime(2009,
12,
6,
16,
29,
43,
79043,
tzinfo=pytz.UTC),
end_date=None,
)
# Test without auth
n = 2 # Savepoint and release
n += 1 # query on policy
n += 1 # query on related compliances
n += 1 # query on device
n += 1 # query on other device
# query Last compliance
with django_assert_num_queries(n):
response = client.get(reverse("agency-0.3:compliance-list"))
assert response.status_code == 200
# Check why there is policy more (??? what does it mean?)
assert response.data[0]["id"] == str(compliance.policy.id)
# Now test with a provider ID
response = client.get(reverse("agency-0.3:compliance-list"),
{"provider_id": str(device.provider.id)})
# The provider can fetch a policy that applies to them
# (to all providers in this case)
assert response.status_code == 200
assert response.data[0]["id"] == str(compliance.policy_id)
response = client.get(
reverse("agency-0.3:compliance-list"),
{
"provider_id":
str(device.provider.id),
"end_date":
utils.to_mds_timestamp(
datetime.datetime(
2009, 12, 7, 16, 29, 43, 79043, tzinfo=pytz.UTC)),
},
)
# provider is OK but timestamp is too high
assert response.status_code == 200
assert response.data == []
response = client.get(
reverse("agency-0.3:compliance-list"),
{
"provider_id":
str(device.provider.id),
"end_date":
utils.to_mds_timestamp(
datetime.datetime(
2007, 12, 7, 16, 29, 43, 79043, tzinfo=pytz.UTC)) -
60000, # XXX ?!
},
)
# provider is OK and timestamp is OK
assert response.status_code == 200
assert response.data[0]["id"] == str(compliance.policy_id)
response = client.get(
reverse("agency-0.3:compliance-list"),
{
"end_date":
utils.to_mds_timestamp(
datetime.datetime(1970,
1,
14,
0,
47,
3,
900000,
tzinfo=datetime.timezone.utc))
},
)
# too low
assert response.status_code == 200 and response.data == []
response = client.get(
reverse("agency-0.3:compliance-list"),
{
"end_date":
utils.to_mds_timestamp(
datetime.datetime(2070,
1,
21,
3,
45,
56,
700000,
tzinfo=datetime.timezone.utc))
},
)
# too high but compliance_ongoing is not finished
assert response.status_code == 200
assert response.data[0]["id"] == str(compliance_ongoing.policy_id)
response = client.get(
reverse("agency-0.3:compliance-list"),
{"provider_id": "89b5bbb5-ba98-4498-9649-787eb8ddbb8e"},
) # this provider don't exist
assert response.status_code == 200 and response.data == []
def test_policy_list_range(client):
# Policy from last year
past = factories.Policy( # noqa: F841
name="past",
start_date=timezone.now() - datetime.timedelta(days=365),
end_date=timezone.now() - datetime.timedelta(days=350),
published=True,
)
# Policy with lower and upper bounds (half of the lifespan)
bound = factories.Policy( # noqa: F841
name="bound",
start_date=timezone.now() - datetime.timedelta(days=15),
end_date=timezone.now() + datetime.timedelta(days=15),
published=True,
)
# Same but no upper bound
ongoing = factories.Policy( # noqa: F841
name="ongoing",
start_date=timezone.now() - datetime.timedelta(days=15),
published=True,
)
# Policy for next year
future = factories.Policy( # noqa: F841
name="future",
start_date=timezone.now() + datetime.timedelta(days=365),
published=True,
)
# Current and future policies by default
response = client.get(reverse("agency-0.3:policy-list"))
assert [p["name"] for p in response.data] == ["bound", "ongoing", "future"]
# Current only
response = client.get(
reverse("agency-0.3:policy-list"),
{
"start_time": utils.to_mds_timestamp(timezone.now()),
"end_time": utils.to_mds_timestamp(timezone.now()),
},
)
assert [p["name"] for p in response.data] == ["bound", "ongoing"]
# Ongoing is still.. ongoing, and future
response = client.get(
reverse("agency-0.3:policy-list"),
{
"start_time":
utils.to_mds_timestamp(timezone.now() +
datetime.timedelta(days=30))
},
)
assert [p["name"] for p in response.data] == ["ongoing", "future"]
# Past only
response = client.get(
reverse("agency-0.3:policy-list"),
{
"start_time":
utils.to_mds_timestamp(timezone.now() -
datetime.timedelta(days=365)),
"end_time":
utils.to_mds_timestamp(timezone.now() -
datetime.timedelta(days=30)),
},
)
assert [p["name"] for p in response.data] == ["past"]
# All
response = client.get(
reverse("agency-0.3:policy-list"),
{
"start_time":
utils.to_mds_timestamp(timezone.now() -
datetime.timedelta(days=365)),
"end_time":
utils.to_mds_timestamp(timezone.now() +
datetime.timedelta(days=365)),
},
)
assert [p["name"]
for p in response.data] == ["past", "bound", "ongoing", "future"]
"timestamp": 1_325_376_000_000,
"gps": {"lat": 33.996_339, "lng": -118.48153},
},
},
)
expected_event_device1 = {
"provider_id": str(provider.id),
"provider_name": "Test provider",
"device_id": uuid1,
"vehicle_id": "1AAAAA",
"propulsion_type": ["combustion"],
"event_type_reason": "maintenance",
"event_type": "unavailable",
"vehicle_type": "car",
"event_time": utils.to_mds_timestamp(now),
"event_location": {
"type": "Feature",
"properties": {"timestamp": 1_325_376_000_000},
"geometry": {"type": "Point", "coordinates": [-118.48153, 33.996_339]},
},
"associated_trip": "b3da2d46-065f-4036-903c-49d796f09357",
}
expected_event_device2 = {
"provider_id": str(provider2.id),
"provider_name": "Test another provider",
"device_id": uuid2,
"vehicle_id": "3CCCCC",
"propulsion_type": ["electric"],
"event_type_reason": "trip_start",
"event_type": "trip",
def _poll_status_changes_v0_2(self, api_version):
# Start where we left, it's all based on providers sorting by start_time
# (but we would miss telemetries older than start_time saved after we polled).
# For those that support it, use the recorded time field or equivalent.
polling_cursor = self.cursor or self.provider.api_configuration.get(
"polling_cursor", POLLING_CURSORS.start_time.name
)
logger.info(
f"Starting polling {self.provider} using the field: {polling_cursor}.\n"
"Current state:\n"
+ (
(
f"\tLast event_time: {str(self.provider.last_event_time_polled)},\n"
+ f"\tLast recorded: {str(self.provider.last_recorded_polled)}\n"
+ f"\tLast skip: {self.provider.last_skip_polled}."
)
if not self.cursor
else f"\tCursor value: {self.from_cursor}."
)
)
params = {}
# Cursor param (raise if typo)
param_name = POLLING_CURSORS[polling_cursor].value
if polling_cursor == POLLING_CURSORS.total_events.name:
# Resume from the last line fetched
params[param_name] = self.from_cursor or self.provider.last_skip_polled
elif polling_cursor == POLLING_CURSORS.start_recorded.name:
# Resume from the last "recorded" value
params[param_name] = utils.to_mds_timestamp(
self.from_cursor or self.provider.last_recorded_polled
)
elif polling_cursor == POLLING_CURSORS.start_time.name:
# Resume from the last "event_time" value
last_event_time_polled = self.provider.last_event_time_polled
if not last_event_time_polled:
last_event_time_polled = timezone.now() - datetime.timedelta(
getattr(settings, "PROVIDER_POLLER_LIMIT", 90)
)
# But we now apply a "lag" before actually polling,
# leaving time for the provider to collect data from its devices
polling_lag = self.provider.api_configuration.get("provider_polling_lag")
if polling_lag:
polling_lag = parse_duration(polling_lag)
if (timezone.now() - last_event_time_polled) < polling_lag:
logger.debug("Still under the polling lag, back to sleep.")
return
params[param_name] = utils.to_mds_timestamp(
self.from_cursor or last_event_time_polled
)
# Provider-specific params to optimise polling
try:
params.update(self.provider.api_configuration["status_changes_params"])
except KeyError:
pass
next_url = urllib.parse.urljoin(self.provider.base_api_url, "status_changes")
if self.provider.api_configuration.get("trailing_slash"):
next_url += "/"
if params:
next_url = "%s?%s" % (next_url, urllib.parse.urlencode(params))
skip_polled = (
self.from_cursor
if self.from_cursor and self.cursor == POLLING_CURSORS.total_events.name
else None
)
# Pagination
while next_url:
body = self._get_body(next_url, api_version)
# Translate older versions of data
translated_data = translate_v0_2_to_v0_4(body["data"])
status_changes = translated_data["status_changes"]
if not status_changes:
break
next_url = body.get("links", {}).get("next")
# A transaction for each "page" of data
with transaction.atomic():
# We get the maximum of the recorded and event_types
# from the status changes
event_time_polled, recorded_polled = self._process_status_changes(
status_changes
)
if self.cursor:
if self.cursor == POLLING_CURSORS.total_events.name:
skip_polled += len(status_changes)
if skip_polled >= self.to_cursor:
break
elif self.cursor == POLLING_CURSORS.start_recorded.name:
if recorded_polled >= self.to_cursor:
break
elif self.cursor == POLLING_CURSORS.start_time.name:
if event_time_polled >= self.to_cursor:
break
else:
# We get the new skip from the number of status changes
skip_polled = (
self.provider.last_skip_polled + len(status_changes)
if self.provider.last_skip_polled
else len(status_changes)
)
self.provider.last_event_time_polled = event_time_polled
self.provider.last_recorded_polled = recorded_polled
self.provider.last_skip_polled = skip_polled
self.provider.save(
update_fields=[
"last_event_time_polled",
"last_recorded_polled",
"last_skip_polled",
]
)
logger.info(
f"Polled page using cursor: {polling_cursor}. New state:\n"
+ f"\tLast event_time: {str(event_time_polled)},\n"
+ f"\tLast recorded: {str(recorded_polled)}\n"
+ f"\tLast skip: {skip_polled}."
)
def _poll_status_changes_v0_4(self, api_version):
# Start where we left, it's all based on providers sorting by start_time
# (but we would miss telemetries older than start_time saved after we polled).
# For those that support it, use the recorded time field or equivalent.
polling_cursor = self.cursor or self.provider.api_configuration.get(
"polling_cursor", POLLING_CURSORS.start_time.name
)
if polling_cursor != POLLING_CURSORS.start_time.name:
raise ValueError('Only "start_time" cursor is supported in MDS 0.4+')
logger.info(
f"Starting polling {self.provider} using the field: {polling_cursor}.\n"
"Current state:\n"
+ (
(f"\tLast event_time: {str(self.provider.last_event_time_polled)}.")
if not self.cursor
else f"\tCursor value: {self.from_cursor}."
)
)
params = {}
# Resume from the last "event_time" value
last_event_time_polled = (
self.from_cursor or self.provider.last_event_time_polled
)
if not last_event_time_polled:
last_event_time_polled = timezone.now() - datetime.timedelta(
getattr(settings, "PROVIDER_POLLER_LIMIT", 90)
)
# But we now apply a "lag" before actually polling,
# leaving time for the provider to collect data from its devices
polling_lag = self.provider.api_configuration.get("provider_polling_lag")
if polling_lag:
polling_lag = parse_duration(polling_lag)
if (timezone.now() - last_event_time_polled) < polling_lag:
logger.info("Still under the polling lag, back to sleep.")
return
# The MDS 0.4 Provider API got two endpoints:
# - /events for the real-time or so data (formerly /status_changes)
# but limited to two weeks of history
# - /status_changes for all the data except the current hour
# If we're catching up far in time, begin by polling /status_changes
realtime_threshold = parse_duration( # Default is 9 days
self.provider.api_configuration.get("realtime_threshold", "P9D")
)
next_event_time = last_event_time_polled + datetime.timedelta(hours=1)
if (timezone.now() - next_event_time) > realtime_threshold:
# We have to query the archived status changes with another format
logger.info("last_event_time_polled is too old, asking archives")
# We're done with the events of the last hour, ask the next hour
params["event_time"] = next_event_time.isoformat()[: len("YYYY-MM-DDTHH")]
else:
# Query the real-time endpoint as usual
params["start_time"] = utils.to_mds_timestamp(last_event_time_polled)
# Both bounds are mandatory now, use the lag as the event horizon
# The provider will slice big results using pagination
end_time = timezone.now()
if polling_lag:
# We tested the lag above, so end_time can't be older than start_time
end_time -= polling_lag
params["end_time"] = utils.to_mds_timestamp(end_time)
# Provider-specific params to optimise polling
try:
params.update(self.provider.api_configuration["status_changes_params"])
except KeyError:
pass
endpoint = "events" # The new name for the real-time events endpoint
if "event_time" in params:
# We asked the archived status changes instead
endpoint = "status_changes"
next_url = urllib.parse.urljoin(self.provider.base_api_url, endpoint)
if self.provider.api_configuration.get("trailing_slash"):
next_url += "/"
if params:
next_url = "%s?%s" % (next_url, urllib.parse.urlencode(params))
# Pagination
while next_url:
body = self._get_body(next_url, api_version)
# No translation needed as long as 0.4 is the latest version
status_changes = body["data"]["status_changes"]
next_url = body.get("links", {}).get("next")
# A transaction for each "page" of data
with transaction.atomic():
if status_changes:
# We get the maximum values from the status changes
event_time_polled, _ = self._process_status_changes(status_changes)
elif endpoint == "status_changes":
# This hour frame of archives didn't contain results
event_time_polled = next_event_time
else:
# Try again from this point later
break
if self.cursor:
if self.cursor == POLLING_CURSORS.start_time.name:
if event_time_polled >= self.to_cursor:
break
else:
self.provider.last_event_time_polled = event_time_polled
self.provider.save(update_fields=["last_event_time_polled"])
logger.info(
f"Polled page using cursor: {polling_cursor}. New state:\n"
+ f"\tLast event_time: {str(event_time_polled)}."
)