async def test_find_groups_to_delete_without_timestamp(user):
# Create fake id
device_id = "".join(str(uuid.uuid1()).split("-"))
device = await graph.store_device_id(user, device_id)
group_id = device["id"]
groups = await find_groups_to_delete()
old_device_ids = {group["displayName"] for group in groups}
assert device_id not in old_device_ids
# Mark group for deletion without a timestamp
await graph.graph_request(
f"/groups/{group_id}",
method="PATCH",
body=json.dumps({
graph.extension_attr_name("toDelete"): True,
graph.extension_attr_name("toDeleteDate"): None,
}),
headers={"Content-Type": "application/json"},
)
capture = LogCapture("tornado.application")
# Since there is not timestamp it should not have been deleted
groups = await find_groups_to_delete()
device_ids = {group["displayName"] for group in groups}
assert device_id not in device_ids
capture.check_present(
(
"tornado.application",
"WARNING",
f"Group {device_id} marked toDelete, but no date! Saving for later.",
),
(
"tornado.application",
"INFO",
f"Marking device id group {device_id} for deletion",
),
)
# Calling it again should now delete it
groups = await find_groups_to_delete()
device_ids = {group["displayName"] for group in groups}
assert device_id in device_ids
await graph.delete_device_group(device_id)
capture.uninstall()
async def collect_users():
"""yield all phone number, device group pairs"""
async for user in list_users():
if user.get(extension_attr_name("testCredentials")):
continue
yield user["displayName"]
async def test_list_users():
list_users_length_before = 0
test_creds = graph.extension_attr_name("testCredentials")
test_user_filter = f"{test_creds} eq true"
async for user in graph.list_users(filter=test_user_filter):
list_users_length_before += 1
# create two test phone numbers
phone_numbers = ["+001234567", "+001010101"]
users = []
# check if phone_numbers are associated with existing test users. If yes, delete them and re-create two test users
for phone_number in phone_numbers:
existing_user = await graph.find_user_by_phone(phone_number)
if not existing_user:
users.append(await test_utils.create_test_user(phone_number))
list_users_length_after = 0
async for user in graph.list_users(filter=test_user_filter):
list_users_length_after += 1
assert list_users_length_after == list_users_length_before + len(users)
# delete created test users
for phone_number in phone_numbers:
test_user_deleted = await graph.find_user_by_phone(phone_number)
await test_utils.delete_test_user(test_user_deleted["id"])
async def collect_devices():
"""yield all phone number, device group pairs"""
sem = asyncio.Semaphore(64)
async def process_one(user):
results = []
async with sem:
async for device in device_groups_for_user(user):
results.append((user["displayName"], device))
return sorted(
results, key=lambda user_device: user_device[1]["createdDateTime"]
)
pending = set()
done = set()
async for user in list_users():
if user.get(extension_attr_name("testCredentials")):
continue
pending.add(asyncio.ensure_future(process_one(user)))
done, pending = await asyncio.wait(pending, timeout=1e-3)
for f in done:
for phone, device in f.result():
yield phone, device["displayName"], device["createdDateTime"]
if pending:
for results in await asyncio.gather(*pending):
for phone, device in results:
yield phone, device["displayName"], device["createdDateTime"]
async def test_mark_iot_deleted(user_with_device_id):
current_time = datetime.datetime.now(datetime.timezone.utc)
group = user_with_device_id.group
await graph.mark_iot_deleted(group, current_time.isoformat())
after = await graph.get_group(group["displayName"])
assert (
parse_date(after.get(graph.extension_attr_name("iotDeletedDate")))
== current_time
)
def test_set_group_attribute(user_with_device_id, event_loop):
group = user_with_device_id.group
run = event_loop.run_until_complete
timestamp = datetime.datetime.utcnow().isoformat()
timestamp = datetime.datetime.utcnow().isoformat() + "Z"
run(graph.set_group_attr(group, sqlDeletedDate=timestamp))
after = run(graph.get_group(group["displayName"]))
assert parse_date(
after.get(graph.extension_attr_name("sqlDeletedDate"))
) == parse_date(timestamp)
async def test_register_device_with_consent_revoked_resets_consent(
http_client, base_url):
await clean_test_user()
await test_utils.create_test_user(TEST_PHONE_NUMBER)
os.environ["TESTER_NUMBERS"] += f",{TEST_PHONE_NUMBER}"
capture = LogCapture()
# Get the user
user = await graph.find_user_by_phone(TEST_PHONE_NUMBER)
await graph.store_consent_revoked(user)
capture.check_present((
"tornado.application",
"INFO",
f"Storing revoked consent on user {utils.mask_phone(TEST_PHONE_NUMBER)}",
))
user = await graph.find_user_by_phone(TEST_PHONE_NUMBER)
assert graph.extension_attr_name("consentRevoked") in user
assert user[graph.extension_attr_name("consentRevoked")] is True
with mock.patch.object(corona_backend.onboarding.app.RegisterDeviceHandler,
"get_current_user") as m:
m.return_value = get_test_payload()
await http_client.fetch(f"{base_url}/register-device",
**post_request_args())
capture.check_present((
"tornado.application",
"WARNING",
f"Clearing revoked consent on user {utils.mask_phone(TEST_PHONE_NUMBER)}",
))
user = await graph.find_user_by_phone(TEST_PHONE_NUMBER)
assert graph.extension_attr_name("consentRevoked") not in user
capture.uninstall()
async def collect_users():
"""yield all userPrincipalName fields for users
Excluding:
- test accounts
- 'real' users who aren't phone numbers
"""
async for user in list_users():
if user.get(extension_attr_name("testCredentials")):
# app_log.info(f"skipping {user['displayName']}")
continue
if not user["displayName"].startswith("+"):
app_log.info(f"skipping {user['displayName']}")
continue
yield user["id"]
async def test_reset_consent_and_store_consent_revoked(user):
extra_attrs = [
graph.extension_attr_name(attr)
for attr in ("deviceId", "consentRevoked", "consentRevokedDate")
]
def get_user():
select = "id, displayName," + ",".join(extra_attrs)
return graph.graph_request(f"/users/{user['id']}", params={"$select": select})
await graph.reset_consent(user)
u = await get_user()
for attr in extra_attrs:
assert attr not in u, attr
await graph.store_consent_revoked(user)
u = await get_user()
assert extra_attrs[0] not in u
assert extra_attrs[1] in u
assert extra_attrs[2] in u
assert u[extra_attrs[1]] is True
timestamp = parse_date(u[extra_attrs[2]])
# This is something to be aware of!
now = datetime.datetime.now(datetime.timezone.utc)
# Just put some tolerance here, say 10 seconds
assert (now - timestamp).total_seconds() < 10
await graph.reset_consent(user)
u = await get_user()
# FIXME: Not sure if this is expected behaviour?
for attr in extra_attrs:
assert attr not in u, attr
DELETE_BATCH_SECONDS = int(os.environ.get("DELETE_BATCH_SECONDS") or 30)
# the date before which we assume sql data doesn't need to be deleted again
# because re-running SQL delete is so slow
# this should not normally be set, but it is now while we are re-importing data from the lake to the db
SQL_CUTOFF_DATE = os.environ.get("SQL_CUTOFF_DATE")
if SQL_CUTOFF_DATE:
SQL_CUTOFF_DATE = parse_date(SQL_CUTOFF_DATE)
# backlog date is a date cutoff so we can limit processing to
# only old data when there's been a backlog buildup
BACKLOG_DATE = os.environ.get("BACKLOG_DATE")
PERSISTENT_CHECK_DB = os.environ.get("PERSISTENT_CHECK_DB", "") == "1"
to_delete = graph.extension_attr_name("toDelete")
to_delete_date = graph.extension_attr_name("toDeleteDate")
iot_deleted_date = graph.extension_attr_name("iotDeletedDate")
sql_deleted_date = graph.extension_attr_name("sqlDeletedDate")
lake_deleted_date = graph.extension_attr_name("lakeDeletedDate")
consent_revoked = graph.extension_attr_name("consentRevoked")
def isonow():
"""ISO8601 UTC timestamp for now"""
return datetime.utcnow().isoformat() + "Z"
async def consume_concurrently(
iterable,
process_one,
async def post(self):
phone_number = self.current_user["_phonenumber"]
is_test = False
TESTER_NUMBERS = set(os.environ.get("TESTER_NUMBERS", "").split(","))
if phone_number in TESTER_NUMBERS and self.request.headers.get("Test-Number"):
is_test = True
tester_number = phone_number
phone_number = self.request.headers["Test-Number"]
masked_number = mask_phone(phone_number)
user = await handlers.find_user(phone_number)
if is_test:
if user.get(graph.extension_attr_name("testCredentials")):
app_log.info(
f"Tester {tester_number} is impersonating test user {phone_number}"
)
else:
app_log.error(
f"Tester {tester_number} attempted to impersonate non-test user {phone_number}"
)
raise web.HTTPError(403, f"{phone_number} is not a test user")
# TODO: check and unset consent revocation on new registration
if user.get(graph.extension_attr_name("consentRevoked")):
consent_revoked_date = user.get(
graph.extension_attr_name("consentRevokedDate")
)
app_log.warning(
f"Phone number {masked_number} had previously revoked consent on {consent_revoked_date}."
" Resetting for new device registration."
)
await graph.reset_consent(user)
existing_device_id = user.get(graph.extension_attr_name("deviceId"))
if existing_device_id:
app_log.warning(
f"Phone number {masked_number} is already associated with device id {existing_device_id}. Registering new device."
)
device_future = asyncio.ensure_future(devices.create_new_device())
tic = time.perf_counter()
try:
await asyncio.wait_for(device_future, timeout=PROVISIONING_TIMEOUT)
except asyncio.TimeoutError:
self.settings["consecutive_failures"] += 1
app_log.error(
"Timeout registering device ({consecutive_failures}/{consecutive_failure_limit} before abort)".format(
**self.settings
)
)
if (
self.settings["consecutive_failures"]
>= self.settings["consecutive_failure_limit"]
):
app_log.critical("Aborting due to consecutive failure limit!")
loop = asyncio.get_event_loop()
loop.call_later(2, loop.stop)
raise web.HTTPError(500, "Timeout registering device")
else:
self.settings["consecutive_failures"] = 0
toc = time.perf_counter()
app_log.info(f"Registered device in {int(1000 * (toc-tic))}ms")
device = await device_future
device_id = device["deviceId"]
device_key = device["authentication"]["symmetricKey"]["primaryKey"]
iothub_hostname = devices.iothub_hostname
# store device id on user in AD
try:
await graph.store_device_id(user, device_id)
except Exception:
# failed to associated user with device
# delete the device from IoTHub since nobody is going to use it
await devices.delete_devices(device_id)
raise
self.write(
json.dumps(
{
"DeviceId": device_id,
"PhoneNumber": phone_number,
"HostName": iothub_hostname,
"SharedAccessKey": device_key,
"ConnectionString": ";".join(
[
f"HostName={iothub_hostname}",
f"DeviceId={device_id}",
f"SharedAccessKey={device_key}",
]
),
}
)
)