async def test_list_users():
list_users_length_before = 0
test_creds = graph.extension_attr_name("testCredentials")
test_user_filter = f"{test_creds} eq true"
async for user in graph.list_users(filter=test_user_filter):
list_users_length_before += 1
# create two test phone numbers
phone_numbers = ["+001234567", "+001010101"]
users = []
# check if phone_numbers are associated with existing test users. If yes, delete them and re-create two test users
for phone_number in phone_numbers:
existing_user = await graph.find_user_by_phone(phone_number)
if not existing_user:
users.append(await test_utils.create_test_user(phone_number))
list_users_length_after = 0
async for user in graph.list_users(filter=test_user_filter):
list_users_length_after += 1
assert list_users_length_after == list_users_length_before + len(users)
# delete created test users
for phone_number in phone_numbers:
test_user_deleted = await graph.find_user_by_phone(phone_number)
await test_utils.delete_test_user(test_user_deleted["id"])
async def collect_users():
"""yield all phone number, device group pairs"""
async for user in list_users():
if user.get(extension_attr_name("testCredentials")):
continue
yield user["displayName"]
async def collect_devices():
"""yield all phone number, device group pairs"""
sem = asyncio.Semaphore(64)
async def process_one(user):
results = []
async with sem:
async for device in device_groups_for_user(user):
results.append((user["displayName"], device))
return sorted(
results, key=lambda user_device: user_device[1]["createdDateTime"]
)
pending = set()
done = set()
async for user in list_users():
if user.get(extension_attr_name("testCredentials")):
continue
pending.add(asyncio.ensure_future(process_one(user)))
done, pending = await asyncio.wait(pending, timeout=1e-3)
for f in done:
for phone, device in f.result():
yield phone, device["displayName"], device["createdDateTime"]
if pending:
for results in await asyncio.gather(*pending):
for phone, device in results:
yield phone, device["displayName"], device["createdDateTime"]
async def collect_users():
"""yield all userPrincipalName fields for users
Excluding:
- test accounts
- 'real' users who aren't phone numbers
"""
async for user in list_users():
if user.get(extension_attr_name("testCredentials")):
# app_log.info(f"skipping {user['displayName']}")
continue
if not user["displayName"].startswith("+"):
app_log.info(f"skipping {user['displayName']}")
continue
yield user["id"]
async def find_users_to_delete(limit=None):
"""Find users whose devices should be deleted"""
async for user in graph.list_users(filter=f"{consent_revoked} eq true"):
if user.get(consent_revoked):
app_log.warning(f"User {user['logName']} with consent revoked not removed!")
yield user
else:
app_log.warning(
f"User {user['logName']} without consent revoked shouldn't have been returned by query."
)
inactive = await find_inactive_devices()
app_log.info(f"Found {len(inactive)} inactive devices")
counts = defaultdict(int)
async def process_one(uuid_activity):
uuid, last_activity = uuid_activity
group = await graph.get_group(uuid)
if group is None:
app_log.info(f"No group for inactive device {uuid}")
counts["no_group"] += 1
return
if group.get(consent_revoked):
app_log.info(f"Already marked for deletion: {uuid}")
counts["deleted"] += 1
return
user = await graph.user_for_device(group)
if user is None:
app_log.info(f"No user for inactive device {uuid}")
counts["no_user"] += 1
# FIXME: something went wrong. Mark device id group for deletion?
return
# check other devices on the same user
# in case of new device registrations,
# don't delete data from a user's old phone
other_device_activity = None
device_ids = [uuid]
async for group in graph.device_groups_for_user(user):
device_id = group["displayName"]
if device_id != uuid:
device_ids.append(device_id)
# First check for recent registration (cheap)
if parse_date(group["createdDateTime"]) >= IDLE_CUTOFF:
app_log.info(f"Recently registered device {device_id}")
counts["new"] += 1
if device_id == uuid:
app_log.warning(
f"WRONG activity: recently registered {device_id} not idle"
)
counts["wrong"] += 1
other_device_activity = True
break
try:
device = await devices.get_device(device_id)
except Exception as e:
app_log.warning(f"Failed to get device {device_id}: ({e})")
counts["iot_err"] += 1
pass
else:
if parse_date(device["lastActivityTime"]) >= IDLE_CUTOFF:
counts["iot"] += 1
app_log.info(f"Activity on {device_id} in IoTHub")
if device_id == uuid:
app_log.warning(
f"WRONG activity: iothub active {device_id} not idle"
)
counts["wrong"] += 1
other_device_activity = True
break
if device_id != uuid:
# if not registered since cutoff, check for activity in SQL
if await check_sql_data(device_id, activity_cutoff=IDLE_CUTOFF):
counts["sql"] += 1
app_log.info(f"Activity on {device_id} in SQL")
other_device_activity = True
break
if other_device_activity:
app_log.info(f"{uuid} is associated with other more recent device activity")
counts["active"] += 1
else:
app_log.info(f"User {user['logName']} is inactive since {last_activity}")
app_log.info(f"Inactive devices: {','.join(device_ids)}")
counts["idle"] += 1
return user
yielded = 0
async for user in consume_concurrently(
inactive, process_one, counts=counts, label="Inactive users"
):
if user:
yield user
yielded += 1
if limit and yielded >= limit:
app_log.info(f"Reached idle user limit={limit}")
return