def test_get_unfinished_sessions(self):
with Transaction() as t:
r = VioscreenSessionRepo(t)
cur = t.cursor()
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username for r in obs}, exp)
user1 = VIOSCREEN_SESSION.copy()
user1.username = VIOSCREEN_USERNAME1
obs = r.upsert_session(user1)
self.assertTrue(obs)
# our base session is still unfinished (no end date)
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username for r in obs}, exp)
# set a finished status
user1.status = 'Finished'
user1.endDate = _to_dt(1, 1, 1971)
obs = r.upsert_session(user1)
self.assertTrue(obs)
# our users record is finished so we shouldn't get it back
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username
for r in obs}, exp - {
VIOSCREEN_USERNAME1,
})
def test_get_unfinished_sessions_multiple(self):
with Transaction() as t:
r = VioscreenSessionRepo(t)
cur = t.cursor()
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username for r in obs}, exp)
sess1 = VIOSCREEN_SESSION.copy()
sess1.username = VIOSCREEN_USERNAME1
sess2 = VIOSCREEN_SESSION.copy()
sess2.username = VIOSCREEN_USERNAME1
sess2.sessionId = 'a different sessionid'
obs = r.upsert_session(sess1)
self.assertTrue(obs)
obs = r.upsert_session(sess2)
self.assertTrue(obs)
# our sessions are unfinished
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username for r in obs}, exp)
# set a finished status
sess1.status = 'Finished'
sess1.endDate = _to_dt(1, 1, 1971)
obs = r.upsert_session(sess1)
self.assertTrue(obs)
# one session is finished, and we only actually understand the
# sematics of a single session anyway, so under our current
# operating assumptions, this users FFQ is now complete
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username
for r in obs}, exp - {
VIOSCREEN_USERNAME1,
})
def test_get_missing_ffqs(self):
with Transaction() as t:
cur = t.cursor()
r = VioscreenSessionRepo(t)
pr = VioscreenPercentEnergyRepo(t)
user1 = VIOSCREEN_SESSION.copy()
user1.username = VIOSCREEN_USERNAME1
r.upsert_session(user1)
# our user is not present as they do not have an enddate or
# finished status
obs = r.get_missing_ffqs()
self.assertNotIn(VIOSCREEN_USERNAME1, {o.username for o in obs})
user1.status = 'Finished'
user1.endDate = _to_dt(1, 1, 1971)
r.upsert_session(user1)
# our finished session does not have ffq data
obs = r.get_missing_ffqs()
self.assertIn(VIOSCREEN_USERNAME1, {o.username for o in obs})
# give our user a "completed" ffq
user1_pe = VIOSCREEN_PERCENT_ENERGY
pr.insert_percent_energy(user1_pe)
obs = r.get_missing_ffqs()
self.assertNotIn(VIOSCREEN_USERNAME1, {o.username for o in obs})
# our users record is finished so we shouldn't get it back
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username
for r in obs}, exp - {
VIOSCREEN_USERNAME1,
})
def fetch_ffqs():
MAX_FETCH_SIZE = 100
vio_api = VioscreenAdminAPI(perform_async=False)
# obtain our current unfinished sessions to check
with Transaction() as t:
r = VioscreenSessionRepo(t)
not_represented = r.get_unfinished_sessions()
# collect sessions to update
unable_to_update_session = []
updated_sessions = []
for sess in enumerate(not_represented):
# update session status information
try:
session_detail = vio_api.session_detail(sess)
except: # noqa
unable_to_update_session.append(sess)
else:
updated_sessions.append(session_detail)
# perform the update after we collect everything to reduce teh length of
# time we hold the transaction open
with Transaction() as t:
r = VioscreenSessionRepo(t)
for session_detail in updated_sessions:
r.upsert_session(session_detail)
t.commit()
with Transaction() as t:
vs = VioscreenSessionRepo(t)
ffqs_not_represented = vs.get_missing_ffqs()
# fetch ffq data for sessions we don't yet have it from
failed_ffqs = []
for sess in ffqs_not_represented[:MAX_FETCH_SIZE]:
try:
error, ffq = vio_api.get_ffq(sess.sessionId)
except Exception as e: # noqa
failed_ffqs.append((sess.sessionId, str(e)))
continue
if error:
failed_ffqs.append((sess.sessionId, repr(error)))
else:
# the call to get_ffq is long, and the data from many ffqs is
# large so let's insert as we go
with Transaction() as t:
vs = VioscreenRepo(t)
vs.insert_ffq(ffq)
t.commit()
if len(failed_ffqs) > 0 or len(unable_to_update_session) > 0:
payload = ''.join([
'%s : %s\n' % (repr(s), m)
for s, m in failed_ffqs + unable_to_update_session
])
send_email(SERVER_CONFIG['pester_email'], "pester_daniel", {
"what": "Vioscreen ffq insert failed",
"content": payload
}, EN_US)
def update_session_detail():
# The interaction with the API is occuring within a celery task
# and the recommendation from celery is to avoid depending on synchronous
# tasks from within a task. As such, we'll use non-async calls here. See
# http://docs.celeryq.org/en/latest/userguide/tasks.html#task-synchronous-subtasks
# HOWEVER, we could implement a watch and monitor child tasks, but
# not sure if that would be a particular benefit here
vio_api = VioscreenAdminAPI(perform_async=False)
current_task.update_state(state="PROGRESS",
meta={
"completion": 0,
"status": "PROGRESS",
"message": "Gathering unfinished sessions..."
})
# obtain our current unfinished sessions to check
with Transaction() as t:
r = VioscreenSessionRepo(t)
unfinished_sessions = r.get_unfinished_sessions()
failed_sessions = []
n_to_get = len(unfinished_sessions)
n_updated = 0
for idx, sess in enumerate(unfinished_sessions, 1):
updated = []
# Out of caution, we'll wrap the external resource interaction within
# a blanket try/except
try:
if sess.sessionId is None:
# a session requires a mix of information from Vioscreen's
# representation of a user and a session
user_detail = vio_api.user(sess.username)
details = vio_api.sessions(sess.username)
# account for the possibility of a user having multiple
# sessions
updated.extend([
VioscreenSession.from_vioscreen(detail, user_detail)
for detail in details
])
else:
# update our model inplace
update = vio_api.session_detail(sess.sessionId)
if update.status != sess.status:
updated.append(sess.update_from_vioscreen(update))
except Exception as e: # noqa
failed_sessions.append((sess.sessionId, str(e)))
continue
# commit as we go along to avoid holding any individual transaction
# open for a long period
if len(updated) > 0:
with Transaction() as t:
r = VioscreenSessionRepo(t)
for update in updated:
r.upsert_session(update)
t.commit()
n_updated += len(updated)
current_task.update_state(state="PROGRESS",
meta={
"completion": (idx / n_to_get) * 100,
"status": "PROGRESS",
"message": "Gathering session data..."
})
current_task.update_state(state="SUCCESS",
meta={
"completion": n_to_get,
"status": "SUCCESS",
"message": f"{n_updated} sessions updated"
})
if len(failed_sessions) > 0:
# ...and let's make Daniel feel bad about not having a better means to
# log what hopefully never occurs
payload = ''.join(
['%s : %s\n' % (repr(s), m) for s, m in failed_sessions])
send_email(SERVER_CONFIG['pester_email'], "pester_daniel", {
"what": "Vioscreen sessions failed",
"content": payload
}, EN_US)