def test_get_mpeds_does_not_exist(self):
with Transaction() as t:
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenMPedsRepo(t)
obs = r.get_mpeds('does not exist')
self.assertEqual(obs, None)
def test_get_percent_energy_does_not_exist(self):
with Transaction() as t:
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenPercentEnergyRepo(t)
obs = r.get_percent_energy('does not exist')
self.assertEqual(obs, None)
def test_insert_supplements_does_not_exist(self):
with Transaction() as t:
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenSupplementsRepo(t)
obs = r.insert_supplements(VIOSCREEN_SUPPLEMENTS)
self.assertEqual(obs, 2)
def test_get_sessions_by_username_multiple(self):
to_insert = [
VIOSCREEN_SESSION.copy(),
VIOSCREEN_SESSION.copy(),
VIOSCREEN_SESSION.copy()
]
# NOTE: the username is held constant across these records. This
# emulates a user haveing multiple FFQ sessions
to_insert[0].sessionId = 'session1'
to_insert[1].sessionId = 'session2'
to_insert[2].sessionId = 'session3'
to_insert[0].created = _to_dt(1, 1, 1970)
to_insert[1].created = _to_dt(1, 1, 1980)
to_insert[2].created = _to_dt(1, 1, 1990)
# the third entry, while started the latest, does not have an enddate
# so cannot be complete
to_insert[0].endDate = _to_dt(1, 1, 1971)
to_insert[1].endDate = _to_dt(1, 1, 1981)
to_insert[2].endDate = None
with Transaction() as t:
r = VioscreenSessionRepo(t)
for record in to_insert:
obs = r.upsert_session(record)
self.assertEqual(obs, True)
obs = r.get_sessions_by_username(VIOSCREEN_SESSION.username)
self.assertEqual(obs, to_insert)
def test_get_food_consumption_does_not_exist(self):
with Transaction() as t:
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenFoodConsumptionRepo(t)
obs = r.get_food_consumption('does not exist')
self.assertEqual(obs, None)
def test_get_unfinished_sessions(self):
with Transaction() as t:
r = VioscreenSessionRepo(t)
cur = t.cursor()
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username for r in obs}, exp)
user1 = VIOSCREEN_SESSION.copy()
user1.username = VIOSCREEN_USERNAME1
obs = r.upsert_session(user1)
self.assertTrue(obs)
# our base session is still unfinished (no end date)
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username for r in obs}, exp)
# set a finished status
user1.status = 'Finished'
user1.endDate = _to_dt(1, 1, 1971)
obs = r.upsert_session(user1)
self.assertTrue(obs)
# our users record is finished so we shouldn't get it back
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username
for r in obs}, exp - {
VIOSCREEN_USERNAME1,
})
def test_insert_percent_energy_does_not_exist(self):
with Transaction() as t:
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenPercentEnergyRepo(t)
obs = r.insert_percent_energy(VIOSCREEN_PERCENT_ENERGY)
self.assertEqual(obs, 8)
def test_get_supplements_exists(self):
with Transaction() as t:
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenSupplementsRepo(t)
r.insert_supplements(VIOSCREEN_SUPPLEMENTS)
obs = r.get_supplements(VIOSCREEN_SUPPLEMENTS.sessionId)
self.assertEqual(obs, VIOSCREEN_SUPPLEMENTS)
def test_get_percent_energy_exists(self):
with Transaction() as t:
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenPercentEnergyRepo(t)
r.insert_percent_energy(VIOSCREEN_PERCENT_ENERGY)
obs = r.get_percent_energy(VIOSCREEN_PERCENT_ENERGY.sessionId)
self._assert_unordered_components(obs, VIOSCREEN_PERCENT_ENERGY)
def test_insert_dietary_score_does_not_exist(self):
with Transaction() as t:
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenDietaryScoreRepo(t)
obs = r.insert_dietary_scores([
VIOSCREEN_DIETARY_SCORE,
])
self.assertEqual(obs, 13)
def test_get_sessions_by_username_exists(self):
with Transaction() as t:
r = VioscreenSessionRepo(t)
obs = r.upsert_session(VIOSCREEN_SESSION)
self.assertEqual(obs, True)
obs = r.get_sessions_by_username(VIOSCREEN_SESSION.username)
self.assertEqual(obs, [
VIOSCREEN_SESSION,
])
def test_insert_mpeds_does_not_exist(self):
with Transaction() as t:
with open(get_data_path("mpeds.data")) as data:
MPEDS_DATA = json.load(data)
VIOSCREEN_MPEDS = VioscreenMPeds.from_vioscreen(MPEDS_DATA[0])
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenMPedsRepo(t)
obs = r.insert_mpeds(VIOSCREEN_MPEDS)
self.assertEqual(obs, 40)
def test_insert_food_components_does_not_exist(self):
with Transaction() as t:
with open(get_data_path("foodcomponents.data")) as data:
FC_DATA = json.load(data)
VIOSCREEN_FOOD_COMPONENTS = \
VioscreenFoodComponents.from_vioscreen(FC_DATA[0])
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenFoodComponentsRepo(t)
obs = r.insert_food_components(VIOSCREEN_FOOD_COMPONENTS)
self.assertEqual(obs, 156)
def test_insert_food_consumption_does_not_exist(self):
with Transaction() as t:
with open(get_data_path("foodconsumption.data")) as data:
CONS_DATA = json.load(data)
VIOSCREEN_FOOD_CONSUMPTION = \
VioscreenFoodConsumption.from_vioscreen(CONS_DATA)
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenFoodConsumptionRepo(t)
obs = r.insert_food_consumption(VIOSCREEN_FOOD_CONSUMPTION)
self.assertEqual(obs, 14742)
def test_insert_eating_patterns_does_not_exist(self):
with Transaction() as t:
with open(get_data_path("eatingpatterns.data")) as data:
EP_DATA = json.load(data)
VIOSCREEN_EATING_PATTERNS = \
VioscreenEatingPatterns.from_vioscreen(EP_DATA[0])
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenEatingPatternsRepo(t)
obs = r.insert_eating_patterns(VIOSCREEN_EATING_PATTERNS)
self.assertEqual(obs, 17)
def test_get_dietary_scores_exists(self):
with Transaction() as t:
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenDietaryScoreRepo(t)
r.insert_dietary_scores([
VIOSCREEN_DIETARY_SCORE,
])
obs = r.get_dietary_scores(VIOSCREEN_DIETARY_SCORE.sessionId)
self.assertEqual(obs, [
VIOSCREEN_DIETARY_SCORE,
])
def _get_session_by_account_details(account_id, source_id, sample_id):
with Transaction() as t:
surv_temp = SurveyTemplateRepo(t)
vio_sess = VioscreenSessionRepo(t)
vio_username = surv_temp.get_vioscreen_id_if_exists(
account_id, source_id, sample_id)
if vio_username is None:
return True, (jsonify(code=404, message="Username not found"), 404)
vioscreen_session = vio_sess.get_sessions_by_username(vio_username)
if vioscreen_session is None:
return True, (jsonify(code=404, message="Session not found"), 404)
return False, vioscreen_session
def test_upsert_session_exists(self):
with Transaction() as t:
r = VioscreenSessionRepo(t)
obs = r.upsert_session(VIOSCREEN_SESSION)
self.assertEqual(obs, True)
# upsert of unmodified should have no change
obs = r.upsert_session(VIOSCREEN_SESSION)
# ...however the ON CONFLICT won't realize nothing
# is different and still report something changed
self.assertEqual(obs, True)
session_modified = copy(VIOSCREEN_SESSION)
session_modified.endDate = _to_dt(2, 1, 1970)
obs = r.upsert_session(session_modified)
self.assertEqual(obs, True)
obs = r.get_session(VIOSCREEN_SESSION.sessionId)
self.assertEqual(obs, session_modified)
def test_get_unfinished_sessions_multiple(self):
with Transaction() as t:
r = VioscreenSessionRepo(t)
cur = t.cursor()
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username for r in obs}, exp)
sess1 = VIOSCREEN_SESSION.copy()
sess1.username = VIOSCREEN_USERNAME1
sess2 = VIOSCREEN_SESSION.copy()
sess2.username = VIOSCREEN_USERNAME1
sess2.sessionId = 'a different sessionid'
obs = r.upsert_session(sess1)
self.assertTrue(obs)
obs = r.upsert_session(sess2)
self.assertTrue(obs)
# our sessions are unfinished
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username for r in obs}, exp)
# set a finished status
sess1.status = 'Finished'
sess1.endDate = _to_dt(1, 1, 1971)
obs = r.upsert_session(sess1)
self.assertTrue(obs)
# one session is finished, and we only actually understand the
# sematics of a single session anyway, so under our current
# operating assumptions, this users FFQ is now complete
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username
for r in obs}, exp - {
VIOSCREEN_USERNAME1,
})
def test_get_food_consumption_exists(self):
with Transaction() as t:
with open(get_data_path("foodconsumption.data")) as data:
CONS_DATA = json.load(data)
VIOSCREEN_FOOD_CONSUMPTION = \
VioscreenFoodConsumption.from_vioscreen(CONS_DATA)
s = VioscreenSessionRepo(t)
s.upsert_session(VIOSCREEN_SESSION)
r = VioscreenFoodConsumptionRepo(t)
r.insert_food_consumption(VIOSCREEN_FOOD_CONSUMPTION)
obs = r.get_food_consumption(VIOSCREEN_FOOD_CONSUMPTION.sessionId)
self.assertEqual(obs.sessionId,
VIOSCREEN_FOOD_CONSUMPTION.sessionId)
obs_comps = sorted(obs.components, key=lambda x: x.description)
exp_comps = sorted(VIOSCREEN_FOOD_CONSUMPTION.components,
key=lambda x: x.description)
for obs_comp, exp_comp in zip(obs_comps, exp_comps):
obs_cs = sorted(obs_comp.data, key=lambda x: x.code)
exp_cs = sorted(obs_comp.data, key=lambda x: x.code)
for obs_c, exp_c in zip(obs_cs, exp_cs):
self.assertEqual(obs_c, exp_c)
def test_get_ffq_status_by_sample(self):
session_copy = VIOSCREEN_SESSION.copy()
session_copy.username = VIOSCREEN_USERNAME1
with Transaction() as t:
r = VioscreenSessionRepo(t)
r.upsert_session(session_copy)
session = r.get_sessions_by_username(VIOSCREEN_USERNAME1)[0]
obs = r.get_ffq_status_by_sample(BARCODE_UUID_NOTIN_REGISTRY)
self.assertEqual(obs, (False, False, None))
session.status = 'Finished'
session.endDate = _to_dt(2, 1, 1970)
r.upsert_session(session)
# enumerate the empirically observed states from vioscreen
# (is_complete, has_taken, exact_status)
obs = r.get_ffq_status_by_sample(BARCODE_UUID_FOR_VIOSESSION)
self.assertEqual(obs, (True, True, 'Finished'))
session.status = 'Started'
session.endDate = None
r.upsert_session(session)
obs = r.get_ffq_status_by_sample(BARCODE_UUID_FOR_VIOSESSION)
self.assertEqual(obs, (False, True, 'Started'))
session.status = 'New'
r.upsert_session(session)
obs = r.get_ffq_status_by_sample(BARCODE_UUID_FOR_VIOSESSION)
self.assertEqual(obs, (False, False, 'New'))
session.status = 'Review'
r.upsert_session(session)
obs = r.get_ffq_status_by_sample(BARCODE_UUID_FOR_VIOSESSION)
self.assertEqual(obs, (False, True, 'Review'))
def test_get_missing_ffqs(self):
with Transaction() as t:
cur = t.cursor()
r = VioscreenSessionRepo(t)
pr = VioscreenPercentEnergyRepo(t)
user1 = VIOSCREEN_SESSION.copy()
user1.username = VIOSCREEN_USERNAME1
r.upsert_session(user1)
# our user is not present as they do not have an enddate or
# finished status
obs = r.get_missing_ffqs()
self.assertNotIn(VIOSCREEN_USERNAME1, {o.username for o in obs})
user1.status = 'Finished'
user1.endDate = _to_dt(1, 1, 1971)
r.upsert_session(user1)
# our finished session does not have ffq data
obs = r.get_missing_ffqs()
self.assertIn(VIOSCREEN_USERNAME1, {o.username for o in obs})
# give our user a "completed" ffq
user1_pe = VIOSCREEN_PERCENT_ENERGY
pr.insert_percent_energy(user1_pe)
obs = r.get_missing_ffqs()
self.assertNotIn(VIOSCREEN_USERNAME1, {o.username for o in obs})
# our users record is finished so we shouldn't get it back
cur.execute("SELECT vio_id FROM ag.vioscreen_registry")
exp = {r[0] for r in cur.fetchall()}
obs = r.get_unfinished_sessions()
self.assertEqual({r.username
for r in obs}, exp - {
VIOSCREEN_USERNAME1,
})
def fetch_ffqs():
MAX_FETCH_SIZE = 100
vio_api = VioscreenAdminAPI(perform_async=False)
# obtain our current unfinished sessions to check
with Transaction() as t:
r = VioscreenSessionRepo(t)
not_represented = r.get_unfinished_sessions()
# collect sessions to update
unable_to_update_session = []
updated_sessions = []
for sess in enumerate(not_represented):
# update session status information
try:
session_detail = vio_api.session_detail(sess)
except: # noqa
unable_to_update_session.append(sess)
else:
updated_sessions.append(session_detail)
# perform the update after we collect everything to reduce teh length of
# time we hold the transaction open
with Transaction() as t:
r = VioscreenSessionRepo(t)
for session_detail in updated_sessions:
r.upsert_session(session_detail)
t.commit()
with Transaction() as t:
vs = VioscreenSessionRepo(t)
ffqs_not_represented = vs.get_missing_ffqs()
# fetch ffq data for sessions we don't yet have it from
failed_ffqs = []
for sess in ffqs_not_represented[:MAX_FETCH_SIZE]:
try:
error, ffq = vio_api.get_ffq(sess.sessionId)
except Exception as e: # noqa
failed_ffqs.append((sess.sessionId, str(e)))
continue
if error:
failed_ffqs.append((sess.sessionId, repr(error)))
else:
# the call to get_ffq is long, and the data from many ffqs is
# large so let's insert as we go
with Transaction() as t:
vs = VioscreenRepo(t)
vs.insert_ffq(ffq)
t.commit()
if len(failed_ffqs) > 0 or len(unable_to_update_session) > 0:
payload = ''.join([
'%s : %s\n' % (repr(s), m)
for s, m in failed_ffqs + unable_to_update_session
])
send_email(SERVER_CONFIG['pester_email'], "pester_daniel", {
"what": "Vioscreen ffq insert failed",
"content": payload
}, EN_US)
def test_upsert_session_does_not_exist(self):
with Transaction() as t:
r = VioscreenSessionRepo(t)
obs = r.upsert_session(VIOSCREEN_SESSION)
self.assertEqual(obs, True)
def per_sample(project, barcodes, strip_sampleid):
summaries = []
with Transaction() as t:
admin_repo = AdminRepo(t)
sample_repo = SampleRepo(t)
template_repo = SurveyTemplateRepo(t)
vs_repo = VioscreenSessionRepo(t)
if project is not None:
project_barcodes = admin_repo.get_project_barcodes(project)
else:
project = 'Unspecified'
if barcodes is None:
barcodes = project_barcodes
for barcode in barcodes:
diag = admin_repo.retrieve_diagnostics_by_barcode(barcode)
if diag is None:
raise NotFound(f"Barcode not found: {barcode}")
sample = diag['sample']
account = diag['account']
source = diag['source']
account_email = None if account is None else account.email
source_email = None
source_type = None if source is None else source.source_type
vio_id = None
if source is not None and source_type == Source.SOURCE_TYPE_HUMAN:
source_email = source.source_data.email
vio_id = template_repo.get_vioscreen_id_if_exists(account.id,
source.id,
sample.id)
# at least one sample has been observed that "is_microsetta",
# described in the barcodes.project_barcode table, but which is
# unexpectedly not present in ag.ag_kit_barcodes
if sample is None:
sample_status = None
sample_site = None
ffq_complete = None
ffq_taken = None
else:
sample_status = sample_repo.get_sample_status(
sample.barcode,
sample._latest_scan_timestamp
)
sample_site = sample.site
ffq_complete, ffq_taken, _ = vs_repo.get_ffq_status_by_sample(
sample.id
)
summary = {
"sampleid": None if strip_sampleid else barcode,
"project": project,
"source-type": source_type,
"site-sampled": sample_site,
"source-email": source_email,
"account-email": account_email,
"vioscreen_username": vio_id,
"ffq-taken": ffq_taken,
"ffq-complete": ffq_complete,
"sample-status": sample_status,
"sample-received": sample_status is not None
}
for status in ["sample-is-valid",
"no-associated-source",
"no-registered-account",
"no-collection-info",
"sample-has-inconsistencies",
"received-unknown-validity"]:
summary[status] = sample_status == status
summaries.append(summary)
return summaries
def update_session_detail():
# The interaction with the API is occuring within a celery task
# and the recommendation from celery is to avoid depending on synchronous
# tasks from within a task. As such, we'll use non-async calls here. See
# http://docs.celeryq.org/en/latest/userguide/tasks.html#task-synchronous-subtasks
# HOWEVER, we could implement a watch and monitor child tasks, but
# not sure if that would be a particular benefit here
vio_api = VioscreenAdminAPI(perform_async=False)
current_task.update_state(state="PROGRESS",
meta={
"completion": 0,
"status": "PROGRESS",
"message": "Gathering unfinished sessions..."
})
# obtain our current unfinished sessions to check
with Transaction() as t:
r = VioscreenSessionRepo(t)
unfinished_sessions = r.get_unfinished_sessions()
failed_sessions = []
n_to_get = len(unfinished_sessions)
n_updated = 0
for idx, sess in enumerate(unfinished_sessions, 1):
updated = []
# Out of caution, we'll wrap the external resource interaction within
# a blanket try/except
try:
if sess.sessionId is None:
# a session requires a mix of information from Vioscreen's
# representation of a user and a session
user_detail = vio_api.user(sess.username)
details = vio_api.sessions(sess.username)
# account for the possibility of a user having multiple
# sessions
updated.extend([
VioscreenSession.from_vioscreen(detail, user_detail)
for detail in details
])
else:
# update our model inplace
update = vio_api.session_detail(sess.sessionId)
if update.status != sess.status:
updated.append(sess.update_from_vioscreen(update))
except Exception as e: # noqa
failed_sessions.append((sess.sessionId, str(e)))
continue
# commit as we go along to avoid holding any individual transaction
# open for a long period
if len(updated) > 0:
with Transaction() as t:
r = VioscreenSessionRepo(t)
for update in updated:
r.upsert_session(update)
t.commit()
n_updated += len(updated)
current_task.update_state(state="PROGRESS",
meta={
"completion": (idx / n_to_get) * 100,
"status": "PROGRESS",
"message": "Gathering session data..."
})
current_task.update_state(state="SUCCESS",
meta={
"completion": n_to_get,
"status": "SUCCESS",
"message": f"{n_updated} sessions updated"
})
if len(failed_sessions) > 0:
# ...and let's make Daniel feel bad about not having a better means to
# log what hopefully never occurs
payload = ''.join(
['%s : %s\n' % (repr(s), m) for s, m in failed_sessions])
send_email(SERVER_CONFIG['pester_email'], "pester_daniel", {
"what": "Vioscreen sessions failed",
"content": payload
}, EN_US)
def test_get_session_exists(self):
with Transaction() as t:
r = VioscreenSessionRepo(t)
r.upsert_session(VIOSCREEN_SESSION)
obs = r.get_session(VIOSCREEN_SESSION.sessionId)
self.assertEqual(obs, VIOSCREEN_SESSION)
def test_get_session_does_not_exist(self):
with Transaction() as t:
r = VioscreenSessionRepo(t)
obs = r.get_session('does not exist')
self.assertEqual(obs, None)