def test_get_unfinished_sessions(self): with Transaction() as t: r = VioscreenSessionRepo(t) cur = t.cursor() cur.execute("SELECT vio_id FROM ag.vioscreen_registry") exp = {r[0] for r in cur.fetchall()} obs = r.get_unfinished_sessions() self.assertEqual({r.username for r in obs}, exp) user1 = VIOSCREEN_SESSION.copy() user1.username = VIOSCREEN_USERNAME1 obs = r.upsert_session(user1) self.assertTrue(obs) # our base session is still unfinished (no end date) cur.execute("SELECT vio_id FROM ag.vioscreen_registry") exp = {r[0] for r in cur.fetchall()} obs = r.get_unfinished_sessions() self.assertEqual({r.username for r in obs}, exp) # set a finished status user1.status = 'Finished' user1.endDate = _to_dt(1, 1, 1971) obs = r.upsert_session(user1) self.assertTrue(obs) # our users record is finished so we shouldn't get it back cur.execute("SELECT vio_id FROM ag.vioscreen_registry") exp = {r[0] for r in cur.fetchall()} obs = r.get_unfinished_sessions() self.assertEqual({r.username for r in obs}, exp - { VIOSCREEN_USERNAME1, })
def test_get_unfinished_sessions_multiple(self): with Transaction() as t: r = VioscreenSessionRepo(t) cur = t.cursor() cur.execute("SELECT vio_id FROM ag.vioscreen_registry") exp = {r[0] for r in cur.fetchall()} obs = r.get_unfinished_sessions() self.assertEqual({r.username for r in obs}, exp) sess1 = VIOSCREEN_SESSION.copy() sess1.username = VIOSCREEN_USERNAME1 sess2 = VIOSCREEN_SESSION.copy() sess2.username = VIOSCREEN_USERNAME1 sess2.sessionId = 'a different sessionid' obs = r.upsert_session(sess1) self.assertTrue(obs) obs = r.upsert_session(sess2) self.assertTrue(obs) # our sessions are unfinished cur.execute("SELECT vio_id FROM ag.vioscreen_registry") exp = {r[0] for r in cur.fetchall()} obs = r.get_unfinished_sessions() self.assertEqual({r.username for r in obs}, exp) # set a finished status sess1.status = 'Finished' sess1.endDate = _to_dt(1, 1, 1971) obs = r.upsert_session(sess1) self.assertTrue(obs) # one session is finished, and we only actually understand the # sematics of a single session anyway, so under our current # operating assumptions, this users FFQ is now complete cur.execute("SELECT vio_id FROM ag.vioscreen_registry") exp = {r[0] for r in cur.fetchall()} obs = r.get_unfinished_sessions() self.assertEqual({r.username for r in obs}, exp - { VIOSCREEN_USERNAME1, })
def test_get_missing_ffqs(self): with Transaction() as t: cur = t.cursor() r = VioscreenSessionRepo(t) pr = VioscreenPercentEnergyRepo(t) user1 = VIOSCREEN_SESSION.copy() user1.username = VIOSCREEN_USERNAME1 r.upsert_session(user1) # our user is not present as they do not have an enddate or # finished status obs = r.get_missing_ffqs() self.assertNotIn(VIOSCREEN_USERNAME1, {o.username for o in obs}) user1.status = 'Finished' user1.endDate = _to_dt(1, 1, 1971) r.upsert_session(user1) # our finished session does not have ffq data obs = r.get_missing_ffqs() self.assertIn(VIOSCREEN_USERNAME1, {o.username for o in obs}) # give our user a "completed" ffq user1_pe = VIOSCREEN_PERCENT_ENERGY pr.insert_percent_energy(user1_pe) obs = r.get_missing_ffqs() self.assertNotIn(VIOSCREEN_USERNAME1, {o.username for o in obs}) # our users record is finished so we shouldn't get it back cur.execute("SELECT vio_id FROM ag.vioscreen_registry") exp = {r[0] for r in cur.fetchall()} obs = r.get_unfinished_sessions() self.assertEqual({r.username for r in obs}, exp - { VIOSCREEN_USERNAME1, })
def fetch_ffqs(): MAX_FETCH_SIZE = 100 vio_api = VioscreenAdminAPI(perform_async=False) # obtain our current unfinished sessions to check with Transaction() as t: r = VioscreenSessionRepo(t) not_represented = r.get_unfinished_sessions() # collect sessions to update unable_to_update_session = [] updated_sessions = [] for sess in enumerate(not_represented): # update session status information try: session_detail = vio_api.session_detail(sess) except: # noqa unable_to_update_session.append(sess) else: updated_sessions.append(session_detail) # perform the update after we collect everything to reduce teh length of # time we hold the transaction open with Transaction() as t: r = VioscreenSessionRepo(t) for session_detail in updated_sessions: r.upsert_session(session_detail) t.commit() with Transaction() as t: vs = VioscreenSessionRepo(t) ffqs_not_represented = vs.get_missing_ffqs() # fetch ffq data for sessions we don't yet have it from failed_ffqs = [] for sess in ffqs_not_represented[:MAX_FETCH_SIZE]: try: error, ffq = vio_api.get_ffq(sess.sessionId) except Exception as e: # noqa failed_ffqs.append((sess.sessionId, str(e))) continue if error: failed_ffqs.append((sess.sessionId, repr(error))) else: # the call to get_ffq is long, and the data from many ffqs is # large so let's insert as we go with Transaction() as t: vs = VioscreenRepo(t) vs.insert_ffq(ffq) t.commit() if len(failed_ffqs) > 0 or len(unable_to_update_session) > 0: payload = ''.join([ '%s : %s\n' % (repr(s), m) for s, m in failed_ffqs + unable_to_update_session ]) send_email(SERVER_CONFIG['pester_email'], "pester_daniel", { "what": "Vioscreen ffq insert failed", "content": payload }, EN_US)
def update_session_detail(): # The interaction with the API is occuring within a celery task # and the recommendation from celery is to avoid depending on synchronous # tasks from within a task. As such, we'll use non-async calls here. See # http://docs.celeryq.org/en/latest/userguide/tasks.html#task-synchronous-subtasks # HOWEVER, we could implement a watch and monitor child tasks, but # not sure if that would be a particular benefit here vio_api = VioscreenAdminAPI(perform_async=False) current_task.update_state(state="PROGRESS", meta={ "completion": 0, "status": "PROGRESS", "message": "Gathering unfinished sessions..." }) # obtain our current unfinished sessions to check with Transaction() as t: r = VioscreenSessionRepo(t) unfinished_sessions = r.get_unfinished_sessions() failed_sessions = [] n_to_get = len(unfinished_sessions) n_updated = 0 for idx, sess in enumerate(unfinished_sessions, 1): updated = [] # Out of caution, we'll wrap the external resource interaction within # a blanket try/except try: if sess.sessionId is None: # a session requires a mix of information from Vioscreen's # representation of a user and a session user_detail = vio_api.user(sess.username) details = vio_api.sessions(sess.username) # account for the possibility of a user having multiple # sessions updated.extend([ VioscreenSession.from_vioscreen(detail, user_detail) for detail in details ]) else: # update our model inplace update = vio_api.session_detail(sess.sessionId) if update.status != sess.status: updated.append(sess.update_from_vioscreen(update)) except Exception as e: # noqa failed_sessions.append((sess.sessionId, str(e))) continue # commit as we go along to avoid holding any individual transaction # open for a long period if len(updated) > 0: with Transaction() as t: r = VioscreenSessionRepo(t) for update in updated: r.upsert_session(update) t.commit() n_updated += len(updated) current_task.update_state(state="PROGRESS", meta={ "completion": (idx / n_to_get) * 100, "status": "PROGRESS", "message": "Gathering session data..." }) current_task.update_state(state="SUCCESS", meta={ "completion": n_to_get, "status": "SUCCESS", "message": f"{n_updated} sessions updated" }) if len(failed_sessions) > 0: # ...and let's make Daniel feel bad about not having a better means to # log what hopefully never occurs payload = ''.join( ['%s : %s\n' % (repr(s), m) for s, m in failed_sessions]) send_email(SERVER_CONFIG['pester_email'], "pester_daniel", { "what": "Vioscreen sessions failed", "content": payload }, EN_US)