def test_get_missing_ffqs(self): with Transaction() as t: cur = t.cursor() r = VioscreenSessionRepo(t) pr = VioscreenPercentEnergyRepo(t) user1 = VIOSCREEN_SESSION.copy() user1.username = VIOSCREEN_USERNAME1 r.upsert_session(user1) # our user is not present as they do not have an enddate or # finished status obs = r.get_missing_ffqs() self.assertNotIn(VIOSCREEN_USERNAME1, {o.username for o in obs}) user1.status = 'Finished' user1.endDate = _to_dt(1, 1, 1971) r.upsert_session(user1) # our finished session does not have ffq data obs = r.get_missing_ffqs() self.assertIn(VIOSCREEN_USERNAME1, {o.username for o in obs}) # give our user a "completed" ffq user1_pe = VIOSCREEN_PERCENT_ENERGY pr.insert_percent_energy(user1_pe) obs = r.get_missing_ffqs() self.assertNotIn(VIOSCREEN_USERNAME1, {o.username for o in obs}) # our users record is finished so we shouldn't get it back cur.execute("SELECT vio_id FROM ag.vioscreen_registry") exp = {r[0] for r in cur.fetchall()} obs = r.get_unfinished_sessions() self.assertEqual({r.username for r in obs}, exp - { VIOSCREEN_USERNAME1, })
def fetch_ffqs(): MAX_FETCH_SIZE = 100 vio_api = VioscreenAdminAPI(perform_async=False) # obtain our current unfinished sessions to check with Transaction() as t: r = VioscreenSessionRepo(t) not_represented = r.get_unfinished_sessions() # collect sessions to update unable_to_update_session = [] updated_sessions = [] for sess in enumerate(not_represented): # update session status information try: session_detail = vio_api.session_detail(sess) except: # noqa unable_to_update_session.append(sess) else: updated_sessions.append(session_detail) # perform the update after we collect everything to reduce teh length of # time we hold the transaction open with Transaction() as t: r = VioscreenSessionRepo(t) for session_detail in updated_sessions: r.upsert_session(session_detail) t.commit() with Transaction() as t: vs = VioscreenSessionRepo(t) ffqs_not_represented = vs.get_missing_ffqs() # fetch ffq data for sessions we don't yet have it from failed_ffqs = [] for sess in ffqs_not_represented[:MAX_FETCH_SIZE]: try: error, ffq = vio_api.get_ffq(sess.sessionId) except Exception as e: # noqa failed_ffqs.append((sess.sessionId, str(e))) continue if error: failed_ffqs.append((sess.sessionId, repr(error))) else: # the call to get_ffq is long, and the data from many ffqs is # large so let's insert as we go with Transaction() as t: vs = VioscreenRepo(t) vs.insert_ffq(ffq) t.commit() if len(failed_ffqs) > 0 or len(unable_to_update_session) > 0: payload = ''.join([ '%s : %s\n' % (repr(s), m) for s, m in failed_ffqs + unable_to_update_session ]) send_email(SERVER_CONFIG['pester_email'], "pester_daniel", { "what": "Vioscreen ffq insert failed", "content": payload }, EN_US)