def _do_update_for_organization(organization_id, submission_date, person_file): """ deferred task: creates EhrReceipt and updates ParticipantSummary objects from a person.csv file """ updated_datetime = datetime.datetime.combine(submission_date, datetime.datetime.min.time()) org_dao = OrganizationDao() summary_dao = ParticipantSummaryDao() receipt_dao = EhrReceiptDao() org_external_id = organization_id.upper() org = org_dao.get_by_external_id(org_external_id) if org is None: LOG.info("Organization not found with external_id: {}".format( org_external_id)) receipt = EhrReceipt(organizationId=org.organizationId, receiptTime=updated_datetime) receipt_dao.insert(receipt) for participant_id in _get_participant_ids_from_person_file(person_file): summary = summary_dao.get(participant_id) if summary is None: LOG.info("Participant not found with participant_id: {}".format( participant_id)) continue summary_dao.update_ehr_status(summary, updated_datetime) summary_dao.update(summary)
def setUp(self, **kwargs): super(MetricsEhrApiTestBase, self).setUp(use_mysql=True, **kwargs) self.dao = ParticipantDao() self.ps_dao = ParticipantSummaryDao() self.ehr_receipt_dao = EhrReceiptDao() self.ps = ParticipantSummary() self.calendar_dao = CalendarDao() self.site_dao = SiteDao() self.hpo_dao = HPODao() self.org_dao = OrganizationDao() self.hpo_test = self._make_hpo(hpoId=TEST_HPO_ID, name=TEST_HPO_NAME, displayName='Test', organizationType=OrganizationType.UNSET) self.hpo_foo = self._make_hpo(hpoId=10, name='FOO', displayName='Foo') self.hpo_bar = self._make_hpo(hpoId=11, name='BAR', displayName='Bar') self.org_foo_a = self._make_org(organizationId=10, externalId='FOO_A', displayName='Foo A', hpoId=self.hpo_foo.hpoId) self.org_bar_a = self._make_org(organizationId=11, externalId='BAR_A', displayName='Bar A', hpoId=self.hpo_bar.hpoId)
def setUp(self, with_data=True, use_mysql=True): super(EhrReceiptDaoTest, self).setUp(with_data=with_data, use_mysql=use_mysql) self.setup_fake() self.calendar_dao = CalendarDao() self.org_dao = OrganizationDao() self.hpo_dao = HPODao() self.participant_dao = ParticipantDao() self.summary_dao = ParticipantSummaryDao() self.ehr_receipt_dao = EhrReceiptDao() self._setup_initial_data()
def update_organizations_from_job(job): organization_dao = OrganizationDao() receipt_dao = EhrReceiptDao() for page in job: for row in page: org = organization_dao.get_by_external_id(row.org_id) if org: try: receipt_time = datetime_as_naive_utc(row.person_upload_time) except TypeError: continue receipt_dao.get_or_create( insert_if_created=True, organizationId=org.organizationId, receiptTime=receipt_time )
def setUp(self, **kwargs): super(UpdateEhrStatusUpdatesTestCase, self).setUp(use_mysql=True, **kwargs) self.hpo_dao = HPODao() self.org_dao = OrganizationDao() self.participant_dao = ParticipantDao() self.summary_dao = ParticipantSummaryDao() self.ehr_receipt_dao = EhrReceiptDao() self.hpo_foo = self._make_hpo(int_id=10, string_id='hpo_foo') self.hpo_bar = self._make_hpo(int_id=11, string_id='hpo_bar') self.org_foo_a = self._make_org(hpo=self.hpo_foo, int_id=10, external_id='FOO_A') self.org_foo_b = self._make_org(hpo=self.hpo_foo, int_id=11, external_id='FOO_B') self.org_bar_a = self._make_org(hpo=self.hpo_bar, int_id=12, external_id='BAR_A') self.participants = [ self._make_participant(hpo=self.hpo_foo, org=self.org_foo_a, int_id=11), self._make_participant(hpo=self.hpo_foo, org=self.org_foo_b, int_id=12), self._make_participant(hpo=self.hpo_bar, org=self.org_bar_a, int_id=13), self._make_participant(hpo=self.hpo_bar, org=self.org_bar_a, int_id=14), ]
def __init__(self): super(MetricsEhrService, self).__init__(ParticipantSummary, backup=True) self.ehr_receipt_dao = EhrReceiptDao()
class MetricsEhrService(BaseDao): def __init__(self): super(MetricsEhrService, self).__init__(ParticipantSummary, backup=True) self.ehr_receipt_dao = EhrReceiptDao() def _get_organization_ids_from_hpo_ids(self, hpo_ids): query = (sqlalchemy.select([Organization.organizationId ]).where(Organization.hpoId.in_(hpo_ids))) with self.session() as session: result = session.execute(query) return list(row[0] for row in result) def get_current_metrics(self, organization_ids=None, hpo_ids=None): now = clock.CLOCK.now() if organization_ids is None and hpo_ids is not None: organization_ids = self._get_organization_ids_from_hpo_ids(hpo_ids) with self.session() as session: ehr_data = self._get_current_ehr_data_with_session( session, organization_ids) org_data = self._get_organization_metrics_data_with_session( session, now, organization_ids) return { 'date': now, 'metrics': ehr_data, 'organization_metrics': org_data, } def _get_current_ehr_query(self, organization_ids=None): where_conditions = [ ParticipantSummary.withdrawalStatus == WithdrawalStatus.NOT_WITHDRAWN, Participant.isGhostId.isnot(True), ParticipantSummary.consentForStudyEnrollment == QuestionnaireStatus.SUBMITTED, ParticipantSummary.consentForElectronicHealthRecords == QuestionnaireStatus.SUBMITTED ] if organization_ids: where_conditions.append( ParticipantSummary.organizationId.in_(organization_ids)) query = (sqlalchemy.select([ sqlalchemy.func.count().label('consented_count'), sqlalchemy.func.cast( sqlalchemy.func.sum( sqlalchemy.func.if_( ParticipantSummary.ehrStatus == EhrStatus.PRESENT, 1, 0)), sqlalchemy.Integer).label('received_count'), ]).select_from( sqlalchemy.join( ParticipantSummary, Participant, ParticipantSummary.participantId == Participant.participantId)).where( reduce(sqlalchemy.and_, where_conditions))) return query def get_current_ehr_data(self, organization_ids=None): with self.session() as session: return self._get_current_ehr_data_with_session( session, organization_ids) def _get_current_ehr_data_with_session(self, session, organization_ids=None): query = self._get_current_ehr_query(organization_ids) cursor = session.execute(query) consented_count, received_count = cursor.fetchone() return { 'EHR_CONSENTED': consented_count, 'EHR_RECEIVED': received_count, } def get_metrics(self, start_date, end_date, organization_ids=None, hpo_ids=None, interval=INTERVAL_WEEK): if organization_ids is None and hpo_ids is not None: organization_ids = self._get_organization_ids_from_hpo_ids(hpo_ids) return { 'metrics_over_time': self._get_metrics_over_time_data(start_date, end_date, interval, organization_ids), 'organization_metrics': self.get_organization_metrics_data(end_date, organization_ids), } def _get_metrics_over_time_data(self, start_date, end_date, interval, organization_ids=None): """ combines `Active Organization Counts Over Time` and `EHR Consented vs EHR Received Over Time` """ active_organization_metrics = self.get_organizations_active_over_time_data( start_date, end_date, interval, organization_ids) active_organization_metrics_by_date = { result['date']: result['metrics'] for result in active_organization_metrics } participant_ehr_metrics = self.get_participant_ehr_metrics_over_time_data( start_date, end_date, interval, organization_ids) participant_ehr_metrics_by_date = { result['date']: result['metrics'] for result in participant_ehr_metrics } return [{ 'date': date_key, 'metrics': dict(active_organization_metrics_by_date[date_key], **participant_ehr_metrics_by_date[date_key]) } for date_key in active_organization_metrics_by_date.keys()] def get_organizations_active_over_time_data(self, start_date, end_date, interval, organization_ids=None): """ Count of organizations that have uploaded EHR over time """ active_organization_counts = self.ehr_receipt_dao.get_active_organization_counts_in_interval( start_date, end_date, interval, organization_ids) return [{ 'date': result['start_date'], 'metrics': { 'ORGANIZATIONS_ACTIVE': result['active_organization_count'], } } for result in active_organization_counts] def get_participant_ehr_metrics_over_time_data(self, start_date, end_date, interval, organization_ids=None): """ EHR Consented vs EHR Received over time """ interval_query = CalendarDao.get_interval_query(start=start_date, end=end_date, interval_key=interval) ehr_query = self._get_participant_ehr_metrics_over_time_query( interval_query, organization_ids) with self.session() as session: ehr_cursor = session.execute(ehr_query) return [{ 'date': row_dict['start_date'], 'metrics': { 'EHR_CONSENTED': row_dict['consented_count'], 'EHR_RECEIVED': row_dict['received_count'], } } for row_dict in [dict(zip(ehr_cursor.keys(), row)) for row in ehr_cursor]] @staticmethod def _get_participant_ehr_metrics_over_time_query(interval_query, organization_ids=None): common_subquery_where_arg = ((ParticipantSummary.withdrawalStatus == WithdrawalStatus.NOT_WITHDRAWN) & Participant.isGhostId.isnot(True)) if organization_ids: common_subquery_where_arg &= ParticipantSummary.organizationId.in_( organization_ids) base_subquery = (sqlalchemy.select( [sqlalchemy.func.count()]).select_from( sqlalchemy.join( Participant, ParticipantSummary, Participant.participantId == ParticipantSummary. participantId)).where(common_subquery_where_arg)) subquery_consented_count = base_subquery.where( (ParticipantSummary.consentForElectronicHealthRecords == QuestionnaireStatus.SUBMITTED) & (ParticipantSummary.consentForElectronicHealthRecordsTime <= interval_query.c.start_date)) subquery_received_count = (base_subquery.where( (ParticipantSummary.ehrStatus == EhrStatus.PRESENT) & (ParticipantSummary.ehrReceiptTime <= interval_query.c.start_date ))) return (sqlalchemy.select([ interval_query.c.start_date, subquery_consented_count.label('consented_count'), subquery_received_count.label('received_count'), ]).order_by(interval_query.c.start_date)) def get_organization_metrics_data(self, end_date, organization_ids=None): """ Get organization participant status metrics as of end_date """ with self.session() as session: return self._get_organization_metrics_data_with_session( session, end_date, organization_ids) def _get_organization_metrics_data_with_session(self, session, end_date, organization_ids=None): q = self._get_organization_metrics_query(end_date, organization_ids) cursor = session.execute(q) return { row_dict['organization_id']: row_dict for row_dict in [dict(zip(cursor.keys(), row)) for row in cursor] } @staticmethod def _get_organization_metrics_query(cutoff_date, organization_ids=None): def make_sum_bool_field(condition_expression): return sqlalchemy.func.cast( sqlalchemy.func.sum( sqlalchemy.func.if_(condition_expression, 1, 0)), sqlalchemy.Integer) ppi_baseline_module_count = len( config.getSettingList(config.BASELINE_PPI_QUESTIONNAIRE_FIELDS)) can_be_included = ((ParticipantSummary.withdrawalStatus == WithdrawalStatus.NOT_WITHDRAWN) & Participant.isGhostId.isnot(True)) # condition expression components was_signed_up = Participant.signUpTime <= cutoff_date had_consented_for_study = ( (ParticipantSummary.consentForStudyEnrollment == QuestionnaireStatus.SUBMITTED) & (ParticipantSummary.consentForStudyEnrollmentTime <= cutoff_date)) had_consented_for_ehr = ( (ParticipantSummary.consentForElectronicHealthRecords == QuestionnaireStatus.SUBMITTED) & (ParticipantSummary.consentForElectronicHealthRecordsTime <= cutoff_date)) had_completed_ppi = (ParticipantSummary.numCompletedBaselinePPIModules >= ppi_baseline_module_count) had_physical_measurements = ParticipantSummary.physicalMeasurementsFinalizedTime <= cutoff_date had_biosample = ParticipantSummary.biospecimenOrderTime <= cutoff_date had_ehr_receipt = ((ParticipantSummary.ehrStatus == EhrStatus.PRESENT) & (ParticipantSummary.ehrReceiptTime <= cutoff_date)) # condition expressions was_participant = can_be_included & was_signed_up was_primary = was_participant & had_consented_for_study was_ehr_consented = was_primary & had_consented_for_ehr was_core = (was_ehr_consented & had_completed_ppi & had_physical_measurements & had_biosample) had_ehr_data = was_ehr_consented & had_ehr_receipt # build query receipt_subquery = (sqlalchemy.select([ EhrReceipt.organizationId.label('organization_id'), sqlalchemy.func.max( EhrReceipt.receiptTime).label('ehr_receipt_time') ]).select_from(EhrReceipt).group_by( EhrReceipt.organizationId).alias('receipt_subquery')) fields = [ Organization.externalId.label('organization_id'), Organization.displayName.label('organization_name'), make_sum_bool_field(was_participant).label('total_participants'), make_sum_bool_field(was_primary).label('total_primary_consented'), make_sum_bool_field(was_ehr_consented).label( 'total_ehr_consented'), make_sum_bool_field(was_core).label('total_core_participants'), make_sum_bool_field(had_ehr_data).label('total_ehr_data_received'), sqlalchemy.func.date(receipt_subquery.c.ehr_receipt_time).label( 'last_ehr_submission_date'), ] joined_tables = sqlalchemy.join( sqlalchemy.join( sqlalchemy.outerjoin( Organization, receipt_subquery, Organization.organizationId == receipt_subquery.c.organization_id, ), ParticipantSummary, ParticipantSummary.organizationId == Organization.organizationId), Participant, Participant.participantId == ParticipantSummary.participantId) query = (sqlalchemy.select(fields).select_from(joined_tables).group_by( Organization.organizationId)) if organization_ids: query = query.where( Organization.organizationId.in_(organization_ids)) return query
class MetricsEhrApiTestBase(FlaskTestBase): def setUp(self, **kwargs): super(MetricsEhrApiTestBase, self).setUp(use_mysql=True, **kwargs) self.dao = ParticipantDao() self.ps_dao = ParticipantSummaryDao() self.ehr_receipt_dao = EhrReceiptDao() self.ps = ParticipantSummary() self.calendar_dao = CalendarDao() self.site_dao = SiteDao() self.hpo_dao = HPODao() self.org_dao = OrganizationDao() self.hpo_test = self._make_hpo(hpoId=TEST_HPO_ID, name=TEST_HPO_NAME, displayName='Test', organizationType=OrganizationType.UNSET) self.hpo_foo = self._make_hpo(hpoId=10, name='FOO', displayName='Foo') self.hpo_bar = self._make_hpo(hpoId=11, name='BAR', displayName='Bar') self.org_foo_a = self._make_org(organizationId=10, externalId='FOO_A', displayName='Foo A', hpoId=self.hpo_foo.hpoId) self.org_bar_a = self._make_org(organizationId=11, externalId='BAR_A', displayName='Bar A', hpoId=self.hpo_bar.hpoId) def _make_hpo(self, **kwargs): hpo = HPO(**kwargs) self.hpo_dao.insert(hpo) return hpo def _make_org(self, **kwargs): org = Organization(**kwargs) self.org_dao.insert(org) return org def _make_participant(self, participant, first_name=None, last_name=None, hpo=None, organization=None, unconsented=False, time_int=None, time_study=None, time_mem=None, time_fp=None, time_fp_stored=None, gender_id=None, dob=None, state_id=None): """ Create a participant in a transient test database. Note: copied from ParticipantCountsOverTimeApiTest :param participant: Participant object :param first_name: First name :param last_name: Last name :param time_int: Time that participant fulfilled INTERESTED criteria :param time_mem: Time that participant fulfilled MEMBER criteria :param time_fp: Time that participant fulfilled FULL_PARTICIPANT criteria :return: Participant object """ participant.hpoId = hpo.hpoId participant.organizationId = organization.organizationId if unconsented is True: enrollment_status = None elif time_mem is None: enrollment_status = EnrollmentStatus.INTERESTED elif time_fp is None: enrollment_status = EnrollmentStatus.MEMBER else: enrollment_status = EnrollmentStatus.FULL_PARTICIPANT with FakeClock(time_int): self.dao.insert(participant) participant.providerLink = make_primary_provider_link_for_name( hpo.name) with FakeClock(time_mem): self.dao.update(participant) if enrollment_status is None: return None summary = self.participant_summary(participant) if first_name: summary.firstName = first_name if last_name: summary.lastName = last_name if gender_id: summary.genderIdentityId = gender_id if dob: summary.dateOfBirth = dob else: summary.dateOfBirth = datetime.date(1978, 10, 10) if state_id: summary.stateId = state_id summary.enrollmentStatus = enrollment_status summary.enrollmentStatusMemberTime = time_mem summary.enrollmentStatusCoreOrderedSampleTime = time_fp summary.enrollmentStatusCoreStoredSampleTime = time_fp_stored summary.hpoId = hpo.hpoId summary.organizationId = organization.organizationId if time_study is not None: with FakeClock(time_mem): summary.consentForStudyEnrollment = QuestionnaireStatus.SUBMITTED summary.consentForStudyEnrollmentTime = time_study if time_mem is not None: with FakeClock(time_mem): summary.consentForElectronicHealthRecords = QuestionnaireStatus.SUBMITTED summary.consentForElectronicHealthRecordsTime = time_mem if time_fp is not None: with FakeClock(time_fp): if not summary.consentForElectronicHealthRecords: summary.consentForElectronicHealthRecords = QuestionnaireStatus.SUBMITTED summary.consentForElectronicHealthRecordsTime = time_fp summary.questionnaireOnTheBasicsTime = time_fp summary.questionnaireOnLifestyleTime = time_fp summary.questionnaireOnOverallHealthTime = time_fp summary.physicalMeasurementsFinalizedTime = time_fp summary.physicalMeasurementsTime = time_fp summary.sampleOrderStatus1ED04Time = time_fp summary.sampleOrderStatus1SALTime = time_fp summary.sampleStatus1ED04Time = time_fp summary.sampleStatus1SALTime = time_fp summary.biospecimenOrderTime = time_fp summary.numCompletedBaselinePPIModules = REQUIRED_PPI_MODULE_COUNT self.ps_dao.insert(summary) return summary def _update_ehr(self, participant_summary, update_time): receipt = EhrReceipt(organizationId=participant_summary.organizationId, receiptTime=update_time) self.ehr_receipt_dao.insert(receipt) self.ps_dao.update_ehr_status(participant_summary, update_time) self.ps_dao.update(participant_summary)
class EhrReceiptDaoTest(SqlTestBase): def setUp(self, with_data=True, use_mysql=True): super(EhrReceiptDaoTest, self).setUp(with_data=with_data, use_mysql=use_mysql) self.setup_fake() self.calendar_dao = CalendarDao() self.org_dao = OrganizationDao() self.hpo_dao = HPODao() self.participant_dao = ParticipantDao() self.summary_dao = ParticipantSummaryDao() self.ehr_receipt_dao = EhrReceiptDao() self._setup_initial_data() @staticmethod def _iter_dates_in_range(start, end): current = start while current <= end: yield current current += datetime.timedelta(days=1) def _fill_calendar_range(self, start, end): for date in self._iter_dates_in_range(start, end): self.calendar_dao.insert(Calendar(day=date)) def _make_hpo(self, int_id, string_id): hpo = HPO(hpoId=int_id, name=string_id) self.hpo_dao.insert(hpo) return hpo def _make_org(self, **kwargs): org = Organization(**kwargs) self.org_dao.insert(org) return org def _make_participant(self, org, int_id): participant = self._participant_with_defaults(participantId=int_id, biobankId=int_id) participant.hpoId = org.hpoId participant.organizationId = org.organizationId self.participant_dao.insert(participant) summary = self.participant_summary(participant) summary.hpoId = participant.hpoId summary.organizationId = participant.organizationId self.summary_dao.insert(summary) return participant, summary def _update_ehr(self, participant_summary, update_time): self.summary_dao.update_ehr_status(participant_summary, update_time) self.summary_dao.update(participant_summary) def _save_ehr_receipt(self, org, receipt_time): receipt = EhrReceipt(organizationId=org.organizationId, receiptTime=receipt_time) self.ehr_receipt_dao.insert(receipt) def _setup_initial_data(self): self.hpo_foo = self._make_hpo(int_id=10, string_id='hpo_foo') self.hpo_bar = self._make_hpo(int_id=11, string_id='hpo_bar') self.org_foo_a = self._make_org(organizationId=10, externalId='FOO_A', displayName='Foo A', hpoId=self.hpo_foo.hpoId) self.org_bar_a = self._make_org(organizationId=11, externalId='BAR_A', displayName='Bar A', hpoId=self.hpo_bar.hpoId) participant_and_summary_pairs = [ self._make_participant(org=self.org_foo_a, int_id=11), self._make_participant(org=self.org_foo_a, int_id=12), self._make_participant(org=self.org_bar_a, int_id=13), self._make_participant(org=self.org_bar_a, int_id=14), ] self.participants = { participant.participantId: participant for participant, summary in participant_and_summary_pairs } self.summaries = { participant.participantId: summary for participant, summary in participant_and_summary_pairs } def test_get_active_organization_counts_in_interval_day(self): self._fill_calendar_range(datetime.date(2019, 1, 1), datetime.date(2019, 3, 1)) self._save_ehr_receipt(org=self.org_foo_a, receipt_time=datetime.datetime(2019, 2, 2)) self._save_ehr_receipt(org=self.org_bar_a, receipt_time=datetime.datetime(2019, 2, 2)) self._save_ehr_receipt(org=self.org_foo_a, receipt_time=datetime.datetime(2019, 2, 4)) results = self.ehr_receipt_dao.get_active_organization_counts_in_interval( start_date=datetime.datetime(2019, 2, 1), end_date=datetime.datetime(2019, 2, 7), interval=INTERVAL_DAY) self.assertEqual([(r['start_date'], r['active_organization_count']) for r in results], [ (datetime.date(2019, 2, 1), 0L), (datetime.date(2019, 2, 2), 2L), (datetime.date(2019, 2, 3), 0L), (datetime.date(2019, 2, 4), 1L), (datetime.date(2019, 2, 5), 0L), (datetime.date(2019, 2, 6), 0L), (datetime.date(2019, 2, 7), 0L), ]) def test_get_active_organization_counts_in_interval_week(self): self._fill_calendar_range(datetime.date(2019, 1, 1), datetime.date(2019, 3, 1)) self._save_ehr_receipt(org=self.org_foo_a, receipt_time=datetime.datetime(2019, 2, 4)) self._save_ehr_receipt(org=self.org_bar_a, receipt_time=datetime.datetime(2019, 2, 4)) self._save_ehr_receipt(org=self.org_foo_a, receipt_time=datetime.datetime(2019, 2, 18)) results = self.ehr_receipt_dao.get_active_organization_counts_in_interval( start_date=datetime.datetime(2019, 2, 1), end_date=datetime.datetime(2019, 3, 1), interval=INTERVAL_WEEK) self.assertEqual([(r['start_date'], r['active_organization_count']) for r in results], [ (datetime.date(2019, 1, 27), 0L), (datetime.date(2019, 2, 3), 2L), (datetime.date(2019, 2, 10), 0L), (datetime.date(2019, 2, 17), 1L), (datetime.date(2019, 2, 24), 0L), ]) def test_get_active_organization_counts_in_interval_month(self): self._fill_calendar_range(datetime.date(2018, 12, 1), datetime.date(2019, 7, 1)) self._save_ehr_receipt(org=self.org_foo_a, receipt_time=datetime.datetime(2019, 2, 1)) self._save_ehr_receipt(org=self.org_bar_a, receipt_time=datetime.datetime(2019, 2, 1)) self._save_ehr_receipt(org=self.org_foo_a, receipt_time=datetime.datetime(2019, 4, 1)) results = self.ehr_receipt_dao.get_active_organization_counts_in_interval( start_date=datetime.datetime(2019, 1, 1), end_date=datetime.datetime(2019, 5, 1), interval=INTERVAL_MONTH) self.assertEqual([(r['start_date'], r['active_organization_count']) for r in results], [ (datetime.date(2019, 1, 1), 0L), (datetime.date(2019, 2, 1), 2L), (datetime.date(2019, 3, 1), 0L), (datetime.date(2019, 4, 1), 1L), (datetime.date(2019, 5, 1), 0L), ]) def test_get_active_organization_counts_in_interval_quarter(self): self._fill_calendar_range(datetime.date(2018, 12, 1), datetime.date(2020, 1, 1)) self._save_ehr_receipt(org=self.org_foo_a, receipt_time=datetime.datetime(2019, 5, 1)) self._save_ehr_receipt(org=self.org_bar_a, receipt_time=datetime.datetime(2019, 5, 1)) self._save_ehr_receipt(org=self.org_foo_a, receipt_time=datetime.datetime(2019, 11, 1)) results = self.ehr_receipt_dao.get_active_organization_counts_in_interval( start_date=datetime.datetime(2019, 1, 1), end_date=datetime.datetime(2020, 1, 1), interval=INTERVAL_QUARTER) self.assertEqual([(r['start_date'], r['active_organization_count']) for r in results], [ (datetime.date(2019, 1, 1), 0L), (datetime.date(2019, 4, 1), 2L), (datetime.date(2019, 7, 1), 0L), (datetime.date(2019, 10, 1), 1L), (datetime.date(2020, 1, 1), 0L), ])
class UpdateEhrStatusUpdatesTestCase(SqlTestBase): def setUp(self, **kwargs): super(UpdateEhrStatusUpdatesTestCase, self).setUp(use_mysql=True, **kwargs) self.hpo_dao = HPODao() self.org_dao = OrganizationDao() self.participant_dao = ParticipantDao() self.summary_dao = ParticipantSummaryDao() self.ehr_receipt_dao = EhrReceiptDao() self.hpo_foo = self._make_hpo(int_id=10, string_id='hpo_foo') self.hpo_bar = self._make_hpo(int_id=11, string_id='hpo_bar') self.org_foo_a = self._make_org(hpo=self.hpo_foo, int_id=10, external_id='FOO_A') self.org_foo_b = self._make_org(hpo=self.hpo_foo, int_id=11, external_id='FOO_B') self.org_bar_a = self._make_org(hpo=self.hpo_bar, int_id=12, external_id='BAR_A') self.participants = [ self._make_participant(hpo=self.hpo_foo, org=self.org_foo_a, int_id=11), self._make_participant(hpo=self.hpo_foo, org=self.org_foo_b, int_id=12), self._make_participant(hpo=self.hpo_bar, org=self.org_bar_a, int_id=13), self._make_participant(hpo=self.hpo_bar, org=self.org_bar_a, int_id=14), ] def _make_hpo(self, int_id, string_id): hpo = HPO(hpoId=int_id, name=string_id) self.hpo_dao.insert(hpo) return hpo def _make_org(self, hpo, int_id, external_id): org = Organization(organizationId=int_id, externalId=external_id, displayName='SOME ORG', hpoId=hpo.hpoId) self.org_dao.insert(org) return org def _make_participant(self, hpo, org, int_id): participant = self._participant_with_defaults(participantId=int_id, biobankId=int_id) participant.hpoId = hpo.hpoId participant.organizationId = org.organizationId self.participant_dao.insert(participant) summary = self.participant_summary(participant) self.summary_dao.insert(summary) return participant, summary # Mock BigQuery result types EhrUpdatePidRow = collections.namedtuple('EhrUpdatePidRow', [ 'person_id', ]) TableCountsRow = collections.namedtuple('TableCountsRow', [ 'org_id', 'person_upload_time', ]) @mock.patch('offline.update_ehr_status.update_organizations_from_job') @mock.patch( 'offline.update_ehr_status.update_participant_summaries_from_job') @mock.patch('offline.update_ehr_status.make_update_organizations_job') @mock.patch( 'offline.update_ehr_status.make_update_participant_summaries_job') def test_skips_when_no_job(self, mock_summary_job, mock_organization_job, mock_update_summaries, mock_update_organizations): mock_summary_job.return_value = None mock_organization_job.return_value = None with FakeClock(datetime.datetime(2019, 1, 1)): offline.update_ehr_status.update_ehr_status() self.assertFalse(mock_update_summaries.called) self.assertFalse(mock_update_organizations.called) @mock.patch('offline.update_ehr_status.make_update_organizations_job') @mock.patch( 'offline.update_ehr_status.make_update_participant_summaries_job') def test_updates_participant_summaries(self, mock_summary_job, mock_organization_job): mock_summary_job.return_value.__iter__.return_value = [[ self.EhrUpdatePidRow(11), ]] mock_organization_job.return_value.__iter__.return_value = [] with FakeClock(datetime.datetime(2019, 1, 1)): offline.update_ehr_status.update_ehr_status() mock_summary_job.return_value.__iter__.return_value = [[ self.EhrUpdatePidRow(11), self.EhrUpdatePidRow(12), ]] mock_organization_job.return_value.__iter__.return_value = [] with FakeClock(datetime.datetime(2019, 1, 2)): offline.update_ehr_status.update_ehr_status() summary = self.summary_dao.get(11) self.assertEqual(summary.ehrStatus, EhrStatus.PRESENT) self.assertEqual(summary.ehrReceiptTime, datetime.datetime(2019, 1, 1)) self.assertEqual(summary.ehrUpdateTime, datetime.datetime(2019, 1, 2)) summary = self.summary_dao.get(12) self.assertEqual(summary.ehrStatus, EhrStatus.PRESENT) self.assertEqual(summary.ehrReceiptTime, datetime.datetime(2019, 1, 2)) self.assertEqual(summary.ehrUpdateTime, datetime.datetime(2019, 1, 2)) @mock.patch('offline.update_ehr_status.make_update_organizations_job') @mock.patch( 'offline.update_ehr_status.make_update_participant_summaries_job') def test_creates_receipts(self, mock_summary_job, mock_organization_job): mock_summary_job.return_value.__iter__.return_value = [] mock_organization_job.return_value.__iter__.return_value = [ [ self.TableCountsRow(org_id='FOO_A', person_upload_time=datetime.datetime( 2019, 1, 1).replace(tzinfo=pytz.UTC)), ], ] with FakeClock(datetime.datetime(2019, 1, 1)): offline.update_ehr_status.update_ehr_status() foo_a_receipts = self.ehr_receipt_dao.get_by_organization_id( self.org_foo_a.organizationId) self.assertEqual(len(foo_a_receipts), 1) self.assertEqual(foo_a_receipts[0].receiptTime, datetime.datetime(2019, 1, 1)) foo_b_receipts = self.ehr_receipt_dao.get_by_organization_id( self.org_foo_b.organizationId) self.assertEqual(len(foo_b_receipts), 0) mock_summary_job.return_value.__iter__.return_value = [] mock_organization_job.return_value.__iter__.return_value = [ [ self.TableCountsRow(org_id='FOO_A', person_upload_time=datetime.datetime( 2019, 1, 1).replace(tzinfo=pytz.UTC)), self.TableCountsRow(org_id='FOO_A', person_upload_time=datetime.datetime( 2019, 1, 2).replace(tzinfo=pytz.UTC)), self.TableCountsRow(org_id='FOO_B', person_upload_time=datetime.datetime( 2019, 1, 2).replace(tzinfo=pytz.UTC)), ], ] with FakeClock(datetime.datetime(2019, 1, 2)): offline.update_ehr_status.update_ehr_status() foo_a_receipts = self.ehr_receipt_dao.get_by_organization_id( self.org_foo_a.organizationId) self.assertEqual(len(foo_a_receipts), 2) self.assertEqual(foo_a_receipts[0].receiptTime, datetime.datetime(2019, 1, 1)) self.assertEqual(foo_a_receipts[1].receiptTime, datetime.datetime(2019, 1, 2)) foo_b_receipts = self.ehr_receipt_dao.get_by_organization_id( self.org_foo_b.organizationId) self.assertEqual(len(foo_b_receipts), 1) self.assertEqual(foo_b_receipts[0].receiptTime, datetime.datetime(2019, 1, 2)) @mock.patch('offline.update_ehr_status.make_update_organizations_job') @mock.patch( 'offline.update_ehr_status.make_update_participant_summaries_job') def test_ignores_bad_data(self, mock_summary_job, mock_organization_job): invalid_participant_id = -1 mock_summary_job.return_value.__iter__.return_value = [[ self.EhrUpdatePidRow(invalid_participant_id), ]] mock_organization_job.return_value.__iter__.return_value = [ [ self.TableCountsRow( org_id='FOO_A', person_upload_time="an invalid date string"), self.TableCountsRow(org_id='AN_ORG_THAT_DOESNT_EXIST', person_upload_time=datetime.datetime( 2019, 1, 1).replace(tzinfo=pytz.UTC)), self.TableCountsRow(org_id='AN_ORG_THAT_DOESNT_EXIST', person_upload_time=None), ], ] with FakeClock(datetime.datetime(2019, 1, 1)): offline.update_ehr_status.update_ehr_status() foo_a_receipts = self.ehr_receipt_dao.get_all() self.assertEqual(len(foo_a_receipts), 0)