def _do_update_for_organization(organization_id, submission_date, person_file):
"""
deferred task: creates EhrReceipt and updates ParticipantSummary objects from a person.csv file
"""
updated_datetime = datetime.datetime.combine(submission_date,
datetime.datetime.min.time())
org_dao = OrganizationDao()
summary_dao = ParticipantSummaryDao()
receipt_dao = EhrReceiptDao()
org_external_id = organization_id.upper()
org = org_dao.get_by_external_id(org_external_id)
if org is None:
LOG.info("Organization not found with external_id: {}".format(
org_external_id))
receipt = EhrReceipt(organizationId=org.organizationId,
receiptTime=updated_datetime)
receipt_dao.insert(receipt)
for participant_id in _get_participant_ids_from_person_file(person_file):
summary = summary_dao.get(participant_id)
if summary is None:
LOG.info("Participant not found with participant_id: {}".format(
participant_id))
continue
summary_dao.update_ehr_status(summary, updated_datetime)
summary_dao.update(summary)
def setUp(self, **kwargs):
super(MetricsEhrApiTestBase, self).setUp(use_mysql=True, **kwargs)
self.dao = ParticipantDao()
self.ps_dao = ParticipantSummaryDao()
self.ehr_receipt_dao = EhrReceiptDao()
self.ps = ParticipantSummary()
self.calendar_dao = CalendarDao()
self.site_dao = SiteDao()
self.hpo_dao = HPODao()
self.org_dao = OrganizationDao()
self.hpo_test = self._make_hpo(hpoId=TEST_HPO_ID,
name=TEST_HPO_NAME,
displayName='Test',
organizationType=OrganizationType.UNSET)
self.hpo_foo = self._make_hpo(hpoId=10, name='FOO', displayName='Foo')
self.hpo_bar = self._make_hpo(hpoId=11, name='BAR', displayName='Bar')
self.org_foo_a = self._make_org(organizationId=10,
externalId='FOO_A',
displayName='Foo A',
hpoId=self.hpo_foo.hpoId)
self.org_bar_a = self._make_org(organizationId=11,
externalId='BAR_A',
displayName='Bar A',
hpoId=self.hpo_bar.hpoId)
def setUp(self, with_data=True, use_mysql=True):
super(EhrReceiptDaoTest, self).setUp(with_data=with_data,
use_mysql=use_mysql)
self.setup_fake()
self.calendar_dao = CalendarDao()
self.org_dao = OrganizationDao()
self.hpo_dao = HPODao()
self.participant_dao = ParticipantDao()
self.summary_dao = ParticipantSummaryDao()
self.ehr_receipt_dao = EhrReceiptDao()
self._setup_initial_data()
def update_organizations_from_job(job):
organization_dao = OrganizationDao()
receipt_dao = EhrReceiptDao()
for page in job:
for row in page:
org = organization_dao.get_by_external_id(row.org_id)
if org:
try:
receipt_time = datetime_as_naive_utc(row.person_upload_time)
except TypeError:
continue
receipt_dao.get_or_create(
insert_if_created=True,
organizationId=org.organizationId,
receiptTime=receipt_time
)
def setUp(self, **kwargs):
super(UpdateEhrStatusUpdatesTestCase, self).setUp(use_mysql=True,
**kwargs)
self.hpo_dao = HPODao()
self.org_dao = OrganizationDao()
self.participant_dao = ParticipantDao()
self.summary_dao = ParticipantSummaryDao()
self.ehr_receipt_dao = EhrReceiptDao()
self.hpo_foo = self._make_hpo(int_id=10, string_id='hpo_foo')
self.hpo_bar = self._make_hpo(int_id=11, string_id='hpo_bar')
self.org_foo_a = self._make_org(hpo=self.hpo_foo,
int_id=10,
external_id='FOO_A')
self.org_foo_b = self._make_org(hpo=self.hpo_foo,
int_id=11,
external_id='FOO_B')
self.org_bar_a = self._make_org(hpo=self.hpo_bar,
int_id=12,
external_id='BAR_A')
self.participants = [
self._make_participant(hpo=self.hpo_foo,
org=self.org_foo_a,
int_id=11),
self._make_participant(hpo=self.hpo_foo,
org=self.org_foo_b,
int_id=12),
self._make_participant(hpo=self.hpo_bar,
org=self.org_bar_a,
int_id=13),
self._make_participant(hpo=self.hpo_bar,
org=self.org_bar_a,
int_id=14),
]
def __init__(self):
super(MetricsEhrService, self).__init__(ParticipantSummary,
backup=True)
self.ehr_receipt_dao = EhrReceiptDao()
class MetricsEhrService(BaseDao):
def __init__(self):
super(MetricsEhrService, self).__init__(ParticipantSummary,
backup=True)
self.ehr_receipt_dao = EhrReceiptDao()
def _get_organization_ids_from_hpo_ids(self, hpo_ids):
query = (sqlalchemy.select([Organization.organizationId
]).where(Organization.hpoId.in_(hpo_ids)))
with self.session() as session:
result = session.execute(query)
return list(row[0] for row in result)
def get_current_metrics(self, organization_ids=None, hpo_ids=None):
now = clock.CLOCK.now()
if organization_ids is None and hpo_ids is not None:
organization_ids = self._get_organization_ids_from_hpo_ids(hpo_ids)
with self.session() as session:
ehr_data = self._get_current_ehr_data_with_session(
session, organization_ids)
org_data = self._get_organization_metrics_data_with_session(
session, now, organization_ids)
return {
'date': now,
'metrics': ehr_data,
'organization_metrics': org_data,
}
def _get_current_ehr_query(self, organization_ids=None):
where_conditions = [
ParticipantSummary.withdrawalStatus ==
WithdrawalStatus.NOT_WITHDRAWN,
Participant.isGhostId.isnot(True),
ParticipantSummary.consentForStudyEnrollment ==
QuestionnaireStatus.SUBMITTED,
ParticipantSummary.consentForElectronicHealthRecords ==
QuestionnaireStatus.SUBMITTED
]
if organization_ids:
where_conditions.append(
ParticipantSummary.organizationId.in_(organization_ids))
query = (sqlalchemy.select([
sqlalchemy.func.count().label('consented_count'),
sqlalchemy.func.cast(
sqlalchemy.func.sum(
sqlalchemy.func.if_(
ParticipantSummary.ehrStatus == EhrStatus.PRESENT, 1,
0)), sqlalchemy.Integer).label('received_count'),
]).select_from(
sqlalchemy.join(
ParticipantSummary, Participant,
ParticipantSummary.participantId ==
Participant.participantId)).where(
reduce(sqlalchemy.and_, where_conditions)))
return query
def get_current_ehr_data(self, organization_ids=None):
with self.session() as session:
return self._get_current_ehr_data_with_session(
session, organization_ids)
def _get_current_ehr_data_with_session(self,
session,
organization_ids=None):
query = self._get_current_ehr_query(organization_ids)
cursor = session.execute(query)
consented_count, received_count = cursor.fetchone()
return {
'EHR_CONSENTED': consented_count,
'EHR_RECEIVED': received_count,
}
def get_metrics(self,
start_date,
end_date,
organization_ids=None,
hpo_ids=None,
interval=INTERVAL_WEEK):
if organization_ids is None and hpo_ids is not None:
organization_ids = self._get_organization_ids_from_hpo_ids(hpo_ids)
return {
'metrics_over_time':
self._get_metrics_over_time_data(start_date, end_date, interval,
organization_ids),
'organization_metrics':
self.get_organization_metrics_data(end_date, organization_ids),
}
def _get_metrics_over_time_data(self,
start_date,
end_date,
interval,
organization_ids=None):
"""
combines `Active Organization Counts Over Time` and `EHR Consented vs EHR Received Over Time`
"""
active_organization_metrics = self.get_organizations_active_over_time_data(
start_date, end_date, interval, organization_ids)
active_organization_metrics_by_date = {
result['date']: result['metrics']
for result in active_organization_metrics
}
participant_ehr_metrics = self.get_participant_ehr_metrics_over_time_data(
start_date, end_date, interval, organization_ids)
participant_ehr_metrics_by_date = {
result['date']: result['metrics']
for result in participant_ehr_metrics
}
return [{
'date':
date_key,
'metrics':
dict(active_organization_metrics_by_date[date_key],
**participant_ehr_metrics_by_date[date_key])
} for date_key in active_organization_metrics_by_date.keys()]
def get_organizations_active_over_time_data(self,
start_date,
end_date,
interval,
organization_ids=None):
"""
Count of organizations that have uploaded EHR over time
"""
active_organization_counts = self.ehr_receipt_dao.get_active_organization_counts_in_interval(
start_date, end_date, interval, organization_ids)
return [{
'date': result['start_date'],
'metrics': {
'ORGANIZATIONS_ACTIVE': result['active_organization_count'],
}
} for result in active_organization_counts]
def get_participant_ehr_metrics_over_time_data(self,
start_date,
end_date,
interval,
organization_ids=None):
"""
EHR Consented vs EHR Received over time
"""
interval_query = CalendarDao.get_interval_query(start=start_date,
end=end_date,
interval_key=interval)
ehr_query = self._get_participant_ehr_metrics_over_time_query(
interval_query, organization_ids)
with self.session() as session:
ehr_cursor = session.execute(ehr_query)
return [{
'date': row_dict['start_date'],
'metrics': {
'EHR_CONSENTED': row_dict['consented_count'],
'EHR_RECEIVED': row_dict['received_count'],
}
} for row_dict in
[dict(zip(ehr_cursor.keys(), row)) for row in ehr_cursor]]
@staticmethod
def _get_participant_ehr_metrics_over_time_query(interval_query,
organization_ids=None):
common_subquery_where_arg = ((ParticipantSummary.withdrawalStatus
== WithdrawalStatus.NOT_WITHDRAWN)
& Participant.isGhostId.isnot(True))
if organization_ids:
common_subquery_where_arg &= ParticipantSummary.organizationId.in_(
organization_ids)
base_subquery = (sqlalchemy.select(
[sqlalchemy.func.count()]).select_from(
sqlalchemy.join(
Participant, ParticipantSummary,
Participant.participantId == ParticipantSummary.
participantId)).where(common_subquery_where_arg))
subquery_consented_count = base_subquery.where(
(ParticipantSummary.consentForElectronicHealthRecords ==
QuestionnaireStatus.SUBMITTED)
& (ParticipantSummary.consentForElectronicHealthRecordsTime <=
interval_query.c.start_date))
subquery_received_count = (base_subquery.where(
(ParticipantSummary.ehrStatus == EhrStatus.PRESENT)
& (ParticipantSummary.ehrReceiptTime <= interval_query.c.start_date
)))
return (sqlalchemy.select([
interval_query.c.start_date,
subquery_consented_count.label('consented_count'),
subquery_received_count.label('received_count'),
]).order_by(interval_query.c.start_date))
def get_organization_metrics_data(self, end_date, organization_ids=None):
"""
Get organization participant status metrics as of end_date
"""
with self.session() as session:
return self._get_organization_metrics_data_with_session(
session, end_date, organization_ids)
def _get_organization_metrics_data_with_session(self,
session,
end_date,
organization_ids=None):
q = self._get_organization_metrics_query(end_date, organization_ids)
cursor = session.execute(q)
return {
row_dict['organization_id']: row_dict
for row_dict in [dict(zip(cursor.keys(), row)) for row in cursor]
}
@staticmethod
def _get_organization_metrics_query(cutoff_date, organization_ids=None):
def make_sum_bool_field(condition_expression):
return sqlalchemy.func.cast(
sqlalchemy.func.sum(
sqlalchemy.func.if_(condition_expression, 1, 0)),
sqlalchemy.Integer)
ppi_baseline_module_count = len(
config.getSettingList(config.BASELINE_PPI_QUESTIONNAIRE_FIELDS))
can_be_included = ((ParticipantSummary.withdrawalStatus
== WithdrawalStatus.NOT_WITHDRAWN)
& Participant.isGhostId.isnot(True))
# condition expression components
was_signed_up = Participant.signUpTime <= cutoff_date
had_consented_for_study = (
(ParticipantSummary.consentForStudyEnrollment
== QuestionnaireStatus.SUBMITTED)
&
(ParticipantSummary.consentForStudyEnrollmentTime <= cutoff_date))
had_consented_for_ehr = (
(ParticipantSummary.consentForElectronicHealthRecords
== QuestionnaireStatus.SUBMITTED)
& (ParticipantSummary.consentForElectronicHealthRecordsTime <=
cutoff_date))
had_completed_ppi = (ParticipantSummary.numCompletedBaselinePPIModules
>= ppi_baseline_module_count)
had_physical_measurements = ParticipantSummary.physicalMeasurementsFinalizedTime <= cutoff_date
had_biosample = ParticipantSummary.biospecimenOrderTime <= cutoff_date
had_ehr_receipt = ((ParticipantSummary.ehrStatus == EhrStatus.PRESENT)
&
(ParticipantSummary.ehrReceiptTime <= cutoff_date))
# condition expressions
was_participant = can_be_included & was_signed_up
was_primary = was_participant & had_consented_for_study
was_ehr_consented = was_primary & had_consented_for_ehr
was_core = (was_ehr_consented
& had_completed_ppi
& had_physical_measurements
& had_biosample)
had_ehr_data = was_ehr_consented & had_ehr_receipt
# build query
receipt_subquery = (sqlalchemy.select([
EhrReceipt.organizationId.label('organization_id'),
sqlalchemy.func.max(
EhrReceipt.receiptTime).label('ehr_receipt_time')
]).select_from(EhrReceipt).group_by(
EhrReceipt.organizationId).alias('receipt_subquery'))
fields = [
Organization.externalId.label('organization_id'),
Organization.displayName.label('organization_name'),
make_sum_bool_field(was_participant).label('total_participants'),
make_sum_bool_field(was_primary).label('total_primary_consented'),
make_sum_bool_field(was_ehr_consented).label(
'total_ehr_consented'),
make_sum_bool_field(was_core).label('total_core_participants'),
make_sum_bool_field(had_ehr_data).label('total_ehr_data_received'),
sqlalchemy.func.date(receipt_subquery.c.ehr_receipt_time).label(
'last_ehr_submission_date'),
]
joined_tables = sqlalchemy.join(
sqlalchemy.join(
sqlalchemy.outerjoin(
Organization,
receipt_subquery,
Organization.organizationId ==
receipt_subquery.c.organization_id,
), ParticipantSummary, ParticipantSummary.organizationId ==
Organization.organizationId), Participant,
Participant.participantId == ParticipantSummary.participantId)
query = (sqlalchemy.select(fields).select_from(joined_tables).group_by(
Organization.organizationId))
if organization_ids:
query = query.where(
Organization.organizationId.in_(organization_ids))
return query
class MetricsEhrApiTestBase(FlaskTestBase):
def setUp(self, **kwargs):
super(MetricsEhrApiTestBase, self).setUp(use_mysql=True, **kwargs)
self.dao = ParticipantDao()
self.ps_dao = ParticipantSummaryDao()
self.ehr_receipt_dao = EhrReceiptDao()
self.ps = ParticipantSummary()
self.calendar_dao = CalendarDao()
self.site_dao = SiteDao()
self.hpo_dao = HPODao()
self.org_dao = OrganizationDao()
self.hpo_test = self._make_hpo(hpoId=TEST_HPO_ID,
name=TEST_HPO_NAME,
displayName='Test',
organizationType=OrganizationType.UNSET)
self.hpo_foo = self._make_hpo(hpoId=10, name='FOO', displayName='Foo')
self.hpo_bar = self._make_hpo(hpoId=11, name='BAR', displayName='Bar')
self.org_foo_a = self._make_org(organizationId=10,
externalId='FOO_A',
displayName='Foo A',
hpoId=self.hpo_foo.hpoId)
self.org_bar_a = self._make_org(organizationId=11,
externalId='BAR_A',
displayName='Bar A',
hpoId=self.hpo_bar.hpoId)
def _make_hpo(self, **kwargs):
hpo = HPO(**kwargs)
self.hpo_dao.insert(hpo)
return hpo
def _make_org(self, **kwargs):
org = Organization(**kwargs)
self.org_dao.insert(org)
return org
def _make_participant(self,
participant,
first_name=None,
last_name=None,
hpo=None,
organization=None,
unconsented=False,
time_int=None,
time_study=None,
time_mem=None,
time_fp=None,
time_fp_stored=None,
gender_id=None,
dob=None,
state_id=None):
"""
Create a participant in a transient test database.
Note: copied from ParticipantCountsOverTimeApiTest
:param participant: Participant object
:param first_name: First name
:param last_name: Last name
:param time_int: Time that participant fulfilled INTERESTED criteria
:param time_mem: Time that participant fulfilled MEMBER criteria
:param time_fp: Time that participant fulfilled FULL_PARTICIPANT criteria
:return: Participant object
"""
participant.hpoId = hpo.hpoId
participant.organizationId = organization.organizationId
if unconsented is True:
enrollment_status = None
elif time_mem is None:
enrollment_status = EnrollmentStatus.INTERESTED
elif time_fp is None:
enrollment_status = EnrollmentStatus.MEMBER
else:
enrollment_status = EnrollmentStatus.FULL_PARTICIPANT
with FakeClock(time_int):
self.dao.insert(participant)
participant.providerLink = make_primary_provider_link_for_name(
hpo.name)
with FakeClock(time_mem):
self.dao.update(participant)
if enrollment_status is None:
return None
summary = self.participant_summary(participant)
if first_name:
summary.firstName = first_name
if last_name:
summary.lastName = last_name
if gender_id:
summary.genderIdentityId = gender_id
if dob:
summary.dateOfBirth = dob
else:
summary.dateOfBirth = datetime.date(1978, 10, 10)
if state_id:
summary.stateId = state_id
summary.enrollmentStatus = enrollment_status
summary.enrollmentStatusMemberTime = time_mem
summary.enrollmentStatusCoreOrderedSampleTime = time_fp
summary.enrollmentStatusCoreStoredSampleTime = time_fp_stored
summary.hpoId = hpo.hpoId
summary.organizationId = organization.organizationId
if time_study is not None:
with FakeClock(time_mem):
summary.consentForStudyEnrollment = QuestionnaireStatus.SUBMITTED
summary.consentForStudyEnrollmentTime = time_study
if time_mem is not None:
with FakeClock(time_mem):
summary.consentForElectronicHealthRecords = QuestionnaireStatus.SUBMITTED
summary.consentForElectronicHealthRecordsTime = time_mem
if time_fp is not None:
with FakeClock(time_fp):
if not summary.consentForElectronicHealthRecords:
summary.consentForElectronicHealthRecords = QuestionnaireStatus.SUBMITTED
summary.consentForElectronicHealthRecordsTime = time_fp
summary.questionnaireOnTheBasicsTime = time_fp
summary.questionnaireOnLifestyleTime = time_fp
summary.questionnaireOnOverallHealthTime = time_fp
summary.physicalMeasurementsFinalizedTime = time_fp
summary.physicalMeasurementsTime = time_fp
summary.sampleOrderStatus1ED04Time = time_fp
summary.sampleOrderStatus1SALTime = time_fp
summary.sampleStatus1ED04Time = time_fp
summary.sampleStatus1SALTime = time_fp
summary.biospecimenOrderTime = time_fp
summary.numCompletedBaselinePPIModules = REQUIRED_PPI_MODULE_COUNT
self.ps_dao.insert(summary)
return summary
def _update_ehr(self, participant_summary, update_time):
receipt = EhrReceipt(organizationId=participant_summary.organizationId,
receiptTime=update_time)
self.ehr_receipt_dao.insert(receipt)
self.ps_dao.update_ehr_status(participant_summary, update_time)
self.ps_dao.update(participant_summary)
class EhrReceiptDaoTest(SqlTestBase):
def setUp(self, with_data=True, use_mysql=True):
super(EhrReceiptDaoTest, self).setUp(with_data=with_data,
use_mysql=use_mysql)
self.setup_fake()
self.calendar_dao = CalendarDao()
self.org_dao = OrganizationDao()
self.hpo_dao = HPODao()
self.participant_dao = ParticipantDao()
self.summary_dao = ParticipantSummaryDao()
self.ehr_receipt_dao = EhrReceiptDao()
self._setup_initial_data()
@staticmethod
def _iter_dates_in_range(start, end):
current = start
while current <= end:
yield current
current += datetime.timedelta(days=1)
def _fill_calendar_range(self, start, end):
for date in self._iter_dates_in_range(start, end):
self.calendar_dao.insert(Calendar(day=date))
def _make_hpo(self, int_id, string_id):
hpo = HPO(hpoId=int_id, name=string_id)
self.hpo_dao.insert(hpo)
return hpo
def _make_org(self, **kwargs):
org = Organization(**kwargs)
self.org_dao.insert(org)
return org
def _make_participant(self, org, int_id):
participant = self._participant_with_defaults(participantId=int_id,
biobankId=int_id)
participant.hpoId = org.hpoId
participant.organizationId = org.organizationId
self.participant_dao.insert(participant)
summary = self.participant_summary(participant)
summary.hpoId = participant.hpoId
summary.organizationId = participant.organizationId
self.summary_dao.insert(summary)
return participant, summary
def _update_ehr(self, participant_summary, update_time):
self.summary_dao.update_ehr_status(participant_summary, update_time)
self.summary_dao.update(participant_summary)
def _save_ehr_receipt(self, org, receipt_time):
receipt = EhrReceipt(organizationId=org.organizationId,
receiptTime=receipt_time)
self.ehr_receipt_dao.insert(receipt)
def _setup_initial_data(self):
self.hpo_foo = self._make_hpo(int_id=10, string_id='hpo_foo')
self.hpo_bar = self._make_hpo(int_id=11, string_id='hpo_bar')
self.org_foo_a = self._make_org(organizationId=10,
externalId='FOO_A',
displayName='Foo A',
hpoId=self.hpo_foo.hpoId)
self.org_bar_a = self._make_org(organizationId=11,
externalId='BAR_A',
displayName='Bar A',
hpoId=self.hpo_bar.hpoId)
participant_and_summary_pairs = [
self._make_participant(org=self.org_foo_a, int_id=11),
self._make_participant(org=self.org_foo_a, int_id=12),
self._make_participant(org=self.org_bar_a, int_id=13),
self._make_participant(org=self.org_bar_a, int_id=14),
]
self.participants = {
participant.participantId: participant
for participant, summary in participant_and_summary_pairs
}
self.summaries = {
participant.participantId: summary
for participant, summary in participant_and_summary_pairs
}
def test_get_active_organization_counts_in_interval_day(self):
self._fill_calendar_range(datetime.date(2019, 1, 1),
datetime.date(2019, 3, 1))
self._save_ehr_receipt(org=self.org_foo_a,
receipt_time=datetime.datetime(2019, 2, 2))
self._save_ehr_receipt(org=self.org_bar_a,
receipt_time=datetime.datetime(2019, 2, 2))
self._save_ehr_receipt(org=self.org_foo_a,
receipt_time=datetime.datetime(2019, 2, 4))
results = self.ehr_receipt_dao.get_active_organization_counts_in_interval(
start_date=datetime.datetime(2019, 2, 1),
end_date=datetime.datetime(2019, 2, 7),
interval=INTERVAL_DAY)
self.assertEqual([(r['start_date'], r['active_organization_count'])
for r in results], [
(datetime.date(2019, 2, 1), 0L),
(datetime.date(2019, 2, 2), 2L),
(datetime.date(2019, 2, 3), 0L),
(datetime.date(2019, 2, 4), 1L),
(datetime.date(2019, 2, 5), 0L),
(datetime.date(2019, 2, 6), 0L),
(datetime.date(2019, 2, 7), 0L),
])
def test_get_active_organization_counts_in_interval_week(self):
self._fill_calendar_range(datetime.date(2019, 1, 1),
datetime.date(2019, 3, 1))
self._save_ehr_receipt(org=self.org_foo_a,
receipt_time=datetime.datetime(2019, 2, 4))
self._save_ehr_receipt(org=self.org_bar_a,
receipt_time=datetime.datetime(2019, 2, 4))
self._save_ehr_receipt(org=self.org_foo_a,
receipt_time=datetime.datetime(2019, 2, 18))
results = self.ehr_receipt_dao.get_active_organization_counts_in_interval(
start_date=datetime.datetime(2019, 2, 1),
end_date=datetime.datetime(2019, 3, 1),
interval=INTERVAL_WEEK)
self.assertEqual([(r['start_date'], r['active_organization_count'])
for r in results], [
(datetime.date(2019, 1, 27), 0L),
(datetime.date(2019, 2, 3), 2L),
(datetime.date(2019, 2, 10), 0L),
(datetime.date(2019, 2, 17), 1L),
(datetime.date(2019, 2, 24), 0L),
])
def test_get_active_organization_counts_in_interval_month(self):
self._fill_calendar_range(datetime.date(2018, 12, 1),
datetime.date(2019, 7, 1))
self._save_ehr_receipt(org=self.org_foo_a,
receipt_time=datetime.datetime(2019, 2, 1))
self._save_ehr_receipt(org=self.org_bar_a,
receipt_time=datetime.datetime(2019, 2, 1))
self._save_ehr_receipt(org=self.org_foo_a,
receipt_time=datetime.datetime(2019, 4, 1))
results = self.ehr_receipt_dao.get_active_organization_counts_in_interval(
start_date=datetime.datetime(2019, 1, 1),
end_date=datetime.datetime(2019, 5, 1),
interval=INTERVAL_MONTH)
self.assertEqual([(r['start_date'], r['active_organization_count'])
for r in results], [
(datetime.date(2019, 1, 1), 0L),
(datetime.date(2019, 2, 1), 2L),
(datetime.date(2019, 3, 1), 0L),
(datetime.date(2019, 4, 1), 1L),
(datetime.date(2019, 5, 1), 0L),
])
def test_get_active_organization_counts_in_interval_quarter(self):
self._fill_calendar_range(datetime.date(2018, 12, 1),
datetime.date(2020, 1, 1))
self._save_ehr_receipt(org=self.org_foo_a,
receipt_time=datetime.datetime(2019, 5, 1))
self._save_ehr_receipt(org=self.org_bar_a,
receipt_time=datetime.datetime(2019, 5, 1))
self._save_ehr_receipt(org=self.org_foo_a,
receipt_time=datetime.datetime(2019, 11, 1))
results = self.ehr_receipt_dao.get_active_organization_counts_in_interval(
start_date=datetime.datetime(2019, 1, 1),
end_date=datetime.datetime(2020, 1, 1),
interval=INTERVAL_QUARTER)
self.assertEqual([(r['start_date'], r['active_organization_count'])
for r in results], [
(datetime.date(2019, 1, 1), 0L),
(datetime.date(2019, 4, 1), 2L),
(datetime.date(2019, 7, 1), 0L),
(datetime.date(2019, 10, 1), 1L),
(datetime.date(2020, 1, 1), 0L),
])
class UpdateEhrStatusUpdatesTestCase(SqlTestBase):
def setUp(self, **kwargs):
super(UpdateEhrStatusUpdatesTestCase, self).setUp(use_mysql=True,
**kwargs)
self.hpo_dao = HPODao()
self.org_dao = OrganizationDao()
self.participant_dao = ParticipantDao()
self.summary_dao = ParticipantSummaryDao()
self.ehr_receipt_dao = EhrReceiptDao()
self.hpo_foo = self._make_hpo(int_id=10, string_id='hpo_foo')
self.hpo_bar = self._make_hpo(int_id=11, string_id='hpo_bar')
self.org_foo_a = self._make_org(hpo=self.hpo_foo,
int_id=10,
external_id='FOO_A')
self.org_foo_b = self._make_org(hpo=self.hpo_foo,
int_id=11,
external_id='FOO_B')
self.org_bar_a = self._make_org(hpo=self.hpo_bar,
int_id=12,
external_id='BAR_A')
self.participants = [
self._make_participant(hpo=self.hpo_foo,
org=self.org_foo_a,
int_id=11),
self._make_participant(hpo=self.hpo_foo,
org=self.org_foo_b,
int_id=12),
self._make_participant(hpo=self.hpo_bar,
org=self.org_bar_a,
int_id=13),
self._make_participant(hpo=self.hpo_bar,
org=self.org_bar_a,
int_id=14),
]
def _make_hpo(self, int_id, string_id):
hpo = HPO(hpoId=int_id, name=string_id)
self.hpo_dao.insert(hpo)
return hpo
def _make_org(self, hpo, int_id, external_id):
org = Organization(organizationId=int_id,
externalId=external_id,
displayName='SOME ORG',
hpoId=hpo.hpoId)
self.org_dao.insert(org)
return org
def _make_participant(self, hpo, org, int_id):
participant = self._participant_with_defaults(participantId=int_id,
biobankId=int_id)
participant.hpoId = hpo.hpoId
participant.organizationId = org.organizationId
self.participant_dao.insert(participant)
summary = self.participant_summary(participant)
self.summary_dao.insert(summary)
return participant, summary
# Mock BigQuery result types
EhrUpdatePidRow = collections.namedtuple('EhrUpdatePidRow', [
'person_id',
])
TableCountsRow = collections.namedtuple('TableCountsRow', [
'org_id',
'person_upload_time',
])
@mock.patch('offline.update_ehr_status.update_organizations_from_job')
@mock.patch(
'offline.update_ehr_status.update_participant_summaries_from_job')
@mock.patch('offline.update_ehr_status.make_update_organizations_job')
@mock.patch(
'offline.update_ehr_status.make_update_participant_summaries_job')
def test_skips_when_no_job(self, mock_summary_job, mock_organization_job,
mock_update_summaries,
mock_update_organizations):
mock_summary_job.return_value = None
mock_organization_job.return_value = None
with FakeClock(datetime.datetime(2019, 1, 1)):
offline.update_ehr_status.update_ehr_status()
self.assertFalse(mock_update_summaries.called)
self.assertFalse(mock_update_organizations.called)
@mock.patch('offline.update_ehr_status.make_update_organizations_job')
@mock.patch(
'offline.update_ehr_status.make_update_participant_summaries_job')
def test_updates_participant_summaries(self, mock_summary_job,
mock_organization_job):
mock_summary_job.return_value.__iter__.return_value = [[
self.EhrUpdatePidRow(11),
]]
mock_organization_job.return_value.__iter__.return_value = []
with FakeClock(datetime.datetime(2019, 1, 1)):
offline.update_ehr_status.update_ehr_status()
mock_summary_job.return_value.__iter__.return_value = [[
self.EhrUpdatePidRow(11),
self.EhrUpdatePidRow(12),
]]
mock_organization_job.return_value.__iter__.return_value = []
with FakeClock(datetime.datetime(2019, 1, 2)):
offline.update_ehr_status.update_ehr_status()
summary = self.summary_dao.get(11)
self.assertEqual(summary.ehrStatus, EhrStatus.PRESENT)
self.assertEqual(summary.ehrReceiptTime, datetime.datetime(2019, 1, 1))
self.assertEqual(summary.ehrUpdateTime, datetime.datetime(2019, 1, 2))
summary = self.summary_dao.get(12)
self.assertEqual(summary.ehrStatus, EhrStatus.PRESENT)
self.assertEqual(summary.ehrReceiptTime, datetime.datetime(2019, 1, 2))
self.assertEqual(summary.ehrUpdateTime, datetime.datetime(2019, 1, 2))
@mock.patch('offline.update_ehr_status.make_update_organizations_job')
@mock.patch(
'offline.update_ehr_status.make_update_participant_summaries_job')
def test_creates_receipts(self, mock_summary_job, mock_organization_job):
mock_summary_job.return_value.__iter__.return_value = []
mock_organization_job.return_value.__iter__.return_value = [
[
self.TableCountsRow(org_id='FOO_A',
person_upload_time=datetime.datetime(
2019, 1, 1).replace(tzinfo=pytz.UTC)),
],
]
with FakeClock(datetime.datetime(2019, 1, 1)):
offline.update_ehr_status.update_ehr_status()
foo_a_receipts = self.ehr_receipt_dao.get_by_organization_id(
self.org_foo_a.organizationId)
self.assertEqual(len(foo_a_receipts), 1)
self.assertEqual(foo_a_receipts[0].receiptTime,
datetime.datetime(2019, 1, 1))
foo_b_receipts = self.ehr_receipt_dao.get_by_organization_id(
self.org_foo_b.organizationId)
self.assertEqual(len(foo_b_receipts), 0)
mock_summary_job.return_value.__iter__.return_value = []
mock_organization_job.return_value.__iter__.return_value = [
[
self.TableCountsRow(org_id='FOO_A',
person_upload_time=datetime.datetime(
2019, 1, 1).replace(tzinfo=pytz.UTC)),
self.TableCountsRow(org_id='FOO_A',
person_upload_time=datetime.datetime(
2019, 1, 2).replace(tzinfo=pytz.UTC)),
self.TableCountsRow(org_id='FOO_B',
person_upload_time=datetime.datetime(
2019, 1, 2).replace(tzinfo=pytz.UTC)),
],
]
with FakeClock(datetime.datetime(2019, 1, 2)):
offline.update_ehr_status.update_ehr_status()
foo_a_receipts = self.ehr_receipt_dao.get_by_organization_id(
self.org_foo_a.organizationId)
self.assertEqual(len(foo_a_receipts), 2)
self.assertEqual(foo_a_receipts[0].receiptTime,
datetime.datetime(2019, 1, 1))
self.assertEqual(foo_a_receipts[1].receiptTime,
datetime.datetime(2019, 1, 2))
foo_b_receipts = self.ehr_receipt_dao.get_by_organization_id(
self.org_foo_b.organizationId)
self.assertEqual(len(foo_b_receipts), 1)
self.assertEqual(foo_b_receipts[0].receiptTime,
datetime.datetime(2019, 1, 2))
@mock.patch('offline.update_ehr_status.make_update_organizations_job')
@mock.patch(
'offline.update_ehr_status.make_update_participant_summaries_job')
def test_ignores_bad_data(self, mock_summary_job, mock_organization_job):
invalid_participant_id = -1
mock_summary_job.return_value.__iter__.return_value = [[
self.EhrUpdatePidRow(invalid_participant_id),
]]
mock_organization_job.return_value.__iter__.return_value = [
[
self.TableCountsRow(
org_id='FOO_A',
person_upload_time="an invalid date string"),
self.TableCountsRow(org_id='AN_ORG_THAT_DOESNT_EXIST',
person_upload_time=datetime.datetime(
2019, 1, 1).replace(tzinfo=pytz.UTC)),
self.TableCountsRow(org_id='AN_ORG_THAT_DOESNT_EXIST',
person_upload_time=None),
],
]
with FakeClock(datetime.datetime(2019, 1, 1)):
offline.update_ehr_status.update_ehr_status()
foo_a_receipts = self.ehr_receipt_dao.get_all()
self.assertEqual(len(foo_a_receipts), 0)
