def test_get_counts_for_unconsented_individuals(self):
# Those who have signed up but not consented should be INTERESTED
p1 = Participant(participantId=1, biobankId=4)
self._insert(p1, 'Alice', 'Aardvark', 'UNSET', unconsented=True, time_int=self.time1)
p2 = Participant(participantId=2, biobankId=5)
self._insert(p2, 'Bob', 'Builder', 'AZ_TUCSON', time_int=self.time1)
p3 = Participant(participantId=3, biobankId=6)
self._insert(p3, 'Chad', 'Caterpillar', 'AZ_TUCSON', time_int=self.time1)
qs = """
bucketSize=1
&stratification=ENROLLMENT_STATUS
&startDate=2017-12-30
&endDate=2018-01-04
"""
qs = ''.join(qs.split()) # Remove all whitespace
response = self.send_get('ParticipantCountsOverTime', query_string=qs)
total_count_day_1 = response[0]['metrics']['INTERESTED']
total_count_day_2 = response[1]['metrics']['INTERESTED']
self.assertEquals(total_count_day_1, 0)
self.assertEquals(total_count_day_2, 3)
def test_get_counts_excluding_interested_participants(self):
# When filtering only for MEMBER, no INTERESTED (neither consented nor unconsented) should be counted
p1 = Participant(participantId=1, biobankId=4)
self._insert(p1, 'Alice', 'Aardvark', 'UNSET', unconsented=True, time_int=self.time1)
p2 = Participant(participantId=2, biobankId=5)
self._insert(p2, 'Bob', 'Builder', 'AZ_TUCSON', time_int=self.time1)
p3 = Participant(participantId=3, biobankId=6)
self._insert(p3, 'Chad', 'Caterpillar', 'AZ_TUCSON', time_int=self.time1, time_mem=self.time1)
qs = """
bucketSize=1
&stratification=ENROLLMENT_STATUS
&startDate=2017-12-30
&endDate=2018-01-04
&enrollmentStatus=MEMBER
"""
qs = ''.join(qs.split()) # Remove all whitespace
response = self.send_get('ParticipantCountsOverTime', query_string=qs)
interested_count_day_2 = response[1]['metrics']['INTERESTED']
member_count_day_2 = response[1]['metrics']['MEMBER']
self.assertEquals(interested_count_day_2, 0)
self.assertEquals(member_count_day_2, 1)
def test_get_counts_excluding_withdrawn_participants(self):
# Withdrawn participants should not appear in counts
p1 = Participant(participantId=1, biobankId=4)
self._insert(p1, 'Alice', 'Aardvark', 'PITT', time_int=self.time1)
p2 = Participant(participantId=2, biobankId=5)
self._insert(p2, 'Bob', 'Builder', 'AZ_TUCSON', time_int=self.time1)
p3 = Participant(participantId=3, biobankId=6)
ps3 = self._insert(p3, 'Chad', 'Caterpillar', 'AZ_TUCSON', time_int=self.time1)
ps3.withdrawalStatus = WithdrawalStatus.NO_USE # Chad withdrew from the study
self.ps_dao.update(ps3)
qs = """
bucketSize=1
&stratification=TOTAL
&startDate=2017-12-30
&endDate=2018-01-04
"""
qs = ''.join(qs.split()) # Remove all whitespace
response = self.send_get('ParticipantCountsOverTime', query_string=qs)
total_count_day_1 = response[0]['metrics']['TOTAL']
total_count_day_2 = response[1]['metrics']['TOTAL']
self.assertEquals(total_count_day_1, 0)
self.assertEquals(total_count_day_2, 2)
def test_get_counts_with_multiple_various_filters(self):
# Do the awardee and enrollment status filters work when passed multiple values?
p1 = Participant(participantId=1, biobankId=4)
self._insert(p1, 'Alice', 'Aardvark', 'PITT', time_int=self.time1)
p2 = Participant(participantId=2, biobankId=5)
self._insert(p2, 'Bob', 'Builder', 'AZ_TUCSON', time_int=self.time1)
p3 = Participant(participantId=3, biobankId=6)
self._insert(p3, 'Chad', 'Caterpillar', 'AZ_TUCSON', time_int=self.time1)
qs = """
bucketSize=1
&stratification=ENROLLMENT_STATUS
&startDate=2017-12-30
&endDate=2018-01-04
&awardee=AZ_TUCSON,PITT
&enrollmentStatus=INTERESTED,MEMBER,FULL_PARTICIPANT
"""
qs = ''.join(qs.split()) # Remove all whitespace
response = self.send_get('ParticipantCountsOverTime', query_string=qs)
interested_count_day_1 = response[0]['metrics']['INTERESTED']
interested_count_day_2 = response[1]['metrics']['INTERESTED']
self.assertEquals(interested_count_day_1, 0)
self.assertEquals(interested_count_day_2, 3)
def test_get_counts_with_total_stratification_unfiltered(self):
p1 = Participant(participantId=1, biobankId=4)
self._insert(p1, 'Alice', 'Aardvark', 'PITT', time_int=self.time1)
p2 = Participant(participantId=2, biobankId=5)
self._insert(p2, 'Bob', 'Builder', 'AZ_TUCSON', time_int=self.time1)
p3 = Participant(participantId=3, biobankId=6)
self._insert(p3, 'Chad', 'Caterpillar', 'AZ_TUCSON', time_int=self.time1)
qs = """
bucketSize=1
&stratification=TOTAL
&startDate=2017-12-30
&endDate=2018-01-04
"""
qs = ''.join(qs.split()) # Remove all whitespace
response = self.send_get('ParticipantCountsOverTime', query_string=qs)
total_count_day_1 = response[0]['metrics']['TOTAL']
total_count_day_2 = response[1]['metrics']['TOTAL']
self.assertEquals(total_count_day_1, 0)
self.assertEquals(total_count_day_2, 3)
def test_update_from_samples(self):
# baseline_tests = ['BASELINE1', 'BASELINE2']
baseline_tests = ["1PST8", "2PST8"]
config.override_setting(config.BASELINE_SAMPLE_TEST_CODES, baseline_tests)
self.dao.update_from_biobank_stored_samples() # safe noop
p_baseline_samples = self._insert(Participant(participantId=1, biobankId=11))
p_mixed_samples = self._insert(Participant(participantId=2, biobankId=22))
p_no_samples = self._insert(Participant(participantId=3, biobankId=33))
p_unconfirmed = self._insert(Participant(participantId=4, biobankId=44))
self.assertEquals(self.dao.get(p_baseline_samples.participantId).numBaselineSamplesArrived, 0)
def get_p_baseline_last_modified():
return self.dao.get(p_baseline_samples.participantId).lastModified
p_baseline_last_modified1 = get_p_baseline_last_modified()
sample_dao = BiobankStoredSampleDao()
def add_sample(participant, test_code, sample_id):
TIME = datetime.datetime(2018, 3, 2)
sample_dao.insert(BiobankStoredSample(
biobankStoredSampleId=sample_id, biobankId=participant.biobankId,
biobankOrderIdentifier='KIT', test=test_code, confirmed=TIME))
add_sample(p_baseline_samples, baseline_tests[0], '11111')
add_sample(p_baseline_samples, baseline_tests[1], '22223')
add_sample(p_mixed_samples, baseline_tests[0], '11112')
add_sample(p_mixed_samples, 'NOT1', '44441')
# add unconfirmed sample
sample_dao.insert(BiobankStoredSample(biobankStoredSampleId=55555,
biobankId=p_unconfirmed.biobankId,
biobankOrderIdentifier='KIT', test=baseline_tests[1],
confirmed=None))
# sleep 1 sec to make lastModified different
time.sleep(1)
self.dao.update_from_biobank_stored_samples()
p_baseline_last_modified2 = get_p_baseline_last_modified()
self.assertNotEquals(p_baseline_last_modified2, p_baseline_last_modified1)
self.assertEquals(self.dao.get(p_baseline_samples.participantId).numBaselineSamplesArrived, 2)
self.assertEquals(self.dao.get(p_mixed_samples.participantId).numBaselineSamplesArrived, 1)
self.assertEquals(self.dao.get(p_no_samples.participantId).numBaselineSamplesArrived, 0)
self.assertEquals(self.dao.get(p_unconfirmed.participantId).numBaselineSamplesArrived, 0)
M_baseline_samples = self._insert(Participant(participantId=9, biobankId=99))
add_sample(M_baseline_samples, baseline_tests[0], '999')
M_first_update = self.dao.get(M_baseline_samples.participantId)
# sleep 1 sec to make lastModified different
time.sleep(1)
self.dao.update_from_biobank_stored_samples()
add_sample(M_baseline_samples, baseline_tests[1], '9999')
M_second_update = self.dao.get(M_baseline_samples.participantId)
# sleep 1 sec to make lastModified different
time.sleep(1)
self.dao.update_from_biobank_stored_samples()
self.assertNotEqual(M_first_update.lastModified, M_second_update.lastModified)
self.assertEquals(get_p_baseline_last_modified(), p_baseline_last_modified2)
def test_pairing_at_different_levels(self):
p = Participant()
time = datetime.datetime(2016, 1, 1)
with random_ids([1, 2]):
with FakeClock(time):
self.dao.insert(p)
p.version = 1
p.siteId = 1
time2 = datetime.datetime(2016, 1, 2)
with FakeClock(time2):
self.dao.update(p)
p2 = self.dao.get(1)
ep = self._participant_with_defaults(participantId=1,
version=2,
biobankId=2,
lastModified=time2,
signUpTime=time,
hpoId=PITT_HPO_ID,
siteId=1,
organizationId=PITT_ORG_ID,
providerLink=p2.providerLink)
self.assertEquals(ep.siteId, p2.siteId)
# ensure that p2 get paired with expected awardee and organization from update().
self.assertEquals(ep.hpoId, p2.hpoId)
self.assertEquals(ep.organizationId, p2.organizationId)
def test_insert_with_external_id(self):
p = Participant(externalId=3)
time = datetime.datetime(2016, 1, 1)
with random_ids([1, 2]):
with FakeClock(time):
self.dao.insert(p)
expected_participant = self._participant_with_defaults(
participantId=1,
externalId=3,
version=1,
biobankId=2,
lastModified=time,
signUpTime=time)
self.assertEquals(expected_participant.asdict(), p.asdict())
p2 = self.dao.get(1)
self.assertEquals(p.asdict(), p2.asdict())
# Creating a participant also creates a ParticipantHistory row, but not a ParticipantSummary row
ps = self.participant_summary_dao.get(1)
self.assertIsNone(ps)
ph = self.participant_history_dao.get([1, 1])
expected_ph = self._participant_history_with_defaults(
participantId=1,
externalId=3,
biobankId=2,
lastModified=time,
signUpTime=time)
self.assertEquals(expected_ph.asdict(), ph.asdict())
def test_update_from_samples(self):
# baseline_tests = ['BASELINE1', 'BASELINE2']
baseline_tests = ["1PST8", "2PST8"]
config.override_setting(config.BASELINE_SAMPLE_TEST_CODES,
baseline_tests)
self.dao.update_from_biobank_stored_samples() # safe noop
p_baseline_samples = self._insert(
Participant(participantId=1, biobankId=11))
p_mixed_samples = self._insert(
Participant(participantId=2, biobankId=22))
p_no_samples = self._insert(Participant(participantId=3, biobankId=33))
self.assertEquals(
self.dao.get(
p_baseline_samples.participantId).numBaselineSamplesArrived, 0)
sample_dao = BiobankStoredSampleDao()
def add_sample(participant, test_code, sample_id):
TIME = datetime.datetime(2018, 3, 2)
sample_dao.insert(
BiobankStoredSample(biobankStoredSampleId=sample_id,
biobankId=participant.biobankId,
biobankOrderIdentifier='KIT',
test=test_code,
confirmed=TIME))
add_sample(p_baseline_samples, baseline_tests[0], '11111')
add_sample(p_baseline_samples, baseline_tests[1], '22223')
add_sample(p_mixed_samples, baseline_tests[0], '11112')
add_sample(p_mixed_samples, 'NOT1', '44441')
self.dao.update_from_biobank_stored_samples()
test_last_modified_doesnt_change_below = self.dao.get(1).lastModified
self.assertEquals(
self.dao.get(
p_baseline_samples.participantId).numBaselineSamplesArrived, 2)
self.assertEquals(
self.dao.get(
p_mixed_samples.participantId).numBaselineSamplesArrived, 1)
self.assertEquals(
self.dao.get(p_no_samples.participantId).numBaselineSamplesArrived,
0)
M_baseline_samples = self._insert(
Participant(participantId=9, biobankId=99))
add_sample(M_baseline_samples, baseline_tests[0], '999')
M_first_update = self.dao.get(M_baseline_samples.participantId)
self.dao.update_from_biobank_stored_samples()
add_sample(M_baseline_samples, baseline_tests[1], '9999')
M_second_update = self.dao.get(M_baseline_samples.participantId)
self.dao.update_from_biobank_stored_samples()
p_baseline_update = self.dao.get(p_baseline_samples.participantId)
self.assertNotEqual(M_first_update.lastModified,
M_second_update.lastModified)
self.assertEquals(p_baseline_update.lastModified,
test_last_modified_doesnt_change_below)
def setUp(self, **kwargs):
super(MetricsEhrMultiEndpointTest, self).setUp(**kwargs)
#
# insert some test data
#
for date in iter_dates(datetime.date(2017, 12, 30),
datetime.date(2018, 2, 1)):
calendar_day = Calendar(day=date)
CalendarDao().insert(calendar_day)
# noinspection PyArgumentList
participant_1 = Participant(participantId=1, biobankId=4)
summary_1 = self._make_participant(
participant_1,
'Alice',
'Aardvark',
self.hpo_foo,
self.org_foo_a,
time_int=datetime.datetime(2018, 1, 2),
time_mem=datetime.datetime(2018, 1, 3),
time_fp=datetime.datetime(2018, 1, 4))
# noinspection PyArgumentList
participant_2 = Participant(participantId=2, biobankId=5)
summary_2 = self._make_participant(
participant_2,
'Bo',
'Badger',
self.hpo_bar,
self.org_bar_a,
time_int=datetime.datetime(2018, 1, 3),
time_mem=datetime.datetime(2018, 1, 4),
time_fp=datetime.datetime(2018, 1, 5))
self._update_ehr(summary_1, update_time=datetime.datetime(2018, 1, 5))
self._update_ehr(summary_2, update_time=datetime.datetime(2018, 1, 6))
def testQuery_threeSummaries_paginate(self):
participant_1 = Participant(participantId=1, biobankId=4)
self._insert(participant_1, 'Alice', 'Aardvark')
participant_2 = Participant(participantId=2, biobankId=1)
self._insert(participant_2, 'Bob', 'Builder')
participant_3 = Participant(participantId=3, biobankId=3)
self._insert(participant_3, 'Chad', 'Caterpillar')
ps_1 = self.dao.get(1)
ps_2 = self.dao.get(2)
ps_3 = self.dao.get(3)
self.assert_results(self.no_filter_query, [ps_1, ps_2],
_make_pagination_token(['Builder', 'Bob', None, 2]))
self.assert_results(self.one_filter_query, [ps_1])
self.assert_no_results(self.two_filter_query)
self.assert_results(self.ascending_biobank_id_query, [ps_2, ps_3],
_make_pagination_token([3, 'Caterpillar', 'Chad', None, 3]))
self.assert_results(self.descending_biobank_id_query, [ps_1, ps_3],
_make_pagination_token([3, 'Caterpillar', 'Chad', None, 3]))
self.assert_results(_with_token(self.no_filter_query,
_make_pagination_token(['Builder', 'Bob', None, 2])), [ps_3])
self.assert_results(_with_token(self.ascending_biobank_id_query,
_make_pagination_token([3, 'Caterpillar', 'Chad', None, 3])),
[ps_1])
self.assert_results(_with_token(self.descending_biobank_id_query,
_make_pagination_token([3, 'Caterpillar', 'Chad', None, 3])),
[ps_2])
def test_get_counts_with_enrollment_status_member_filter(self):
p1 = Participant(participantId=1, biobankId=4)
self._insert(p1, 'Alice', 'Aardvark', 'PITT', time_int=self.time1)
p2 = Participant(participantId=2, biobankId=5)
self._insert(p2, 'Bob', 'Builder', 'AZ_TUCSON', time_int=self.time1, time_mem=self.time2)
p3 = Participant(participantId=3, biobankId=6)
self._insert(p3, 'Chad', 'Caterpillar', 'AZ_TUCSON', time_int=self.time1, time_mem=self.time2)
p4 = Participant(participantId=4, biobankId=7)
self._insert(p4, 'Debra', 'Dinosaur', 'PITT', time_int=self.time1, time_mem=self.time3)
# awardee param intentionally left blank to test we can handle it
qs = """
bucketSize=1
&stratification=ENROLLMENT_STATUS
&startDate=2017-12-30
&endDate=2018-01-04
&awardee=
&enrollmentStatus=MEMBER
"""
qs = ''.join(qs.split()) # Remove all whitespace
response = self.send_get('ParticipantCountsOverTime', query_string=qs)
member_count_day_1 = response[0]['metrics']['MEMBER']
member_count_day_2 = response[1]['metrics']['MEMBER']
member_count_day_3 = response[2]['metrics']['MEMBER']
member_count_day_4 = response[3]['metrics']['MEMBER']
interested_count_day_4 = response[1]['metrics']['INTERESTED']
self.assertEquals(member_count_day_1, 0)
self.assertEquals(member_count_day_2, 0)
self.assertEquals(member_count_day_3, 2)
self.assertEquals(member_count_day_4, 3)
self.assertEquals(interested_count_day_4, 0)
qs = """
bucketSize=1
&stratification=TOTAL
&startDate=2017-12-30
&endDate=2018-01-04
&enrollmentStatus=MEMBER
"""
qs = ''.join(qs.split()) # Remove all whitespace
response = self.send_get('ParticipantCountsOverTime', query_string=qs)
total_count_day_1 = response[0]['metrics']['TOTAL']
total_count_day_2 = response[1]['metrics']['TOTAL']
self.assertEquals(total_count_day_1, 0)
self.assertEquals(total_count_day_2, 3)
def test_bad_hpo_update(self):
p = Participant(participantId=1, biobankId=2)
time = datetime.datetime(2016, 1, 1)
with FakeClock(time):
self.dao.insert(p)
p.providerLink = make_primary_provider_link_for_name('FOO')
with self.assertRaises(BadRequest):
self.dao.update(p)
def _setup_participants(self):
self.participant1 = Participant(participantId=1, biobankId=1)
self.participant2 = Participant(participantId=2, biobankId=2)
self.participant_dao.insert(self.participant1)
self.participant_dao.insert(self.participant2)
self.assertEqual(self.participant1.isGhostId, None)
self.assertEqual(self.participant1.dateAddedGhost, None)
self.assertEqual(self.participant2.isGhostId, None)
self.assertEqual(self.participant2.dateAddedGhost, None)
def test_update_no_expected_version_with_ps(self):
p = Participant()
time = datetime.datetime(2016, 1, 1)
with random_ids([1, 2]):
with FakeClock(time):
self.dao.insert(p)
p.providerLink = make_primary_provider_link_for_name('PITT')
time2 = datetime.datetime(2016, 1, 2)
with FakeClock(time2):
self.dao.update(p)
summary = self.participant_summary(p)
self.participant_summary_dao.insert(summary)
# lastModified, hpoId, version is updated on p after being passed in
p2 = self.dao.get(1)
expected_participant = self._participant_with_defaults(
participantId=1,
version=2,
biobankId=2,
lastModified=time2,
signUpTime=time,
hpoId=PITT_HPO_ID,
providerLink=p2.providerLink)
self.assertEquals(expected_participant.asdict(), p2.asdict())
self.assertEquals(p.asdict(), p2.asdict())
# Updating the participant provider link also updates the HPO ID on the participant summary.
ps = self.participant_summary_dao.get(1)
expected_ps = self._participant_summary_with_defaults(
participantId=1,
biobankId=2,
signUpTime=time,
hpoId=PITT_HPO_ID,
firstName=summary.firstName,
lastName=summary.lastName,
email=summary.email)
self.assertEquals(expected_ps.asdict(), ps.asdict())
expected_ph = self._participant_history_with_defaults(
participantId=1, biobankId=2, lastModified=time, signUpTime=time)
# And updating the participant adds a new ParticipantHistory row.
ph = self.participant_history_dao.get([1, 1])
self.assertEquals(expected_ph.asdict(), ph.asdict())
ph2 = self.participant_history_dao.get([1, 2])
expected_ph2 = self._participant_history_with_defaults(
participantId=1,
version=2,
biobankId=2,
lastModified=time2,
signUpTime=time,
hpoId=PITT_HPO_ID,
providerLink=p2.providerLink)
self.assertEquals(expected_ph2.asdict(), ph2.asdict())
def test_insert_duplicate_biobank_id_give_up(self):
p = Participant()
with random_ids([1, 2]):
self.dao.insert(p)
rand_ints = []
for i in range(0, MAX_INSERT_ATTEMPTS):
rand_ints.append(i + 2)
rand_ints.append(2)
p2 = Participant()
with random_ids(rand_ints):
with self.assertRaises(ServiceUnavailable):
self.dao.insert(p2)
def testQuery_twoSummaries(self):
participant_1 = Participant(participantId=1, biobankId=2)
self._insert(participant_1, 'Alice', 'Smith')
participant_2 = Participant(participantId=2, biobankId=1)
self._insert(participant_2, 'Zed', 'Zebra')
ps_1 = self.dao.get(1)
ps_2 = self.dao.get(2)
self.assert_results(self.no_filter_query, [ps_1, ps_2])
self.assert_results(self.one_filter_query, [ps_1])
self.assert_no_results(self.two_filter_query)
self.assert_results(self.ascending_biobank_id_query, [ps_2, ps_1])
self.assert_results(self.descending_biobank_id_query, [ps_1, ps_2])
def test_update_wrong_expected_version(self):
p = Participant()
time = datetime.datetime(2016, 1, 1)
with random_ids([1, 2]):
with FakeClock(time):
self.dao.insert(p)
p.version = 2
p.providerLink = make_primary_provider_link_for_name('PITT')
time2 = datetime.datetime(2016, 1, 2)
with FakeClock(time2):
with self.assertRaises(PreconditionFailed):
self.dao.update(p)
def test_get_counts_with_single_awardee_filter(self):
# Does the awardee filter work?
p1 = Participant(participantId=1, biobankId=4)
self._insert(p1, 'Alice', 'Aardvark', 'PITT', time_int=self.time1)
p1 = Participant(participantId=2, biobankId=5)
self._insert(p1, 'Bob', 'Builder', 'AZ_TUCSON', time_int=self.time1)
p1 = Participant(participantId=3, biobankId=6)
self._insert(p1, 'Chad', 'Caterpillar', 'AZ_TUCSON', time_int=self.time1)
# enrollmentStatus param left blank to test we can handle it
qs = """
bucketSize=1
&stratification=ENROLLMENT_STATUS
&startDate=2017-12-30
&endDate=2018-01-04
&awardee=PITT
&enrollmentStatus=
"""
qs = ''.join(qs.split()) # Remove all whitespace
response = self.send_get('ParticipantCountsOverTime', query_string=qs)
interested_count_day_1 = response[0]['metrics']['INTERESTED']
interested_count_day_2 = response[1]['metrics']['INTERESTED']
self.assertEquals(interested_count_day_1, 0)
self.assertEquals(interested_count_day_2, 1)
qs = """
bucketSize=1
&stratification=ENROLLMENT_STATUS
&startDate=2017-12-30
&endDate=2018-01-04
&awardee=AZ_TUCSON
"""
qs = ''.join(qs.split()) # Remove all whitespace
response = self.send_get('ParticipantCountsOverTime', query_string=qs)
interested_count_day_1 = response[0]['metrics']['INTERESTED']
interested_count_day_2 = response[1]['metrics']['INTERESTED']
self.assertEquals(interested_count_day_1, 0)
self.assertEquals(interested_count_day_2, 2)
def test_end_to_end(self):
dao = BiobankStoredSampleDao()
self.assertEquals(dao.count(), 0)
# Create 3 participants and pass their (random) IDs into sample rows.
summary_dao = ParticipantSummaryDao()
biobank_ids = []
participant_ids = []
for _ in xrange(3):
participant = self.participant_dao.insert(Participant())
summary_dao.insert(self.participant_summary(participant))
participant_ids.append(participant.participantId)
biobank_ids.append(participant.biobankId)
self.assertEquals(
summary_dao.get(
participant.participantId).numBaselineSamplesArrived, 0)
test1, test2, test3 = random.sample(_BASELINE_TESTS, 3)
samples_file = test_data.open_biobank_samples(*biobank_ids,
test1=test1,
test2=test2,
test3=test3)
input_filename = 'cloud%s.csv' % self._naive_utc_to_naive_central(
clock.CLOCK.now()).strftime(
biobank_samples_pipeline.INPUT_CSV_TIME_FORMAT)
self._write_cloud_csv(input_filename, samples_file.read())
biobank_samples_pipeline.upsert_from_latest_csv()
self.assertEquals(dao.count(), 3)
self._check_summary(participant_ids[0], test1, '2016-11-29T12:19:32')
self._check_summary(participant_ids[1], test2, '2016-11-29T12:38:58')
self._check_summary(participant_ids[2], test3, '2016-11-29T12:41:26')
def from_client_json(self, resource_json, id_=None, expected_version=None, client_id=None):
parse_json_enum(resource_json, 'withdrawalStatus', WithdrawalStatus)
parse_json_enum(resource_json, 'withdrawalReason', WithdrawalReason)
parse_json_enum(resource_json, 'suspensionStatus', SuspensionStatus)
if 'withdrawalTimeStamp' in resource_json and resource_json['withdrawalTimeStamp'] is not None:
try:
resource_json['withdrawalTimeStamp'] = datetime.datetime\
.utcfromtimestamp(float(resource_json['withdrawalTimeStamp'])/1000)
except (ValueError, TypeError):
raise ValueError("Could not parse {} as TIMESTAMP"
.format(resource_json['withdrawalTimeStamp']))
# biobankId, lastModified, signUpTime are set by DAO.
return Participant(
participantId=id_,
externalId=resource_json.get('externalId'),
version=expected_version,
providerLink=json.dumps(resource_json.get('providerLink')),
clientId=client_id,
withdrawalStatus=resource_json.get('withdrawalStatus'),
withdrawalReason=resource_json.get('withdrawalReason'),
withdrawalAuthored=resource_json.get('withdrawalTimeStamp'),
withdrawalReasonJustification=resource_json.get('withdrawalReasonJustification'),
suspensionStatus=resource_json.get('suspensionStatus'),
organizationId=get_organization_id_from_external_id(resource_json, self.organization_dao),
hpoId=get_awardee_id_from_name(resource_json, self.hpo_dao),
siteId=get_site_id_from_google_group(resource_json, self.site_dao))
def test_insert_duplicate_participant_id_retry(self):
p = Participant()
with random_ids([1, 2]):
self.dao.insert(p)
p2 = Participant()
time = datetime.datetime(2016, 1, 1)
with random_ids([1, 3, 2, 3]):
with FakeClock(time):
p2 = self.dao.insert(p2)
expected_participant = self._participant_with_defaults(
participantId=2,
version=1,
biobankId=3,
lastModified=time,
signUpTime=time)
self.assertEquals(expected_participant.asdict(), p2.asdict())
def test_insert_duplicate(self):
self.insert_codes()
p = Participant(participantId=1, biobankId=2)
self.participant_dao.insert(p)
self._setup_questionnaire()
qr = QuestionnaireResponse(questionnaireResponseId=1,
questionnaireId=1,
questionnaireVersion=1,
participantId=1,
resource=QUESTIONNAIRE_RESPONSE_RESOURCE)
qr.answers.extend(self._names_and_email_answers())
self.questionnaire_response_dao.insert(qr)
qr2 = QuestionnaireResponse(questionnaireResponseId=1,
questionnaireId=1,
questionnaireVersion=1,
participantId=1,
resource=QUESTIONNAIRE_RESPONSE_RESOURCE_2)
qr2.answers.append(
QuestionnaireResponseAnswer(questionnaireResponseAnswerId=2,
questionnaireResponseId=1,
questionId=2,
valueSystem='c',
valueCodeId=4))
with self.assertRaises(IntegrityError):
self.questionnaire_response_dao.insert(qr2)
def test_get_after_withdrawal_fails(self):
self.insert_codes()
p = Participant(participantId=1, biobankId=2)
self.participant_dao.insert(p)
self._setup_questionnaire()
qr = QuestionnaireResponse(questionnaireResponseId=1,
questionnaireId=1,
questionnaireVersion=1,
participantId=1,
resource=QUESTIONNAIRE_RESPONSE_RESOURCE)
qr.answers.extend(self._names_and_email_answers())
self.questionnaire_response_dao.insert(qr)
p.withdrawalStatus = WithdrawalStatus.NO_USE
self.participant_dao.update(p)
with self.assertRaises(Forbidden):
self.questionnaire_response_dao.get(qr.questionnaireResponseId)
def test_overwrite_existing_pairing(self):
participant_id = 99
created = self.dao.insert(
Participant(participantId=participant_id,
biobankId=2,
hpoId=self._test_db.hpo_id,
providerLink=make_primary_provider_link_for_id(
self._test_db.hpo_id)))
self.participant_summary_dao.insert(self.participant_summary(created))
self.assertEquals(created.hpoId, self._test_db.hpo_id) # sanity check
other_hpo = HPODao().insert(
HPO(hpoId=PITT_HPO_ID + 1, name='DIFFERENT_HPO'))
other_site = SiteDao().insert(
Site(hpoId=other_hpo.hpoId,
siteName='Arbitrary Site',
googleGroup='*****@*****.**'))
with self.dao.session() as session:
self.dao.add_missing_hpo_from_site(session, participant_id,
other_site.siteId)
# Original Participant + summary is affected.
refetched = self.dao.get(participant_id)
self.assertEquals(refetched.hpoId, other_hpo.hpoId)
self.assertEquals(refetched.providerLink,
make_primary_provider_link_for_id(other_hpo.hpoId))
self.assertEquals(
self.participant_summary_dao.get(participant_id).hpoId,
other_hpo.hpoId)
def test_get_counts_with_default_parameters(self):
# The most basic test in this class
p1 = Participant(participantId=1, biobankId=4)
self._insert(p1, 'Alice', 'Aardvark', 'PITT', time_int=self.time1)
# TODO: remove bucketSize from these parameters in all tests
qs = """
bucketSize=1
&stratification=ENROLLMENT_STATUS
&startDate=2017-12-30
&endDate=2018-01-04
"""
qs = ''.join(qs.split()) # Remove all whitespace
response = self.send_get('ParticipantCountsOverTime', query_string=qs)
# You can debug API responses easily by uncommenting the lines below
# print('response')
# print(response)
interested_count_day_1 = response[0]['metrics']['INTERESTED']
interested_count_day_2 = response[1]['metrics']['INTERESTED']
self.assertEquals(interested_count_day_1, 0)
self.assertEquals(interested_count_day_2, 1)
def test_dv_order_sample_update(self):
"""
Test Biobank Direct Volunteer order
"""
participant = self.participant_dao.insert(Participant())
self.summary_dao.insert(self.participant_summary(participant))
created_ts = datetime.datetime(2019, 03, 22, 18, 30, 45)
confirmed_ts = datetime.datetime(2019, 03, 23, 12, 13, 00)
bo = self._make_biobank_order(participantId=participant.participantId)
BiobankOrderDao().insert(bo)
boi = bo.identifiers[0]
bss = BiobankStoredSample(biobankStoredSampleId=u'23523523',
biobankId=participant.biobankId,
test=u'1SAL2',
created=created_ts,
biobankOrderIdentifier=boi.value,
confirmed=confirmed_ts)
with self.participant_dao.session() as session:
session.add(bss)
ps = self.summary_dao.get(participant.participantId)
self.assertIsNone(ps.sampleStatusDV1SAL2)
self.assertIsNone(ps.sampleStatusDV1SAL2Time)
self.summary_dao.update_from_biobank_stored_samples()
ps = self.summary_dao.get(participant.participantId)
self.assertEqual(ps.sampleStatus1SAL2, SampleStatus.RECEIVED)
self.assertEqual(ps.sampleStatus1SAL2Time, confirmed_ts)
def test_amending_biobank_order_distinct_visit_count(self):
self.participant = self._insert(Participant(participantId=9, biobankId=13))
with clock.FakeClock(TIME_5):
order = self.order_dao.insert(self._make_biobank_order(biobankOrderId='2', identifiers=[
BiobankOrderIdentifier(system='b', value='d')], samples=[BiobankOrderedSample(
biobankOrderId='2',
finalized=TIME_5,
test=BIOBANK_TESTS[0],
description='description',
processingRequired=True)]))
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 1)
with clock.FakeClock(TIME_7):
amend_order = self._get_amended_info(order)
with self.order_dao.session() as session:
self.order_dao._do_update(session, amend_order, order)
# Shouldn't change on a simple amendment (unless finalized time on samples change)
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 1)
with clock.FakeClock(TIME_7_5):
cancel_request = cancel_biobank_order()
order = self.order_dao.update_with_patch(order.biobankOrderId, cancel_request, order.version)
# A cancelled order (even after amending) should reduce count (unless some other valid order on same day)
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 0)
def test_only_update_dna_sample(self):
dna_tests = ["1ED10", "1SAL2"]
config.override_setting(config.DNA_SAMPLE_TEST_CODES, dna_tests)
self.dao.update_from_biobank_stored_samples() # safe noop
p_dna_samples = self._insert(Participant(participantId=1, biobankId=11))
self.assertEquals(self.dao.get(p_dna_samples.participantId).samplesToIsolateDNA, None)
self.assertEquals(
self.dao.get(p_dna_samples.participantId).enrollmentStatusCoreStoredSampleTime, None)
self.assertEquals(
self.dao.get(p_dna_samples.participantId).enrollmentStatusCoreOrderedSampleTime, None)
sample_dao = BiobankStoredSampleDao()
def add_sample(participant, test_code, sample_id, confirmed_time):
sample_dao.insert(BiobankStoredSample(
biobankStoredSampleId=sample_id, biobankId=participant.biobankId,
biobankOrderIdentifier='KIT', test=test_code, confirmed=confirmed_time))
confirmed_time_0 = datetime.datetime(2018, 3, 1)
add_sample(p_dna_samples, dna_tests[0], '11111', confirmed_time_0)
self.dao.update_from_biobank_stored_samples()
self.assertEquals(self.dao.get(p_dna_samples.participantId).samplesToIsolateDNA,
SampleStatus.RECEIVED)
# only update dna sample will not update enrollmentStatusCoreStoredSampleTime
self.assertEquals(
self.dao.get(p_dna_samples.participantId).enrollmentStatusCoreStoredSampleTime, None)
self.assertEquals(
self.dao.get(p_dna_samples.participantId).enrollmentStatusCoreOrderedSampleTime, None)
def test_update_from_samples_changed_tests(self):
baseline_tests = ["1PST8", "2PST8"]
config.override_setting(config.BASELINE_SAMPLE_TEST_CODES, baseline_tests)
self.dao.update_from_biobank_stored_samples() # safe noop
participant = self._insert(Participant(participantId=1, biobankId=11))
self.assertEquals(self.dao.get(participant.participantId).numBaselineSamplesArrived, 0)
sample_dao = BiobankStoredSampleDao()
def add_sample(test_code, sample_id):
TIME = datetime.datetime(2018, 3, 2)
sample_dao.insert(BiobankStoredSample(
biobankStoredSampleId=sample_id, biobankId=participant.biobankId,
biobankOrderIdentifier='KIT', test=test_code, confirmed=TIME))
add_sample(baseline_tests[0], '11111')
add_sample(baseline_tests[1], '22223')
self.dao.update_from_biobank_stored_samples()
summary = self.dao.get(participant.participantId)
init_last_modified = summary.lastModified
self.assertEquals(summary.numBaselineSamplesArrived, 2)
# sleep 1 sec to make lastModified different
time.sleep(1)
# Simulate removal of one of the baseline tests from config.json.
baseline_tests.pop()
config.override_setting(config.BASELINE_SAMPLE_TEST_CODES, baseline_tests)
self.dao.update_from_biobank_stored_samples()
summary = self.dao.get(participant.participantId)
self.assertEquals(summary.numBaselineSamplesArrived, 1)
self.assertNotEqual(init_last_modified, summary.lastModified)
def show(self):
r = Participant.get(self.params.get('id'))
def index(self):
query = Participant.all()
self.result = query.fetch(limit=1000)