def testDeidentifiedExport_participantIds(self):
TableExporter.export_tables('rdr', ['ppi_participant_view'], 'dir', deidentify=True)
p1 = self._participant_with_defaults(
participantId=1,
version=2,
biobankId=2,
providerLink=make_primary_provider_link_for_name('PITT'))
ParticipantDao().insert(p1)
p2 = self._participant_with_defaults(
participantId=2,
version=3,
biobankId=3,
providerLink=make_primary_provider_link_for_name('PITT'))
ParticipantDao().insert(p2)
tasks = self.taskqueue_stub.get_filtered_tasks()
self.assertEqual(len(tasks), 1)
csv_path = deferred.run(tasks[0].payload)
with cloudstorage_api.open('/' + csv_path, mode='r') as output:
reader = csv.reader(output)
rows = list(reader)[1:]
self.assertEqual(2, len(rows))
pmi_ids = set([p1.participantId, p2.participantId])
obf_ids = set([row[0] for row in rows])
self.assertFalse(pmi_ids.intersection(obf_ids),
'should be no overlap between pmi_ids and obfuscated IDs')
self.assertEquals(2, len(obf_ids))
def test_bad_hpo_insert(self):
p = Participant(
participantId=1,
version=1,
biobankId=2,
providerLink=make_primary_provider_link_for_name('FOO'))
with self.assertRaises(BadRequest):
self.dao.insert(p)
def test_bad_hpo_update(self):
p = Participant(participantId=1, biobankId=2)
time = datetime.datetime(2016, 1, 1)
with FakeClock(time):
self.dao.insert(p)
p.providerLink = make_primary_provider_link_for_name('FOO')
with self.assertRaises(BadRequest):
self.dao.update(p)
def test_update_no_expected_version_with_ps(self):
p = Participant()
time = datetime.datetime(2016, 1, 1)
with random_ids([1, 2]):
with FakeClock(time):
self.dao.insert(p)
p.providerLink = make_primary_provider_link_for_name('PITT')
time2 = datetime.datetime(2016, 1, 2)
with FakeClock(time2):
self.dao.update(p)
summary = self.participant_summary(p)
self.participant_summary_dao.insert(summary)
# lastModified, hpoId, version is updated on p after being passed in
p2 = self.dao.get(1)
expected_participant = self._participant_with_defaults(
participantId=1,
version=2,
biobankId=2,
lastModified=time2,
signUpTime=time,
hpoId=PITT_HPO_ID,
providerLink=p2.providerLink)
self.assertEquals(expected_participant.asdict(), p2.asdict())
self.assertEquals(p.asdict(), p2.asdict())
# Updating the participant provider link also updates the HPO ID on the participant summary.
ps = self.participant_summary_dao.get(1)
expected_ps = self._participant_summary_with_defaults(
participantId=1,
biobankId=2,
signUpTime=time,
hpoId=PITT_HPO_ID,
firstName=summary.firstName,
lastName=summary.lastName,
email=summary.email)
self.assertEquals(expected_ps.asdict(), ps.asdict())
expected_ph = self._participant_history_with_defaults(
participantId=1, biobankId=2, lastModified=time, signUpTime=time)
# And updating the participant adds a new ParticipantHistory row.
ph = self.participant_history_dao.get([1, 1])
self.assertEquals(expected_ph.asdict(), ph.asdict())
ph2 = self.participant_history_dao.get([1, 2])
expected_ph2 = self._participant_history_with_defaults(
participantId=1,
version=2,
biobankId=2,
lastModified=time2,
signUpTime=time,
hpoId=PITT_HPO_ID,
providerLink=p2.providerLink)
self.assertEquals(expected_ph2.asdict(), ph2.asdict())
def test_update_wrong_expected_version(self):
p = Participant()
time = datetime.datetime(2016, 1, 1)
with random_ids([1, 2]):
with FakeClock(time):
self.dao.insert(p)
p.version = 2
p.providerLink = make_primary_provider_link_for_name('PITT')
time2 = datetime.datetime(2016, 1, 2)
with FakeClock(time2):
with self.assertRaises(PreconditionFailed):
self.dao.update(p)
def test_update_withdrawn_hpo_succeeds(self):
p = Participant(withdrawalStatus=WithdrawalStatus.NO_USE)
time = datetime.datetime(2016, 1, 1)
with random_ids([1, 2]):
with FakeClock(time):
self.dao.insert(p)
expected_participant = self._participant_with_defaults(
participantId=1,
version=1,
biobankId=2,
lastModified=time,
signUpTime=time,
withdrawalStatus=WithdrawalStatus.NO_USE)
self.assertEquals(expected_participant.asdict(), p.asdict())
p2 = self.dao.get(1)
self.assertEquals(p.asdict(), p2.asdict())
p.version = 1
p.providerLink = make_primary_provider_link_for_name('PITT')
self.dao.update(p)
def test_update_right_expected_version(self):
p = Participant()
time = datetime.datetime(2016, 1, 1)
with random_ids([1, 2]):
with FakeClock(time):
self.dao.insert(p)
p.version = 1
p.providerLink = make_primary_provider_link_for_name('PITT')
time2 = datetime.datetime(2016, 1, 2)
with FakeClock(time2):
self.dao.update(p)
p2 = self.dao.get(1)
expected_participant = self._participant_with_defaults(
participantId=1,
version=2,
biobankId=2,
lastModified=time2,
signUpTime=time,
hpoId=PITT_HPO_ID,
providerLink=p2.providerLink)
self.assertEquals(expected_participant.asdict(), p2.asdict())
def _create_data(self):
HPODao().insert(HPO(hpoId=PITT_HPO_ID + 1, name='AZ_TUCSON_2'))
HPODao().insert(HPO(hpoId=PITT_HPO_ID + 4, name='TEST'))
SqlTestBase.setup_codes(
ANSWER_FIELD_TO_QUESTION_CODE.values() + [EHR_CONSENT_QUESTION_CODE],
code_type=CodeType.QUESTION)
SqlTestBase.setup_codes(
FIELD_TO_QUESTIONNAIRE_MODULE_CODE.values(), code_type=CodeType.MODULE)
# Import codes for white and female, but not male or black.
SqlTestBase.setup_codes([
RACE_WHITE_CODE, CONSENT_PERMISSION_YES_CODE, RACE_NONE_OF_THESE_CODE,
PMI_PREFER_NOT_TO_ANSWER_CODE, CONSENT_PERMISSION_NO_CODE, 'female', 'PIIState_VA',
PMI_SKIP_CODE
], code_type=CodeType.ANSWER)
participant_dao = ParticipantDao()
questionnaire_id = self.create_questionnaire('questionnaire3.json')
questionnaire_id_2 = self.create_questionnaire('questionnaire4.json')
questionnaire_id_3 = self.create_questionnaire('all_consents_questionnaire.json')
pl_tucson = make_primary_provider_link_for_name('AZ_TUCSON')
pl_test = make_primary_provider_link_for_name('TEST')
pl_pitt = make_primary_provider_link_for_name('PITT')
with FakeClock(TIME):
participant = Participant(participantId=1, biobankId=2, providerLink=pl_tucson)
participant_dao.insert(participant)
self.send_consent('P1', email='*****@*****.**')
# Participant 2 starts out unpaired; later gets paired automatically when their physical
# measurements come in.
participant2 = Participant(participantId=2, biobankId=3)
participant_dao.insert(participant2)
self.send_consent('P2', email='*****@*****.**')
# Test HPO affiliation; this test participant is ignored.
participant3 = Participant(participantId=3, biobankId=4, providerLink=pl_test)
participant_dao.insert(participant3)
self.send_consent('P3', email='*****@*****.**')
# example.com e-mail; this test participant is ignored, too.
participant4 = Participant(participantId=4, biobankId=5, providerLink=pl_pitt)
participant_dao.insert(participant4)
self.send_consent('P4', email='*****@*****.**')
participant5 = Participant(participantId=5, biobankId=6, providerLink=pl_tucson)
participant_dao.insert(participant5)
self.send_consent('P5', email='*****@*****.**')
with FakeClock(TIME_2):
# FIXME: The test passes, but the following "update" doesn't actually make much sense. The
# providerlink is not changed but the HPO ID actually is (at this point in time
# `participant.hpoId` evaluates to 4, which is the value given in `unit_test_util.AZ_HPO_ID`).
# The original intent of the test is not clear.
# This update to participant has no effect, as the HPO ID didn't change.
participant = self._participant_with_defaults(
participantId=1, version=1, biobankId=2,
hpoId=3, # <<<< Had to add hpoId here, default is UNSET_HPO_ID
providerLink=pl_tucson
)
participant_dao.update(participant)
self.submit_questionnaire_response('P1', questionnaire_id,
race_code=RACE_WHITE_CODE,
gender_code='male',
state=PMI_SKIP_CODE,
date_of_birth=datetime.date(1980, 1, 2))
self.submit_questionnaire_response('P2', questionnaire_id,
race_code=RACE_NONE_OF_THESE_CODE,
gender_code=None,
state=None,
date_of_birth=None)
self.submit_questionnaire_response('P5', questionnaire_id,
race_code=PMI_SKIP_CODE,
gender_code=PMI_SKIP_CODE,
state=None,
date_of_birth=None)
with FakeClock(TIME_3):
# Re-pair the original participant
participant.version = 2
participant.providerLink = pl_pitt
participant_dao.update(participant)
self.send_post('Participant/P2/PhysicalMeasurements', load_measurement_json(2))
self.send_post('Participant/P2/BiobankOrder', load_biobank_order_json(2))
self.submit_questionnaire_response('P1', questionnaire_id,
race_code='black',
gender_code='female',
state=None,
date_of_birth=datetime.date(1980, 1, 3))
self.submit_questionnaire_response('P2', questionnaire_id,
race_code=None,
gender_code=PMI_PREFER_NOT_TO_ANSWER_CODE,
state=None,
date_of_birth=None)
self.submit_questionnaire_response('P2', questionnaire_id_2,
race_code=None,
gender_code=None,
state='PIIState_VA',
date_of_birth=None)
self.submit_consent_questionnaire_response('P1', questionnaire_id_3,
CONSENT_PERMISSION_NO_CODE)
self.submit_consent_questionnaire_response('P2', questionnaire_id_3,
CONSENT_PERMISSION_YES_CODE)
sample_dao = BiobankStoredSampleDao()
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='abc',
biobankId=2,
biobankOrderIdentifier='KIT',
test='test',
confirmed=TIME_2))
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='def',
biobankId=3,
biobankOrderIdentifier='KIT',
test='1SAL',
confirmed=TIME_2))
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='xyz',
biobankId=4,
biobankOrderIdentifier='KIT',
test='1SAL',
confirmed=TIME_2))
def _create_data(self):
HPODao().insert(HPO(hpoId=PITT_HPO_ID + 1, name='AZ_TUCSON'))
HPODao().insert(HPO(hpoId=PITT_HPO_ID + 2, name='TEST'))
SqlTestBase.setup_codes(
ANSWER_FIELD_TO_QUESTION_CODE.values() + [EHR_CONSENT_QUESTION_CODE],
code_type=CodeType.QUESTION)
SqlTestBase.setup_codes(
FIELD_TO_QUESTIONNAIRE_MODULE_CODE.values(), code_type=CodeType.MODULE)
# Import codes for white and female, but not male or black.
SqlTestBase.setup_codes(
[
RACE_WHITE_CODE, CONSENT_PERMISSION_YES_CODE,
CONSENT_PERMISSION_NO_CODE, 'female', 'PIIState_VA'
],
code_type=CodeType.ANSWER)
participant_dao = ParticipantDao()
questionnaire_id = self.create_questionnaire('questionnaire3.json')
questionnaire_id_2 = self.create_questionnaire('questionnaire4.json')
questionnaire_id_3 = self.create_questionnaire(
'all_consents_questionnaire.json')
with FakeClock(TIME):
participant = Participant(
participantId=1,
biobankId=2,
providerLink=make_primary_provider_link_for_name('AZ_TUCSON'))
participant_dao.insert(participant)
self.send_consent('P1', email='*****@*****.**')
with FakeClock(TIME):
# Participant 2 starts out unpaired; later gets paired automatically when their physical
# measurements come in.
participant2 = Participant(
participantId=2,
biobankId=3)
participant_dao.insert(participant2)
self.send_consent('P2', email='*****@*****.**')
with FakeClock(TIME):
# Test HPO affiliation; this test participant is ignored.
participant3 = Participant(participantId=3, biobankId=4,
providerLink=make_primary_provider_link_for_name('TEST'))
participant_dao.insert(participant3)
self.send_consent('P3', email='*****@*****.**')
# example.com e-mail; this test participant is ignored, too.
participant4 = Participant(participantId=4, biobankId=5,
providerLink=make_primary_provider_link_for_name('PITT'))
participant_dao.insert(participant4)
self.send_consent('P4', email='*****@*****.**')
with FakeClock(TIME_2):
# This update to participant has no effect, as the HPO ID didn't change.
participant = self._participant_with_defaults(
participantId=1, version=1, biobankId=2,
providerLink=make_primary_provider_link_for_name('AZ_TUCSON'))
participant_dao.update(participant)
self.submit_questionnaire_response('P1', questionnaire_id,
RACE_WHITE_CODE, 'male', None,
datetime.date(1980, 1, 2))
self.submit_questionnaire_response('P2', questionnaire_id, None, None,
None, None)
with FakeClock(TIME_3):
participant = self._participant_with_defaults(
participantId=1,
version=2,
biobankId=2,
providerLink=make_primary_provider_link_for_name('PITT'))
participant_dao.update(participant)
self.send_post('Participant/P2/PhysicalMeasurements',
load_measurement_json(2))
self.send_post('Participant/P2/BiobankOrder', load_biobank_order_json(2))
self.submit_questionnaire_response('P1', questionnaire_id, 'black',
'female', None,
datetime.date(1980, 1, 3))
self.submit_questionnaire_response('P2', questionnaire_id_2, None, None,
'PIIState_VA', None)
self._submit_consent_questionnaire_response('P1', questionnaire_id_3,
CONSENT_PERMISSION_NO_CODE)
self._submit_consent_questionnaire_response('P2', questionnaire_id_3,
CONSENT_PERMISSION_YES_CODE)
sample_dao = BiobankStoredSampleDao()
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='abc',
biobankId=2,
test='test',
confirmed=TIME_2))
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='def',
biobankId=3,
test='1SAL',
confirmed=TIME_2))
def _create_data(self):
HPODao().insert(HPO(hpoId=AZ_HPO_ID + 1, name='AZ_TUCSON_2'))
HPODao().insert(HPO(hpoId=PITT_HPO_ID + 4, name='TEST'))
SqlTestBase.setup_codes(
ANSWER_FIELD_TO_QUESTION_CODE.values() + [EHR_CONSENT_QUESTION_CODE],
code_type=CodeType.QUESTION)
SqlTestBase.setup_codes(
FIELD_TO_QUESTIONNAIRE_MODULE_CODE.values(), code_type=CodeType.MODULE)
# Import codes for white and female, but not male or black.
SqlTestBase.setup_codes(
[
RACE_WHITE_CODE, CONSENT_PERMISSION_YES_CODE,
RACE_NONE_OF_THESE_CODE, PMI_PREFER_NOT_TO_ANSWER_CODE,
CONSENT_PERMISSION_NO_CODE, 'female', 'PIIState_VA'
],
code_type=CodeType.ANSWER)
participant_dao = ParticipantDao()
questionnaire_id = self.create_questionnaire('questionnaire3.json')
questionnaire_id_2 = self.create_questionnaire('questionnaire4.json')
questionnaire_id_3 = self.create_questionnaire(
'all_consents_questionnaire.json')
with FakeClock(TIME):
participant = self._participant_with_defaults(
participantId=1,
version=2,
biobankId=2,
providerLink=make_primary_provider_link_for_name('PITT'))
participant_dao.insert(participant)
self.send_consent('P1', email='*****@*****.**')
# Participant 2 starts out unpaired; later gets paired automatically when their physical
# measurements come in.
participant2 = Participant(participantId=2, biobankId=3)
participant_dao.insert(participant2)
self.send_consent('P2', email='*****@*****.**')
# Test HPO affiliation; this test participant is ignored.
participant3 = Participant(
participantId=3,
biobankId=4,
providerLink=make_primary_provider_link_for_name('TEST'))
participant_dao.insert(participant3)
self.send_consent('P3', email='*****@*****.**')
# example.com e-mail; this test participant is ignored, too.
participant4 = Participant(
participantId=4,
biobankId=5,
providerLink=make_primary_provider_link_for_name('PITT'))
participant_dao.insert(participant4)
self.send_consent('P4', email='*****@*****.**')
participant5 = Participant(
participantId=5,
biobankId=6,
providerLink=make_primary_provider_link_for_name('PITT'))
participant_dao.insert(participant5)
self.send_consent('P5', email='*****@*****.**')
# A withdrawn participant should be excluded from metrics.
participant6 = Participant(
participantId=6,
biobankId=7,
providerLink=make_primary_provider_link_for_name('PITT')
)
participant_dao.insert(participant6)
self.send_consent('P6', email='*****@*****.**')
participant6.withdrawalStatus=WithdrawalStatus.NO_USE
participant_dao.update(participant6)
self.send_post('Participant/P2/PhysicalMeasurements',
load_measurement_json(2))
self.send_post('Participant/P2/BiobankOrder', load_biobank_order_json(2))
self.submit_questionnaire_response('P1', questionnaire_id,
RACE_WHITE_CODE, 'female', None,
datetime.date(1980, 1, 2))
self.submit_questionnaire_response(
'P2', questionnaire_id, PMI_PREFER_NOT_TO_ANSWER_CODE, 'male', None,
datetime.date(1920, 1, 3))
self.submit_questionnaire_response('P2', questionnaire_id_2, None, None,
'PIIState_VA', None)
self.submit_questionnaire_response('P5', questionnaire_id, None, None,
None, datetime.date(1970, 1, 2))
self.submit_consent_questionnaire_response('P1', questionnaire_id_3,
CONSENT_PERMISSION_NO_CODE)
self.submit_consent_questionnaire_response('P2', questionnaire_id_3,
CONSENT_PERMISSION_YES_CODE)
sample_dao = BiobankStoredSampleDao()
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='abc',
biobankId=2,
biobankOrderIdentifier='KIT',
test='test',
confirmed=TIME))
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='def',
biobankId=3,
biobankOrderIdentifier='KIT',
test='1SAL',
confirmed=TIME))
# Required to update the HPO linkage (and test filtering for P3).
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='xyz',
biobankId=4,
biobankOrderIdentifier='KIT',
test='1SAL',
confirmed=TIME))
