def testDeidentifiedExport_participantIds(self): TableExporter.export_tables('rdr', ['ppi_participant_view'], 'dir', deidentify=True) p1 = self._participant_with_defaults( participantId=1, version=2, biobankId=2, providerLink=make_primary_provider_link_for_name('PITT')) ParticipantDao().insert(p1) p2 = self._participant_with_defaults( participantId=2, version=3, biobankId=3, providerLink=make_primary_provider_link_for_name('PITT')) ParticipantDao().insert(p2) tasks = self.taskqueue_stub.get_filtered_tasks() self.assertEqual(len(tasks), 1) csv_path = deferred.run(tasks[0].payload) with cloudstorage_api.open('/' + csv_path, mode='r') as output: reader = csv.reader(output) rows = list(reader)[1:] self.assertEqual(2, len(rows)) pmi_ids = set([p1.participantId, p2.participantId]) obf_ids = set([row[0] for row in rows]) self.assertFalse( pmi_ids.intersection(obf_ids), 'should be no overlap between pmi_ids and obfuscated IDs') self.assertEquals(2, len(obf_ids))
def test_bad_hpo_insert(self): p = Participant( participantId=1, version=1, biobankId=2, providerLink=make_primary_provider_link_for_name('FOO')) with self.assertRaises(BadRequest): self.dao.insert(p)
def test_bad_hpo_update(self): p = Participant(participantId=1, biobankId=2) time = datetime.datetime(2016, 1, 1) with FakeClock(time): self.dao.insert(p) p.providerLink = make_primary_provider_link_for_name('FOO') with self.assertRaises(BadRequest): self.dao.update(p)
def test_update_wrong_expected_version(self): p = Participant() time = datetime.datetime(2016, 1, 1) with random_ids([1, 2]): with FakeClock(time): self.dao.insert(p) p.version = 2 p.providerLink = make_primary_provider_link_for_name('PITT') time2 = datetime.datetime(2016, 1, 2) with FakeClock(time2): with self.assertRaises(PreconditionFailed): self.dao.update(p)
def test_update_suspend(self): p = Participant() time = datetime.datetime(2016, 1, 1) with random_ids([1, 2]): with FakeClock(time): self.dao.insert(p) p.version = 1 p.suspensionStatus = SuspensionStatus.NO_CONTACT time2 = datetime.datetime(2016, 1, 2) with FakeClock(time2): self.dao.update(p) p2 = self.dao.get(1) expected_participant = self._participant_with_defaults( participantId=1, version=2, biobankId=2, lastModified=time2, signUpTime=time, suspensionStatus=SuspensionStatus.NO_CONTACT, suspensionTime=time2) self.assertEquals(expected_participant.asdict(), p2.asdict()) p.version = 2 p.providerLink = make_primary_provider_link_for_name('PITT') p.suspensionTime = None time3 = datetime.datetime(2016, 1, 3) with FakeClock(time3): self.dao.update(p) # Withdrawal time should get copied over. p2 = self.dao.get(1) expected_participant = self._participant_with_defaults( participantId=1, version=3, biobankId=2, lastModified=time3, signUpTime=time, suspensionStatus=SuspensionStatus.NO_CONTACT, suspensionTime=time2, hpoId=PITT_HPO_ID, providerLink=p2.providerLink) self.assertEquals(expected_participant.asdict(), p2.asdict())
def test_update_no_expected_version_no_ps(self): p = Participant() time = datetime.datetime(2016, 1, 1) with random_ids([1, 2]): with FakeClock(time): self.dao.insert(p) p.providerLink = make_primary_provider_link_for_name('PITT') time2 = datetime.datetime(2016, 1, 2) with FakeClock(time2): self.dao.update(p) # lastModified, hpoId, version is updated on p after being passed in p2 = self.dao.get(1) expected_participant = self._participant_with_defaults( participantId=1, version=2, biobankId=2, lastModified=time2, signUpTime=time, hpoId=PITT_HPO_ID, providerLink=p2.providerLink) self.assertEquals(expected_participant.asdict(), p2.asdict()) self.assertEquals(p.asdict(), p2.asdict()) ps = self.participant_summary_dao.get(1) self.assertIsNone(ps) expected_ph = self._participant_history_with_defaults( participantId=1, biobankId=2, lastModified=time, signUpTime=time) # Updating the participant adds a new ParticipantHistory row. ph = self.participant_history_dao.get([1, 1]) self.assertEquals(expected_ph.asdict(), ph.asdict()) ph2 = self.participant_history_dao.get([1, 2]) expected_ph2 = self._participant_history_with_defaults( participantId=1, version=2, biobankId=2, lastModified=time2, signUpTime=time, hpoId=PITT_HPO_ID, providerLink=p2.providerLink) self.assertEquals(expected_ph2.asdict(), ph2.asdict())
def test_update_withdrawn_hpo_succeeds(self): p = Participant(withdrawalStatus=WithdrawalStatus.NO_USE) time = datetime.datetime(2016, 1, 1) with random_ids([1, 2]): with FakeClock(time): self.dao.insert(p) expected_participant = self._participant_with_defaults( participantId=1, version=1, biobankId=2, lastModified=time, signUpTime=time, withdrawalStatus=WithdrawalStatus.NO_USE) self.assertEquals(expected_participant.asdict(), p.asdict()) p2 = self.dao.get(1) self.assertEquals(p.asdict(), p2.asdict()) p.version = 1 p.providerLink = make_primary_provider_link_for_name('PITT') self.dao.update(p)
def test_update_right_expected_version(self): p = Participant() time = datetime.datetime(2016, 1, 1) with random_ids([1, 2]): with FakeClock(time): self.dao.insert(p) p.version = 1 p.providerLink = make_primary_provider_link_for_name('PITT') time2 = datetime.datetime(2016, 1, 2) with FakeClock(time2): self.dao.update(p) p2 = self.dao.get(1) expected_participant = self._participant_with_defaults( participantId=1, version=2, biobankId=2, lastModified=time2, signUpTime=time, hpoId=PITT_HPO_ID, providerLink=p2.providerLink) self.assertEquals(expected_participant.asdict(), p2.asdict())
def test_update_no_expected_version_with_ps(self): p = Participant() time = datetime.datetime(2016, 1, 1) with random_ids([1, 2]): with FakeClock(time): self.dao.insert(p) p.providerLink = make_primary_provider_link_for_name('PITT') time2 = datetime.datetime(2016, 1, 2) with FakeClock(time2): self.dao.update(p) summary = self.participant_summary(p) self.participant_summary_dao.insert(summary) # lastModified, hpoId, version is updated on p after being passed in p2 = self.dao.get(1) expected_participant = self._participant_with_defaults( participantId=1, version=2, biobankId=2, lastModified=time2, signUpTime=time, hpoId=PITT_HPO_ID, providerLink=p2.providerLink) self.assertEquals(expected_participant.asdict(), p2.asdict()) self.assertEquals(p.asdict(), p2.asdict()) # Updating the participant provider link also updates the HPO ID on the participant summary. ps = self.participant_summary_dao.get(1) expected_ps = self._participant_summary_with_defaults( participantId=1, biobankId=2, signUpTime=time, hpoId=PITT_HPO_ID, lastModified=time2, firstName=summary.firstName, lastName=summary.lastName, email=summary.email) self.assertEquals(expected_ps.asdict(), ps.asdict()) p2_last_modified = p2.lastModified p2.hpoId = 2 self.dao.update(p2) p2_update = self.dao.get(1) self.assertNotEquals(p2_last_modified, p2_update.lastModified) self.assertEquals(p2_update.lastModified, p2.lastModified) expected_ph = self._participant_history_with_defaults( participantId=1, biobankId=2, lastModified=time, signUpTime=time) # And updating the participant adds a new ParticipantHistory row. ph = self.participant_history_dao.get([1, 1]) self.assertEquals(expected_ph.asdict(), ph.asdict()) ph2 = self.participant_history_dao.get([1, 2]) expected_ph2 = self._participant_history_with_defaults( participantId=1, version=2, biobankId=2, lastModified=time2, signUpTime=time, hpoId=PITT_HPO_ID, providerLink=p2.providerLink) self.assertEquals(expected_ph2.asdict(), ph2.asdict())
def _insert(self, participant, first_name=None, last_name=None, hpo_name=None, unconsented=False, time_int=None, time_mem=None, time_fp=None): """ Create a participant in a transient test database. :param participant: Participant object :param first_name: First name :param last_name: Last name :param hpo_name: HPO name (one of PITT or AZ_TUCSON) :param time_int: Time that participant fulfilled INTERESTED criteria :param time_mem: Time that participant fulfilled MEMBER criteria :param time_fp: Time that participant fulfilled FULL_PARTICIPANT criteria :return: Participant object """ if unconsented is True: enrollment_status = None elif time_mem is None: enrollment_status = EnrollmentStatus.INTERESTED elif time_fp is None: enrollment_status = EnrollmentStatus.MEMBER else: enrollment_status = EnrollmentStatus.FULL_PARTICIPANT with FakeClock(time_int): self.dao.insert(participant) participant.providerLink = make_primary_provider_link_for_name(hpo_name) with FakeClock(time_mem): self.dao.update(participant) if enrollment_status is None: return None summary = self.participant_summary(participant) if first_name: summary.firstName = first_name if last_name: summary.lastName = last_name summary.dateOfBirth = datetime.date(1978, 10, 10) summary.enrollmentStatus = enrollment_status summary.hpoId = self.hpo_dao.get_by_name(hpo_name).hpoId if time_mem is not None: with FakeClock(time_mem): summary.consentForElectronicHealthRecordsTime = time_mem if time_fp is not None: with FakeClock(time_fp): summary.consentForElectronicHealthRecordsTime = time_fp summary.questionnaireOnTheBasicsTime = time_fp summary.questionnaireOnLifestyleTime = time_fp summary.questionnaireOnOverallHealthTime = time_fp summary.physicalMeasurementsFinalizedTime = time_fp summary.sampleOrderStatus1ED04Time = time_fp summary.sampleOrderStatus1SALTime = time_fp self.ps_dao.insert(summary) return summary
def _create_data(self): HPODao().insert(HPO(hpoId=AZ_HPO_ID + 1, name='AZ_TUCSON_2')) HPODao().insert(HPO(hpoId=PITT_HPO_ID + 4, name='TEST')) SqlTestBase.setup_codes(ANSWER_FIELD_TO_QUESTION_CODE.values() + [EHR_CONSENT_QUESTION_CODE], code_type=CodeType.QUESTION) SqlTestBase.setup_codes(FIELD_TO_QUESTIONNAIRE_MODULE_CODE.values(), code_type=CodeType.MODULE) # Import codes for white and female, but not male or black. SqlTestBase.setup_codes([ RACE_WHITE_CODE, CONSENT_PERMISSION_YES_CODE, RACE_NONE_OF_THESE_CODE, PMI_PREFER_NOT_TO_ANSWER_CODE, CONSENT_PERMISSION_NO_CODE, 'female', 'PIIState_VA' ], code_type=CodeType.ANSWER) participant_dao = ParticipantDao() questionnaire_id = self.create_questionnaire('questionnaire3.json') questionnaire_id_2 = self.create_questionnaire('questionnaire4.json') questionnaire_id_3 = self.create_questionnaire( 'all_consents_questionnaire.json') with FakeClock(TIME): participant = self._participant_with_defaults( participantId=1, version=2, biobankId=2, providerLink=make_primary_provider_link_for_name('PITT')) participant_dao.insert(participant) self.send_consent('P1', email='*****@*****.**') # Participant 2 starts out unpaired; later gets paired automatically when their physical # measurements come in. participant2 = Participant(participantId=2, biobankId=3) participant_dao.insert(participant2) self.send_consent('P2', email='*****@*****.**') # Test HPO affiliation; this test participant is ignored. participant3 = Participant( participantId=3, biobankId=4, providerLink=make_primary_provider_link_for_name('TEST')) participant_dao.insert(participant3) self.send_consent('P3', email='*****@*****.**') # example.com e-mail; this test participant is ignored, too. participant4 = Participant( participantId=4, biobankId=5, providerLink=make_primary_provider_link_for_name('PITT')) participant_dao.insert(participant4) self.send_consent('P4', email='*****@*****.**') participant5 = Participant( participantId=5, biobankId=6, providerLink=make_primary_provider_link_for_name('PITT')) participant_dao.insert(participant5) self.send_consent('P5', email='*****@*****.**') # A withdrawn participant should be excluded from metrics. participant6 = Participant( participantId=6, biobankId=7, providerLink=make_primary_provider_link_for_name('PITT')) participant_dao.insert(participant6) self.send_consent('P6', email='*****@*****.**') participant6.withdrawalStatus = WithdrawalStatus.NO_USE participant_dao.update(participant6) self.send_post('Participant/P2/PhysicalMeasurements', load_measurement_json(2)) self.send_post('Participant/P2/BiobankOrder', load_biobank_order_json(2)) self.submit_questionnaire_response('P1', questionnaire_id, RACE_WHITE_CODE, 'female', None, datetime.date(1980, 1, 2)) self.submit_questionnaire_response('P2', questionnaire_id, PMI_PREFER_NOT_TO_ANSWER_CODE, 'male', None, datetime.date(1920, 1, 3)) self.submit_questionnaire_response('P2', questionnaire_id_2, None, None, 'PIIState_VA', None) self.submit_questionnaire_response('P5', questionnaire_id, None, None, None, datetime.date(1970, 1, 2)) self.submit_consent_questionnaire_response( 'P1', questionnaire_id_3, CONSENT_PERMISSION_NO_CODE) self.submit_consent_questionnaire_response( 'P2', questionnaire_id_3, CONSENT_PERMISSION_YES_CODE) sample_dao = BiobankStoredSampleDao() sample_dao.insert( BiobankStoredSample(biobankStoredSampleId='abc', biobankId=2, biobankOrderIdentifier='KIT', test='test', confirmed=TIME)) sample_dao.insert( BiobankStoredSample(biobankStoredSampleId='def', biobankId=3, biobankOrderIdentifier='KIT', test='1SAL', confirmed=TIME)) # Required to update the HPO linkage (and test filtering for P3). sample_dao.insert( BiobankStoredSample(biobankStoredSampleId='xyz', biobankId=4, biobankOrderIdentifier='KIT', test='1SAL', confirmed=TIME))
def _make_participant(self, participant, first_name=None, last_name=None, hpo=None, organization=None, unconsented=False, time_int=None, time_study=None, time_mem=None, time_fp=None, time_fp_stored=None, gender_id=None, dob=None, state_id=None): """ Create a participant in a transient test database. Note: copied from ParticipantCountsOverTimeApiTest :param participant: Participant object :param first_name: First name :param last_name: Last name :param time_int: Time that participant fulfilled INTERESTED criteria :param time_mem: Time that participant fulfilled MEMBER criteria :param time_fp: Time that participant fulfilled FULL_PARTICIPANT criteria :return: Participant object """ participant.hpoId = hpo.hpoId participant.organizationId = organization.organizationId if unconsented is True: enrollment_status = None elif time_mem is None: enrollment_status = EnrollmentStatus.INTERESTED elif time_fp is None: enrollment_status = EnrollmentStatus.MEMBER else: enrollment_status = EnrollmentStatus.FULL_PARTICIPANT with FakeClock(time_int): self.dao.insert(participant) participant.providerLink = make_primary_provider_link_for_name( hpo.name) with FakeClock(time_mem): self.dao.update(participant) if enrollment_status is None: return None summary = self.participant_summary(participant) if first_name: summary.firstName = first_name if last_name: summary.lastName = last_name if gender_id: summary.genderIdentityId = gender_id if dob: summary.dateOfBirth = dob else: summary.dateOfBirth = datetime.date(1978, 10, 10) if state_id: summary.stateId = state_id summary.enrollmentStatus = enrollment_status summary.enrollmentStatusMemberTime = time_mem summary.enrollmentStatusCoreOrderedSampleTime = time_fp summary.enrollmentStatusCoreStoredSampleTime = time_fp_stored summary.hpoId = hpo.hpoId summary.organizationId = organization.organizationId if time_study is not None: with FakeClock(time_mem): summary.consentForStudyEnrollment = QuestionnaireStatus.SUBMITTED summary.consentForStudyEnrollmentTime = time_study if time_mem is not None: with FakeClock(time_mem): summary.consentForElectronicHealthRecords = QuestionnaireStatus.SUBMITTED summary.consentForElectronicHealthRecordsTime = time_mem if time_fp is not None: with FakeClock(time_fp): if not summary.consentForElectronicHealthRecords: summary.consentForElectronicHealthRecords = QuestionnaireStatus.SUBMITTED summary.consentForElectronicHealthRecordsTime = time_fp summary.questionnaireOnTheBasicsTime = time_fp summary.questionnaireOnLifestyleTime = time_fp summary.questionnaireOnOverallHealthTime = time_fp summary.physicalMeasurementsFinalizedTime = time_fp summary.physicalMeasurementsTime = time_fp summary.sampleOrderStatus1ED04Time = time_fp summary.sampleOrderStatus1SALTime = time_fp summary.sampleStatus1ED04Time = time_fp summary.sampleStatus1SALTime = time_fp summary.biospecimenOrderTime = time_fp summary.numCompletedBaselinePPIModules = REQUIRED_PPI_MODULE_COUNT self.ps_dao.insert(summary) return summary
def _create_data(self): HPODao().insert(HPO(hpoId=PITT_HPO_ID + 1, name='AZ_TUCSON_2')) HPODao().insert(HPO(hpoId=PITT_HPO_ID + 4, name='TEST')) SqlTestBase.setup_codes(ANSWER_FIELD_TO_QUESTION_CODE.values() + [EHR_CONSENT_QUESTION_CODE], code_type=CodeType.QUESTION) SqlTestBase.setup_codes(FIELD_TO_QUESTIONNAIRE_MODULE_CODE.values(), code_type=CodeType.MODULE) # Import codes for white and female, but not male or black. SqlTestBase.setup_codes([ RACE_WHITE_CODE, CONSENT_PERMISSION_YES_CODE, RACE_NONE_OF_THESE_CODE, PMI_PREFER_NOT_TO_ANSWER_CODE, CONSENT_PERMISSION_NO_CODE, 'female', 'PIIState_VA', PMI_SKIP_CODE ], code_type=CodeType.ANSWER) participant_dao = ParticipantDao() questionnaire_id = self.create_questionnaire('questionnaire3.json') questionnaire_id_2 = self.create_questionnaire('questionnaire4.json') questionnaire_id_3 = self.create_questionnaire( 'all_consents_questionnaire.json') pl_tucson = make_primary_provider_link_for_name('AZ_TUCSON') pl_test = make_primary_provider_link_for_name('TEST') pl_pitt = make_primary_provider_link_for_name('PITT') with FakeClock(TIME): participant = Participant(participantId=1, biobankId=2, providerLink=pl_tucson) participant_dao.insert(participant) self.send_consent('P1', email='*****@*****.**') # Participant 2 starts out unpaired; later gets paired automatically when their physical # measurements come in. participant2 = Participant(participantId=2, biobankId=3) participant_dao.insert(participant2) self.send_consent('P2', email='*****@*****.**') # Test HPO affiliation; this test participant is ignored. participant3 = Participant(participantId=3, biobankId=4, providerLink=pl_test) participant_dao.insert(participant3) self.send_consent('P3', email='*****@*****.**') # example.com e-mail; this test participant is ignored, too. participant4 = Participant(participantId=4, biobankId=5, providerLink=pl_pitt) participant_dao.insert(participant4) self.send_consent('P4', email='*****@*****.**') participant5 = Participant(participantId=5, biobankId=6, providerLink=pl_tucson) participant_dao.insert(participant5) self.send_consent('P5', email='*****@*****.**') participant6 = Participant(participantId=6, biobankId=7, providerLink=pl_tucson) participant_dao.insert(participant6) self.send_consent('P6', email='*****@*****.**') # Participant that starts at PITT but winds up in TEST; should be ignored. participant7 = Participant(participantId=7, biobankId=8, providerLink=pl_pitt) participant_dao.insert(participant7) self.send_consent('P7', email='*****@*****.**') # Participant that re-pairs and then withdraws; should be ignored. participant8 = Participant(participantId=8, biobankId=9, providerLink=pl_pitt) participant_dao.insert(participant8) self.send_consent('P8', email='*****@*****.**') # Participant that has cancelled biobank order should not have order in metrics. participant9 = Participant(participantId=9, biobankId=10, providerLink=pl_pitt) participant_dao.insert(participant9) self.send_consent('P9', email='*****@*****.**') with FakeClock(TIME_2): # FIXME: The test passes, but the following "update" doesn't actually make much sense. The # providerlink is not changed but the HPO ID actually is (at this point in time # `participant.hpoId` evaluates to 4, which is the value given in `unit_test_util.AZ_HPO_ID`). # The original intent of the test is not clear. # This update to participant has no effect, as the HPO ID didn't change. participant = self._participant_with_defaults( participantId=1, version=1, biobankId=2, hpoId=3, # <<<< Had to add hpoId here, default is UNSET_HPO_ID providerLink=pl_tucson) participant_dao.update(participant) participant8.providerLink = pl_tucson participant_dao.update(participant8) self.submit_questionnaire_response('P1', questionnaire_id, race_code=RACE_WHITE_CODE, gender_code='male', state=PMI_SKIP_CODE, date_of_birth=datetime.date( 1980, 1, 2)) self.submit_questionnaire_response( 'P2', questionnaire_id, race_code=RACE_NONE_OF_THESE_CODE, gender_code=None, state=None, date_of_birth=None) self.submit_questionnaire_response('P5', questionnaire_id, race_code=PMI_SKIP_CODE, gender_code=PMI_SKIP_CODE, state=None, date_of_birth=None) self.submit_questionnaire_response('P6', questionnaire_id, race_code=PMI_SKIP_CODE, gender_code=PMI_SKIP_CODE, state=None, date_of_birth=None) self.submit_questionnaire_response('P9', questionnaire_id, race_code=PMI_SKIP_CODE, gender_code=PMI_SKIP_CODE, state=None, date_of_birth=None) with FakeClock(TIME_3): t3 = TIME_3.strftime(TIME_FORMAT) # Re-pair the original participant participant.version = 2 participant.providerLink = pl_pitt participant_dao.update(participant) participant7.providerLink = pl_test participant_dao.update(participant7) participant8.withdrawalStatus = WithdrawalStatus.NO_USE participant8.withdrawalReason = WithdrawalReason.TEST participant8.withdrawalReasonJustification = 'test account' participant_dao.update(participant8) self.send_post('Participant/P2/PhysicalMeasurements', load_measurement_json(2, t3)) self.send_post('Participant/P2/BiobankOrder', load_biobank_order_json(2)) self.submit_questionnaire_response('P1', questionnaire_id, race_code='black', gender_code='female', state=None, date_of_birth=datetime.date( 1980, 1, 3)) self.submit_questionnaire_response( 'P2', questionnaire_id, race_code=None, gender_code=PMI_PREFER_NOT_TO_ANSWER_CODE, state=None, date_of_birth=None) self.submit_questionnaire_response('P2', questionnaire_id_2, race_code=None, gender_code=None, state='PIIState_VA', date_of_birth=None) self.submit_questionnaire_response('P6', questionnaire_id_2, race_code=None, gender_code=None, state='PIIState_VA', date_of_birth=None) self.submit_consent_questionnaire_response( 'P1', questionnaire_id_3, CONSENT_PERMISSION_NO_CODE) self.submit_consent_questionnaire_response( 'P2', questionnaire_id_3, CONSENT_PERMISSION_YES_CODE) self.submit_consent_questionnaire_response( 'P6', questionnaire_id_3, CONSENT_PERMISSION_YES_CODE) path = 'Participant/P9/BiobankOrder' new_order = self.send_post(path, load_biobank_order_json(9)) path = path + '/' + new_order['id'] self.send_get(path) self.send_patch(path, request_data=cancel_biobank_order(), headers={'If-Match': 'W/"1"'}) sample_dao = BiobankStoredSampleDao() sample_dao.insert( BiobankStoredSample(biobankStoredSampleId='abc', biobankId=2, biobankOrderIdentifier='KIT', test='test', confirmed=TIME_2)) sample_dao.insert( BiobankStoredSample(biobankStoredSampleId='def', biobankId=3, biobankOrderIdentifier='KIT', test='1SAL', confirmed=TIME_2)) sample_dao.insert( BiobankStoredSample(biobankStoredSampleId='xyz', biobankId=4, biobankOrderIdentifier='KIT', test='1SAL', confirmed=TIME_2)) # participant 6 withdrawl shouldn't be in csv's. participant6.withdrawalStatus = WithdrawalStatus.NO_USE participant_dao.update(participant6)