def insert_biobank_order(self, pid, resource): obj = BiobankOrder() obj.participantId = long(pid) obj.created = clock.CLOCK.now() obj.created = datetime.datetime.now() obj.orderStatus = BiobankOrderStatus.UNSET obj.biobankOrderId = resource['biobankOrderId'] test = self.get(resource['id']) obj.dvOrders = [test] bod = BiobankOrderDao() obj.samples = [ BiobankOrderedSample(test='1SAL2', processingRequired=False, description=u'salivary pilot kit') ] self._add_identifiers_and_main_id(obj, ObjectView(resource)) bod.insert(obj)
class BigQuerySyncDaoTest(SqlTestBase): TIME_1 = datetime.datetime(2018, 9, 20, 5, 49, 11) TIME_2 = datetime.datetime(2018, 9, 24, 14, 21, 01) site = None hpo = None participant = None summary = None pm_json = None pm = None bio_order = None def setUp(self): super(BigQuerySyncDaoTest, self).setUp(use_mysql=True, with_consent_codes=True) self.dao = ParticipantDao() with self.dao.session() as session: self.site = session.query(Site).filter( Site.googleGroup == 'hpo-site-monroeville').first() self.hpo = session.query(HPO).filter(HPO.name == 'PITT').first() with clock.FakeClock(self.TIME_1): self.participant = Participant(participantId=123, biobankId=555) self.participant.hpoId = self.hpo.hpoId self.participant.siteId = self.site.siteId self.dao.insert(self.participant) ps = ParticipantSummary( participantId=123, biobankId=555, firstName='john', lastName='doe', withdrawalStatus=WithdrawalStatus.NOT_WITHDRAWN, suspensionStatus=SuspensionStatus.NOT_SUSPENDED) ps.hpoId = self.hpo.hpoId ps.siteId = self.site.siteId self.summary = ParticipantSummaryDao().insert(ps) self.pm_json = json.dumps( load_measurement_json(self.participant.participantId, self.TIME_1.isoformat())) self.pm = PhysicalMeasurementsDao().insert( self._make_physical_measurements()) with clock.FakeClock(self.TIME_2): self.dao = BiobankOrderDao() self.bio_order = BiobankOrderDao().insert( self._make_biobank_order( participantId=self.participant.participantId)) def _make_physical_measurements(self, **kwargs): """Makes a new PhysicalMeasurements (same values every time) with valid/complete defaults. Kwargs pass through to PM constructor, overriding defaults. """ for k, default_value in (('physicalMeasurementsId', 1), ('participantId', self.participant.participantId), ('resource', self.pm_json), ('createdSiteId', self.site.siteId), ('finalizedSiteId', self.site.siteId)): if k not in kwargs: kwargs[k] = default_value return PhysicalMeasurements(**kwargs) def _make_biobank_order(self, **kwargs): """Makes a new BiobankOrder (same values every time) with valid/complete defaults. Kwargs pass through to BiobankOrder constructor, overriding defaults. """ for k, default_value in (('biobankOrderId', '1'), ('created', clock.CLOCK.now()), ('participantId', self.participant.participantId), ('sourceSiteId', 1), ('sourceUsername', '*****@*****.**'), ('collectedSiteId', 1), ('collectedUsername', '*****@*****.**'), ('processedSiteId', 1), ('processedUsername', '*****@*****.**'), ('finalizedSiteId', 2), ('finalizedUsername', '*****@*****.**'), ('identifiers', [ BiobankOrderIdentifier(system='a', value='c') ]), ('samples', [ BiobankOrderedSample( biobankOrderId='1', test=BIOBANK_TESTS[0], description=u'description', finalized=self.TIME_1, processingRequired=True) ])): if k not in kwargs: kwargs[k] = default_value return BiobankOrder(**kwargs) def test_participant_summary_gen(self): gen = BQParticipantSummaryGenerator() ps_json = gen.make_bqrecord(self.participant.participantId) self.assertIsNotNone(ps_json) self.assertEqual(ps_json.sign_up_time, self.TIME_1) self.assertEqual(ps_json['pm'][0]['pm_finalized_site'], 'hpo-site-monroeville') self.assertEqual(ps_json.suspension_status, 'NOT_SUSPENDED') self.assertEqual(ps_json.withdrawal_status, 'NOT_WITHDRAWN') self.assertEqual(ps_json['pm'][0]['pm_status'], 'COMPLETED')
class MySqlReconciliationTest(FlaskTestBase): """Biobank samples pipeline tests requiring slower MySQL (not SQLite).""" def setUp(self): super(MySqlReconciliationTest, self).setUp(use_mysql=True) self.participant_dao = ParticipantDao() self.summary_dao = ParticipantSummaryDao() self.order_dao = BiobankOrderDao() self.sample_dao = BiobankStoredSampleDao() def _withdraw(self, participant, withdrawal_time): with FakeClock(withdrawal_time): participant.withdrawalStatus = WithdrawalStatus.NO_USE self.participant_dao.update(participant) def _insert_participant(self, race_codes=[]): participant = self.participant_dao.insert(Participant()) # satisfies the consent requirement self.summary_dao.insert(self.participant_summary(participant)) if race_codes: self._submit_race_questionnaire_response( to_client_participant_id(participant.participantId), race_codes) return participant def _insert_order(self, participant, order_id, tests, order_time, finalized_tests=None, kit_id=None, tracking_number=None): order = BiobankOrder(biobankOrderId=order_id, participantId=participant.participantId, sourceSiteId=1, finalizedSiteId=1, finalizedUsername='******', created=order_time, samples=[]) id_1 = BiobankOrderIdentifier( system="https://orders.mayomedicallaboratories.com", value=order_id) id_2 = BiobankOrderIdentifier(system="https://www.pmi-ops.org", value='O%s' % order_id) order.identifiers.append(id_1) order.identifiers.append(id_2) if kit_id: order.identifiers.append( BiobankOrderIdentifier(system=_KIT_ID_SYSTEM, value=kit_id)) if tracking_number: order.identifiers.append( BiobankOrderIdentifier(system=_TRACKING_NUMBER_SYSTEM, value=tracking_number)) for test_code in tests: finalized_time = order_time if finalized_tests and not test_code in finalized_tests: finalized_time = None order.samples.append( BiobankOrderedSample(biobankOrderId=order.biobankOrderId, test=test_code, description=u'test', processingRequired=False, collected=order_time, processed=order_time, finalized=finalized_time)) return self.order_dao.insert(order) def _insert_samples(self, participant, tests, sample_ids, confirmed_time, created_time): for test_code, sample_id in zip(tests, sample_ids): self.sample_dao.insert( BiobankStoredSample(biobankStoredSampleId=sample_id, biobankId=participant.biobankId, test=test_code, confirmed=confirmed_time, created=created_time)) def _submit_race_questionnaire_response(self, participant_id, race_answers): code_answers = [] for answer in race_answers: _add_code_answer(code_answers, "race", answer) qr = make_questionnaire_response_json(participant_id, self._questionnaire_id, code_answers=code_answers) self.send_post('Participant/%s/QuestionnaireResponse' % participant_id, qr) def test_reconciliation_query(self): self.setup_codes([RACE_QUESTION_CODE], CodeType.QUESTION) self.setup_codes([RACE_AIAN_CODE, RACE_WHITE_CODE], CodeType.ANSWER) self._questionnaire_id = self.create_questionnaire( 'questionnaire3.json') # MySQL and Python sub-second rounding differs, so trim micros from generated times. order_time = clock.CLOCK.now().replace(microsecond=0) old_order_time = order_time - datetime.timedelta(days=10) within_24_hours = order_time + datetime.timedelta(hours=23) old_within_24_hours = old_order_time + datetime.timedelta(hours=23) late_time = order_time + datetime.timedelta(hours=25) old_late_time = old_order_time + datetime.timedelta(hours=25) file_time = order_time + datetime.timedelta( hours=23) + datetime.timedelta(minutes=59) two_days_ago = file_time - datetime.timedelta(days=2) # On time, recent order and samples; shows up in rx p_on_time = self._insert_participant() # Extra samples ordered now aren't considered missing or late. self._insert_order(p_on_time, 'GoodOrder', BIOBANK_TESTS[:4], order_time, finalized_tests=BIOBANK_TESTS[:3], kit_id='kit1', tracking_number='t1') self._insert_samples(p_on_time, BIOBANK_TESTS[:2], ['GoodSample1', 'GoodSample2'], within_24_hours, within_24_hours - datetime.timedelta(hours=1)) # On time order and samples from 10 days ago; shows up in rx p_old_on_time = self._insert_participant(race_codes=[RACE_AIAN_CODE]) # Old missing samples from 10 days ago don't show up in missing or late. self._insert_order(p_old_on_time, 'OldGoodOrder', BIOBANK_TESTS[:3], old_order_time, kit_id='kit2') self._insert_samples(p_old_on_time, BIOBANK_TESTS[:2], ['OldGoodSample1', 'OldGoodSample2'], old_within_24_hours, old_within_24_hours - datetime.timedelta(hours=1)) # Late, recent order and samples; shows up in rx and late. (But not missing, as it hasn't been # 24 hours since the order.) p_late_and_missing = self._insert_participant() # Extra missing sample doesn't show up as missing as it hasn't been 24 hours yet. o_late_and_missing = self._insert_order(p_late_and_missing, 'SlowOrder', BIOBANK_TESTS[:3], order_time) self._insert_samples(p_late_and_missing, [BIOBANK_TESTS[0]], ['LateSample'], late_time, late_time - datetime.timedelta(minutes=59)) # Late order and samples from 10 days ago; shows up in rx (but not missing, as it was too # long ago. p_old_late_and_missing = self._insert_participant() self._insert_order(p_old_late_and_missing, 'OldSlowOrder', BIOBANK_TESTS[:2], old_order_time) self._insert_samples(p_old_late_and_missing, [BIOBANK_TESTS[0]], ['OldLateSample'], old_late_time, old_late_time - datetime.timedelta(minutes=59)) # Order with missing sample from 2 days ago; shows up in rx and missing. p_two_days_missing = self._insert_participant() # The third test doesn't wind up in missing, as it was never finalized. self._insert_order(p_two_days_missing, 'TwoDaysMissingOrder', BIOBANK_TESTS[:3], two_days_ago, finalized_tests=BIOBANK_TESTS[:2]) # Recent samples with no matching order; shows up in missing. p_extra = self._insert_participant(race_codes=[RACE_WHITE_CODE]) self._insert_samples(p_extra, [BIOBANK_TESTS[-1]], ['NobodyOrderedThisSample'], order_time, order_time - datetime.timedelta(minutes=59)) # Old samples with no matching order; shows up in rx. p_old_extra = self._insert_participant(race_codes=[RACE_AIAN_CODE]) self._insert_samples(p_old_extra, [BIOBANK_TESTS[-1]], ['OldNobodyOrderedThisSample'], old_order_time, old_order_time - datetime.timedelta(hours=1)) # Withdrawn participants don't show up in any reports except withdrawal report. p_withdrawn_old_on_time = self._insert_participant( race_codes=[RACE_AIAN_CODE]) # This updates the version of the participant and its HPO ID. self._insert_order(p_withdrawn_old_on_time, 'OldWithdrawnGoodOrder', BIOBANK_TESTS[:2], old_order_time) p_withdrawn_old_on_time = self.participant_dao.get( p_withdrawn_old_on_time.participantId) self._insert_samples( p_withdrawn_old_on_time, BIOBANK_TESTS[:2], ['OldWithdrawnGoodSample1', 'OldWithdrawnGoodSample2'], old_within_24_hours, old_within_24_hours - datetime.timedelta(hours=1)) self._withdraw(p_withdrawn_old_on_time, within_24_hours) p_withdrawn_late_and_missing = self._insert_participant() self._insert_order(p_withdrawn_late_and_missing, 'WithdrawnSlowOrder', BIOBANK_TESTS[:2], order_time) self._insert_samples(p_withdrawn_late_and_missing, [BIOBANK_TESTS[0]], ['WithdrawnLateSample'], late_time, late_time - datetime.timedelta(minutes=59)) p_withdrawn_late_and_missing = (self.participant_dao.get( p_withdrawn_late_and_missing.participantId)) self._withdraw(p_withdrawn_late_and_missing, within_24_hours) p_withdrawn_old_late_and_missing = self._insert_participant() self._insert_order(p_withdrawn_old_late_and_missing, 'WithdrawnOldSlowOrder', BIOBANK_TESTS[:2], old_order_time) self._insert_samples(p_withdrawn_old_late_and_missing, [BIOBANK_TESTS[0]], ['WithdrawnOldLateSample'], old_late_time, old_late_time - datetime.timedelta(minutes=59)) p_withdrawn_old_late_and_missing = (self.participant_dao.get( p_withdrawn_old_late_and_missing.participantId)) self._withdraw(p_withdrawn_old_late_and_missing, old_late_time) p_withdrawn_extra = self._insert_participant( race_codes=[RACE_WHITE_CODE]) self._insert_samples(p_withdrawn_extra, [BIOBANK_TESTS[-1]], ['WithdrawnNobodyOrderedThisSample'], order_time, order_time - datetime.timedelta(hours=1)) self._withdraw(p_withdrawn_extra, within_24_hours) p_withdrawn_old_extra = self._insert_participant( race_codes=[RACE_AIAN_CODE]) self._insert_samples(p_withdrawn_old_extra, [BIOBANK_TESTS[-1]], ['WithdrawnOldNobodyOrderedThisSample'], old_order_time, old_order_time - datetime.timedelta(hours=1)) self._withdraw(p_withdrawn_old_extra, within_24_hours) p_withdrawn_race_change = self._insert_participant( race_codes=[RACE_AIAN_CODE]) p_withdrawn_race_change_id = to_client_participant_id( p_withdrawn_race_change.participantId) self._submit_race_questionnaire_response(p_withdrawn_race_change_id, [RACE_WHITE_CODE]) self._withdraw(p_withdrawn_race_change, within_24_hours) # for the same participant/test, 3 orders sent and only 2 samples received. Shows up in both # missing (we are missing one sample) and late (the two samples that were received were after # 24 hours.) p_repeated = self._insert_participant() for repetition in xrange(3): self._insert_order( p_repeated, 'RepeatedOrder%d' % repetition, [BIOBANK_TESTS[0]], two_days_ago + datetime.timedelta(hours=repetition)) if repetition != 2: self._insert_samples( p_repeated, [BIOBANK_TESTS[0]], ['RepeatedSample%d' % repetition], within_24_hours + datetime.timedelta(hours=repetition), within_24_hours + datetime.timedelta(hours=repetition - 1)) received, late, missing, withdrawals = 'rx.csv', 'late.csv', 'missing.csv', 'withdrawals.csv' exporter = InMemorySqlExporter(self) biobank_samples_pipeline._query_and_write_reports( exporter, file_time, received, late, missing, withdrawals) exporter.assertFilesEqual((received, late, missing, withdrawals)) # sent-and-received: 4 on-time, 2 late, none of the missing/extra/repeated ones; # includes orders/samples from more than 7 days ago exporter.assertRowCount(received, 6) exporter.assertColumnNamesEqual(received, _CSV_COLUMN_NAMES) row = exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_on_time.biobankId), 'sent_test': BIOBANK_TESTS[0], 'received_test': BIOBANK_TESTS[0] }) # Also check the values of all remaining fields on one row. self.assertEquals(row['source_site_name'], 'Monroeville Urgent Care Center') self.assertEquals(row['source_site_consortium'], 'Pittsburgh') self.assertEquals(row['source_site_mayolink_client_number'], '7035769') self.assertEquals(row['source_site_hpo'], 'PITT') self.assertEquals(row['source_site_hpo_type'], 'HPO') self.assertEquals(row['finalized_site_name'], 'Monroeville Urgent Care Center') self.assertEquals(row['finalized_site_consortium'], 'Pittsburgh') self.assertEquals(row['finalized_site_mayolink_client_number'], '7035769') self.assertEquals(row['finalized_site_hpo'], 'PITT') self.assertEquals(row['finalized_site_hpo_type'], 'HPO') self.assertEquals(row['finalized_username'], '*****@*****.**') self.assertEquals(row['sent_finalized_time'], database_utils.format_datetime(order_time)) self.assertEquals(row['sent_collection_time'], database_utils.format_datetime(order_time)) self.assertEquals(row['sent_processed_time'], database_utils.format_datetime(order_time)) self.assertEquals(row['received_time'], database_utils.format_datetime(within_24_hours)) self.assertEquals( row['Sample Family Create Date'], database_utils.format_datetime(within_24_hours - datetime.timedelta(hours=1))) self.assertEquals(row['sent_count'], '1') self.assertEquals(row['received_count'], '1') self.assertEquals(row['sent_order_id'], 'OGoodOrder') self.assertEquals(row['received_sample_id'], 'GoodSample1') self.assertEquals(row['biospecimen_kit_id'], 'kit1') self.assertEquals(row['fedex_tracking_number'], 't1') # the other sent-and-received rows exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_on_time.biobankId), 'sent_test': BIOBANK_TESTS[1] }) exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id( p_late_and_missing.biobankId), 'sent_test': BIOBANK_TESTS[0] }) exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_old_on_time.biobankId), 'sent_test': BIOBANK_TESTS[0] }) exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_old_on_time.biobankId), 'sent_test': BIOBANK_TESTS[1] }) exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_old_late_and_missing.biobankId), 'sent_test': BIOBANK_TESTS[0] }) # sent-and-received: 2 late; don't include orders/samples from more than 7 days ago exporter.assertRowCount(late, 2) exporter.assertColumnNamesEqual(late, _CSV_COLUMN_NAMES) exporter.assertHasRow( late, { 'biobank_id': to_client_biobank_id( p_late_and_missing.biobankId), 'sent_order_id': 'O%s' % o_late_and_missing.biobankOrderId, 'elapsed_hours': '24' }) exporter.assertHasRow( late, { 'biobank_id': to_client_biobank_id(p_repeated.biobankId), 'elapsed_hours': '45' }) # orders/samples where something went wrong; don't include orders/samples from more than 7 # days ago, or where 24 hours hasn't elapsed yet. exporter.assertRowCount(missing, 4) exporter.assertColumnNamesEqual(missing, _CSV_COLUMN_NAMES) # sample received, nothing ordered exporter.assertHasRow( missing, { 'biobank_id': to_client_biobank_id(p_extra.biobankId), 'sent_order_id': '' }) # order received, no sample exporter.assertHasRow( missing, { 'biobank_id': to_client_biobank_id( p_two_days_missing.biobankId), 'sent_order_id': 'OTwoDaysMissingOrder', 'sent_test': BIOBANK_TESTS[0] }) exporter.assertHasRow( missing, { 'biobank_id': to_client_biobank_id( p_two_days_missing.biobankId), 'sent_order_id': 'OTwoDaysMissingOrder', 'sent_test': BIOBANK_TESTS[1] }) # 3 orders sent, only 2 received multi_sample_row = exporter.assertHasRow( missing, { 'biobank_id': to_client_biobank_id(p_repeated.biobankId), 'sent_count': '3', 'received_count': '2' }) # Also verify the comma-joined fields of the row with multiple orders/samples. self.assertItemsEqual( multi_sample_row['sent_order_id'].split(','), ['ORepeatedOrder1', 'ORepeatedOrder0', 'ORepeatedOrder2']) self.assertItemsEqual( multi_sample_row['received_sample_id'].split(','), ['RepeatedSample0', 'RepeatedSample1']) # We don't include the old withdrawal. exporter.assertRowCount(withdrawals, 5) exporter.assertHasRow( withdrawals, { 'biobank_id': to_client_biobank_id(p_withdrawn_old_on_time.biobankId), 'withdrawal_time': database_utils.format_datetime(within_24_hours), 'is_native_american': 'Y' }) exporter.assertHasRow( withdrawals, { 'biobank_id': to_client_biobank_id(p_withdrawn_late_and_missing.biobankId), 'withdrawal_time': database_utils.format_datetime(within_24_hours), 'is_native_american': 'N' }) exporter.assertHasRow( withdrawals, { 'biobank_id': to_client_biobank_id( p_withdrawn_extra.biobankId), 'withdrawal_time': database_utils.format_datetime(within_24_hours), 'is_native_american': 'N' }) exporter.assertHasRow( withdrawals, { 'biobank_id': to_client_biobank_id( p_withdrawn_old_extra.biobankId), 'withdrawal_time': database_utils.format_datetime(within_24_hours), 'is_native_american': 'Y' }) exporter.assertHasRow( withdrawals, { 'biobank_id': to_client_biobank_id(p_withdrawn_race_change.biobankId), 'withdrawal_time': database_utils.format_datetime(within_24_hours), 'is_native_american': 'N' })
class BiobankOrderDaoTest(SqlTestBase): _A_TEST = BIOBANK_TESTS[0] _B_TEST = BIOBANK_TESTS[1] TIME_1 = datetime.datetime(2018, 9, 20, 5, 49, 11) TIME_2 = datetime.datetime(2018, 9, 21, 8, 49, 37) def setUp(self): super(BiobankOrderDaoTest, self).setUp(use_mysql=True) self.participant = Participant(participantId=123, biobankId=555) ParticipantDao().insert(self.participant) self.dao = BiobankOrderDao() def _make_biobank_order(self, **kwargs): """Makes a new BiobankOrder (same values every time) with valid/complete defaults. Kwargs pass through to BiobankOrder constructor, overriding defaults. """ for k, default_value in ( ('biobankOrderId', '1'), ('created', clock.CLOCK.now()), ('participantId', self.participant.participantId), ('sourceSiteId', 1), ('sourceUsername', '*****@*****.**'), ('collectedSiteId', 1), ('collectedUsername', '*****@*****.**'), ('processedSiteId', 1), ('processedUsername', '*****@*****.**'), ('finalizedSiteId', 2), ('finalizedUsername', '*****@*****.**'), ('identifiers', [BiobankOrderIdentifier(system='a', value='c')]), ('samples', [BiobankOrderedSample( biobankOrderId='1', test=self._A_TEST, finalized=self.TIME_2, description='description', processingRequired=True)])): if k not in kwargs: kwargs[k] = default_value return BiobankOrder(**kwargs) @staticmethod def _get_cancel_patch(): return { "amendedReason": 'I messed something up :( ', "cancelledInfo": { "author": { "system": "https://www.pmi-ops.org/healthpro-username", "value": "*****@*****.**" }, "site": { "system": "https://www.pmi-ops.org/site-id", "value": "hpo-site-monroeville" } }, "status": "cancelled" } @staticmethod def _get_restore_patch(): return { "amendedReason": 'I didn"t mess something up :( ', "restoredInfo": { "author": { "system": "https://www.pmi-ops.org/healthpro-username", "value": "*****@*****.**" }, "site": { "system": "https://www.pmi-ops.org/site-id", "value": "hpo-site-monroeville" } }, "status": "restored" } @staticmethod def _get_amended_info(order): amendment = dict(amendedReason='I had to change something', amendedInfo={ "author": { "system": "https://www.pmi-ops.org/healthpro-username", "value": "*****@*****.**" }, "site": { "system": "https://www.pmi-ops.org/site-id", "value": "hpo-site-monroeville" } }) order.amendedReason = amendment['amendedReason'] order.amendedInfo = amendment['amendedInfo'] return order def test_bad_participant(self): with self.assertRaises(BadRequest): self.dao.insert(self._make_biobank_order(participantId=999)) def test_from_json(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) order_json = load_biobank_order_json(self.participant.participantId) order = BiobankOrderDao().from_client_json(order_json, participant_id=self.participant.participantId) self.assertEquals(1, order.sourceSiteId) self.assertEquals('*****@*****.**', order.sourceUsername) self.assertEquals(1, order.collectedSiteId) self.assertEquals('*****@*****.**', order.collectedUsername) self.assertEquals(1, order.processedSiteId) self.assertEquals('*****@*****.**', order.processedUsername) self.assertEquals(2, order.finalizedSiteId) self.assertEquals('*****@*****.**', order.finalizedUsername) # testing finalized_time samples_finalized_time = None for sample in order_json['samples']: samples_finalized_time = parse_date(sample['finalized']) break self.assertEquals(samples_finalized_time, order.finalizedTime) def test_to_json(self): order = self._make_biobank_order() order_json = self.dao.to_client_json(order) expected_order_json = load_biobank_order_json(self.participant.participantId) for key in ('createdInfo', 'collectedInfo', 'processedInfo', 'finalizedInfo'): self.assertEquals(expected_order_json[key], order_json.get(key)) def test_duplicate_insert_ok(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) order_1 = self.dao.insert(self._make_biobank_order(created=self.TIME_1)) order_2 = self.dao.insert(self._make_biobank_order(created=self.TIME_1)) self.assertEquals(order_1.asdict(), order_2.asdict()) def test_same_id_different_identifier_not_ok(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) self.dao.insert(self._make_biobank_order( identifiers=[BiobankOrderIdentifier(system='a', value='b')])) with self.assertRaises(Conflict): self.dao.insert(self._make_biobank_order( identifiers=[BiobankOrderIdentifier(system='a', value='c')])) def test_reject_used_identifier(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) self.dao.insert(self._make_biobank_order( biobankOrderId='1', identifiers=[BiobankOrderIdentifier(system='a', value='b')])) with self.assertRaises(BadRequest): self.dao.insert(self._make_biobank_order( biobankOrderId='2', identifiers=[BiobankOrderIdentifier(system='a', value='b')])) def test_order_for_withdrawn_participant_fails(self): self.participant.withdrawalStatus = WithdrawalStatus.NO_USE ParticipantDao().update(self.participant) ParticipantSummaryDao().insert(self.participant_summary(self.participant)) with self.assertRaises(Forbidden): self.dao.insert(self._make_biobank_order(participantId=self.participant.participantId)) def test_get_for_withdrawn_participant_fails(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) self.dao.insert(self._make_biobank_order( biobankOrderId='1', participantId=self.participant.participantId)) self.participant.version += 1 self.participant.withdrawalStatus = WithdrawalStatus.NO_USE ParticipantDao().update(self.participant) with self.assertRaises(Forbidden): self.dao.get(1) def test_store_invalid_test(self): with self.assertRaises(BadRequest): self.dao.insert(self._make_biobank_order( samples=[BiobankOrderedSample( test='InvalidTestName', processingRequired=True, description=u'tested it')])) def test_cancelling_an_order(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) order_1 = self.dao.insert(self._make_biobank_order()) cancelled_request = self._get_cancel_patch() self.assertEqual(order_1.version, 1) self.assertEqual(order_1.orderStatus, None) updated_order = self.dao.update_with_patch(order_1.biobankOrderId, cancelled_request, order_1.version) self.assertEqual(updated_order.version, 2) self.assertEqual(updated_order.cancelledUsername, '*****@*****.**') self.assertEqual(updated_order.orderStatus, BiobankOrderStatus.CANCELLED) self.assertEqual(updated_order.amendedReason, cancelled_request['amendedReason']) ps_dao = ParticipantSummaryDao().get(self.participant.participantId) self.assertEqual(ps_dao.biospecimenFinalizedSiteId, None) def test_cancelled_order_removes_from_participant_summary(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) samples = [BiobankOrderedSample( test=self._B_TEST, processingRequired=True, description=u'new sample')] biobank_order_id = 2 with clock.FakeClock(self.TIME_1): order_1 = self.dao.insert(self._make_biobank_order()) with clock.FakeClock(self.TIME_2): self.dao.insert(self._make_biobank_order(samples=samples, biobankOrderId=biobank_order_id, identifiers=[ BiobankOrderIdentifier(system='z', value='x')])) cancelled_request = self._get_cancel_patch() ps_dao = ParticipantSummaryDao().get(self.participant.participantId) self.assertEqual(ps_dao.sampleOrderStatus1ED10, OrderStatus.FINALIZED) self.assertEqual(ps_dao.sampleOrderStatus1ED10Time, self.TIME_2) self.assertEqual(ps_dao.sampleOrderStatus2ED10, OrderStatus.CREATED) self.assertEqual(ps_dao.sampleOrderStatus2ED10Time, self.TIME_2) self.dao.update_with_patch(order_1.biobankOrderId, cancelled_request, order_1.version) ps_dao = ParticipantSummaryDao().get(self.participant.participantId) self.assertEqual(ps_dao.sampleOrderStatus1ED10, None) self.assertEqual(ps_dao.sampleOrderStatus1ED10Time, None) # should not remove the other order self.assertEqual(ps_dao.sampleOrderStatus2ED10, OrderStatus.CREATED) self.assertEqual(ps_dao.sampleOrderStatus2ED10Time, self.TIME_2) self.assertEqual(ps_dao.biospecimenCollectedSiteId, 1) self.assertEqual(ps_dao.biospecimenFinalizedSiteId, 2) self.assertEqual(ps_dao.biospecimenProcessedSiteId, 1) self.assertEqual(ps_dao.biospecimenStatus, OrderStatus.FINALIZED) def test_restoring_an_order_gets_to_participant_summary(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) order_1 = self.dao.insert(self._make_biobank_order()) cancelled_request = self._get_cancel_patch() cancelled_order = self.dao.update_with_patch(order_1.biobankOrderId, cancelled_request, order_1.version) restore_request = self._get_restore_patch() self.dao.update_with_patch(order_1.biobankOrderId, restore_request, cancelled_order.version) ps_dao = ParticipantSummaryDao().get(self.participant.participantId) self.assertEqual(ps_dao.sampleOrderStatus1ED10, OrderStatus.FINALIZED) self.assertEqual(ps_dao.biospecimenStatus, OrderStatus.FINALIZED) self.assertEqual(ps_dao.biospecimenSourceSiteId, 1) self.assertEqual(ps_dao.biospecimenCollectedSiteId, 1) self.assertEqual(ps_dao.biospecimenFinalizedSiteId, 2) self.assertEqual(ps_dao.biospecimenProcessedSiteId, 1) def test_amending_order_participant_summary(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) order_1 = self.dao.insert(self._make_biobank_order()) amended_info = self._get_amended_info(order_1) amended_info.sourceSiteId = 2 samples = [BiobankOrderedSample( test=self._B_TEST, processingRequired=True, description=u'new sample')] amended_info.samples = samples with self.dao.session() as session: self.dao._do_update(session, amended_info, order_1) amended_order = self.dao.get(1) self.assertEqual(amended_order.version, 2) ps_dao = ParticipantSummaryDao().get(self.participant.participantId) self.assertEqual(ps_dao.sampleOrderStatus2ED10, OrderStatus.CREATED) self.assertEqual(ps_dao.sampleOrderStatus1ED10, OrderStatus.FINALIZED) self.assertEqual(ps_dao.numberDistinctVisits, 1) def test_cancelling_an_order_missing_reason(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) order_1 = self.dao.insert(self._make_biobank_order()) cancelled_request = self._get_cancel_patch() del cancelled_request['amendedReason'] with self.assertRaises(BadRequest): self.dao.update_with_patch(order_1.biobankOrderId, cancelled_request, order_1.version) def test_cancelling_an_order_missing_info(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) order_1 = self.dao.insert(self._make_biobank_order()) # missing cancelled info cancelled_request = self._get_cancel_patch() del cancelled_request['cancelledInfo'] with self.assertRaises(BadRequest): self.dao.update_with_patch(order_1.biobankOrderId, cancelled_request, order_1.version) # missing site cancelled_request = self._get_cancel_patch() del cancelled_request['cancelledInfo']['site'] with self.assertRaises(BadRequest): self.dao.update_with_patch(order_1.biobankOrderId, cancelled_request, order_1.version) # missing author cancelled_request = self._get_cancel_patch() del cancelled_request['cancelledInfo']['author'] with self.assertRaises(BadRequest): self.dao.update_with_patch(order_1.biobankOrderId, cancelled_request, order_1.version) # missing status cancelled_request = self._get_cancel_patch() del cancelled_request['status'] with self.assertRaises(BadRequest): self.dao.update_with_patch(order_1.biobankOrderId, cancelled_request, order_1.version) def test_restoring_an_order(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) order_1 = self.dao.insert(self._make_biobank_order()) cancelled_request = self._get_cancel_patch() cancelled_order = self.dao.update_with_patch(order_1.biobankOrderId, cancelled_request, order_1.version) ps_dao = ParticipantSummaryDao().get(self.participant.participantId) self.assertEqual(ps_dao.numberDistinctVisits, 0) restore_request = self._get_restore_patch() restored_order = self.dao.update_with_patch(order_1.biobankOrderId, restore_request, cancelled_order.version) self.assertEqual(restored_order.version, 3) self.assertEqual(restored_order.restoredUsername, '*****@*****.**') self.assertEqual(restored_order.orderStatus, BiobankOrderStatus.UNSET) self.assertEqual(restored_order.amendedReason, restore_request['amendedReason']) ps_dao = ParticipantSummaryDao().get(self.participant.participantId) self.assertEqual(ps_dao.numberDistinctVisits, 1) def test_amending_an_order(self): ParticipantSummaryDao().insert(self.participant_summary(self.participant)) order_1 = self.dao.insert(self._make_biobank_order()) amended_info = self._get_amended_info(order_1) amended_info.sourceSiteId = 2 with self.dao.session() as session: self.dao._do_update(session, order_1, amended_info) amended_order = self.dao.get(1) self.assertEqual(amended_order.version, 2) self.assertEqual(amended_order.orderStatus, BiobankOrderStatus.AMENDED) self.assertEqual(amended_order.amendedReason, 'I had to change something') self.assertEqual(amended_order.amendedSiteId, 1) self.assertEqual(amended_order.amendedUsername, '*****@*****.**')
class ParticipantSummaryDaoTest(NdbTestBase): def setUp(self): super(ParticipantSummaryDaoTest, self).setUp(use_mysql=True) self.dao = ParticipantSummaryDao() self.order_dao = BiobankOrderDao() self.measurement_dao = PhysicalMeasurementsDao() self.participant_dao = ParticipantDao() self.no_filter_query = Query([], None, 2, None) self.one_filter_query = Query([FieldFilter("participantId", Operator.EQUALS, 1)], None, 2, None) self.two_filter_query = Query([FieldFilter("participantId", Operator.EQUALS, 1), FieldFilter("hpoId", Operator.EQUALS, PITT_HPO_ID)], None, 2, None) self.ascending_biobank_id_query = Query([], OrderBy("biobankId", True), 2, None) self.descending_biobank_id_query = Query([], OrderBy("biobankId", False), 2, None) self.enrollment_status_order_query = Query([], OrderBy("enrollmentStatus", True), 2, None) self.hpo_id_order_query = Query([], OrderBy("hpoId", True), 2, None) self.first_name_order_query = Query([], OrderBy("firstName", True), 2, None) def assert_no_results(self, query): results = self.dao.query(query) self.assertEquals([], results.items) self.assertIsNone(results.pagination_token) def assert_results(self, query, items, pagination_token=None): results = self.dao.query(query) self.assertListAsDictEquals(items, results.items) self.assertEquals(pagination_token, results.pagination_token, "Pagination tokens don't match; decoded = %s, %s" % (_decode_token(pagination_token), _decode_token(results.pagination_token))) def test_query_with_total(self): num_participants = 5 query = Query([], None, 10, None, include_total=True) results = self.dao.query(query) self.assertEqual(results.total, 0) for i in range(num_participants): participant = Participant(participantId=i, biobankId=i) self._insert(participant) results = self.dao.query(query) self.assertEqual(results.total, num_participants) def testQuery_noSummaries(self): self.assert_no_results(self.no_filter_query) self.assert_no_results(self.one_filter_query) self.assert_no_results(self.two_filter_query) self.assert_no_results(self.ascending_biobank_id_query) self.assert_no_results(self.descending_biobank_id_query) def _insert(self, participant, first_name=None, last_name=None): self.participant_dao.insert(participant) summary = self.participant_summary(participant) if first_name: summary.firstName = first_name if last_name: summary.lastName = last_name self.dao.insert(summary) return participant def testQuery_oneSummary(self): participant = Participant(participantId=1, biobankId=2) self._insert(participant) summary = self.dao.get(1) self.assert_results(self.no_filter_query, [summary]) self.assert_results(self.one_filter_query, [summary]) self.assert_no_results(self.two_filter_query) self.assert_results(self.ascending_biobank_id_query, [summary]) self.assert_results(self.descending_biobank_id_query, [summary]) def testUnicodeNameRoundTrip(self): name = self.fake.first_name() with self.assertRaises(UnicodeEncodeError): str(name) # sanity check that the name contains non-ASCII participant = self._insert(Participant(participantId=1, biobankId=2)) summary = self.dao.get(participant.participantId) summary.firstName = name self.dao.update(summary) fetched_summary = self.dao.get(participant.participantId) self.assertEquals(name, fetched_summary.firstName) def testQuery_twoSummaries(self): participant_1 = Participant(participantId=1, biobankId=2) self._insert(participant_1, 'Alice', 'Smith') participant_2 = Participant(participantId=2, biobankId=1) self._insert(participant_2, 'Zed', 'Zebra') ps_1 = self.dao.get(1) ps_2 = self.dao.get(2) self.assert_results(self.no_filter_query, [ps_1, ps_2]) self.assert_results(self.one_filter_query, [ps_1]) self.assert_no_results(self.two_filter_query) self.assert_results(self.ascending_biobank_id_query, [ps_2, ps_1]) self.assert_results(self.descending_biobank_id_query, [ps_1, ps_2]) def testQuery_threeSummaries_paginate(self): participant_1 = Participant(participantId=1, biobankId=4) self._insert(participant_1, 'Alice', 'Aardvark') participant_2 = Participant(participantId=2, biobankId=1) self._insert(participant_2, 'Bob', 'Builder') participant_3 = Participant(participantId=3, biobankId=3) self._insert(participant_3, 'Chad', 'Caterpillar') ps_1 = self.dao.get(1) ps_2 = self.dao.get(2) ps_3 = self.dao.get(3) self.assert_results(self.no_filter_query, [ps_1, ps_2], _make_pagination_token(['Builder', 'Bob', None, 2])) self.assert_results(self.one_filter_query, [ps_1]) self.assert_no_results(self.two_filter_query) self.assert_results(self.ascending_biobank_id_query, [ps_2, ps_3], _make_pagination_token([3, 'Caterpillar', 'Chad', None, 3])) self.assert_results(self.descending_biobank_id_query, [ps_1, ps_3], _make_pagination_token([3, 'Caterpillar', 'Chad', None, 3])) self.assert_results(_with_token(self.no_filter_query, _make_pagination_token(['Builder', 'Bob', None, 2])), [ps_3]) self.assert_results(_with_token(self.ascending_biobank_id_query, _make_pagination_token([3, 'Caterpillar', 'Chad', None, 3])), [ps_1]) self.assert_results(_with_token(self.descending_biobank_id_query, _make_pagination_token([3, 'Caterpillar', 'Chad', None, 3])), [ps_2]) def testQuery_fourFullSummaries_paginate(self): participant_1 = Participant(participantId=1, biobankId=4) self._insert(participant_1, 'Bob', 'Jones') participant_2 = Participant(participantId=2, biobankId=1) self._insert(participant_2, 'Bob', 'Jones') participant_3 = Participant(participantId=3, biobankId=3) self._insert(participant_3, 'Bob', 'Jones') participant_4 = Participant(participantId=4, biobankId=2) self._insert(participant_4, 'Bob', 'Jones') ps_1 = self.dao.get(1) ps_2 = self.dao.get(2) ps_3 = self.dao.get(3) ps_4 = self.dao.get(4) ps_1.lastName = 'Jones' ps_1.firstName = 'Bob' ps_1.dateOfBirth = datetime.date(1978, 10, 9) ps_1.hpoId = PITT_HPO_ID self.dao.update(ps_1) ps_2.lastName = 'Aardvark' ps_2.firstName = 'Bob' ps_2.dateOfBirth = datetime.date(1978, 10, 10) ps_2.enrollmentStatus = EnrollmentStatus.MEMBER self.dao.update(ps_2) ps_3.lastName = 'Jones' ps_3.firstName = 'Bob' ps_3.dateOfBirth = datetime.date(1978, 10, 10) ps_3.hpoId = PITT_HPO_ID ps_3.enrollmentStatus = EnrollmentStatus.MEMBER self.dao.update(ps_3) ps_4.lastName = 'Jones' ps_4.enrollmentStatus = EnrollmentStatus.FULL_PARTICIPANT self.dao.update(ps_4) self.assert_results(self.no_filter_query, [ps_2, ps_4], _make_pagination_token(['Jones', 'Bob', None, 4])) self.assert_results(self.one_filter_query, [ps_1]) self.assert_results(self.two_filter_query, [ps_1]) self.assert_results(self.ascending_biobank_id_query, [ps_2, ps_4], _make_pagination_token([2, 'Jones', 'Bob', None, 4])) self.assert_results(self.descending_biobank_id_query, [ps_1, ps_3], _make_pagination_token([3, 'Jones', 'Bob', datetime.date(1978, 10, 10), 3])) self.assert_results(self.hpo_id_order_query, [ps_2, ps_4], _make_pagination_token([0, 'Jones', 'Bob', None, 4])) self.assert_results(self.enrollment_status_order_query, [ps_1, ps_2], _make_pagination_token(['MEMBER', 'Aardvark', 'Bob', datetime.date(1978, 10, 10), 2])) self.assert_results(_with_token(self.no_filter_query, _make_pagination_token(['Jones', 'Bob', None, 4])), [ps_1, ps_3]) self.assert_results(_with_token(self.ascending_biobank_id_query, _make_pagination_token([2, 'Jones', 'Bob', None, 4])), [ps_3, ps_1]) self.assert_results(_with_token(self.descending_biobank_id_query, _make_pagination_token([3, 'Jones', 'Bob', datetime.date(1978, 10, 10), 3])), [ps_4, ps_2]) self.assert_results(_with_token(self.hpo_id_order_query, _make_pagination_token([0, 'Jones', 'Bob', None, 4])), [ps_1, ps_3]) self.assert_results(_with_token(self.enrollment_status_order_query, _make_pagination_token(['MEMBER', 'Aardvark', 'Bob', datetime.date(1978, 10, 10), 2])), [ps_3, ps_4]) def test_update_from_samples(self): # baseline_tests = ['BASELINE1', 'BASELINE2'] baseline_tests = ["1PST8", "2PST8"] config.override_setting(config.BASELINE_SAMPLE_TEST_CODES, baseline_tests) self.dao.update_from_biobank_stored_samples() # safe noop p_baseline_samples = self._insert(Participant(participantId=1, biobankId=11)) p_mixed_samples = self._insert(Participant(participantId=2, biobankId=22)) p_no_samples = self._insert(Participant(participantId=3, biobankId=33)) p_unconfirmed = self._insert(Participant(participantId=4, biobankId=44)) self.assertEquals(self.dao.get(p_baseline_samples.participantId).numBaselineSamplesArrived, 0) def get_p_baseline_last_modified(): return self.dao.get(p_baseline_samples.participantId).lastModified p_baseline_last_modified1 = get_p_baseline_last_modified() sample_dao = BiobankStoredSampleDao() def add_sample(participant, test_code, sample_id): TIME = datetime.datetime(2018, 3, 2) sample_dao.insert(BiobankStoredSample( biobankStoredSampleId=sample_id, biobankId=participant.biobankId, biobankOrderIdentifier='KIT', test=test_code, confirmed=TIME)) add_sample(p_baseline_samples, baseline_tests[0], '11111') add_sample(p_baseline_samples, baseline_tests[1], '22223') add_sample(p_mixed_samples, baseline_tests[0], '11112') add_sample(p_mixed_samples, 'NOT1', '44441') # add unconfirmed sample sample_dao.insert(BiobankStoredSample(biobankStoredSampleId=55555, biobankId=p_unconfirmed.biobankId, biobankOrderIdentifier='KIT', test=baseline_tests[1], confirmed=None)) # sleep 1 sec to make lastModified different time.sleep(1) self.dao.update_from_biobank_stored_samples() p_baseline_last_modified2 = get_p_baseline_last_modified() self.assertNotEquals(p_baseline_last_modified2, p_baseline_last_modified1) self.assertEquals(self.dao.get(p_baseline_samples.participantId).numBaselineSamplesArrived, 2) self.assertEquals(self.dao.get(p_mixed_samples.participantId).numBaselineSamplesArrived, 1) self.assertEquals(self.dao.get(p_no_samples.participantId).numBaselineSamplesArrived, 0) self.assertEquals(self.dao.get(p_unconfirmed.participantId).numBaselineSamplesArrived, 0) M_baseline_samples = self._insert(Participant(participantId=9, biobankId=99)) add_sample(M_baseline_samples, baseline_tests[0], '999') M_first_update = self.dao.get(M_baseline_samples.participantId) # sleep 1 sec to make lastModified different time.sleep(1) self.dao.update_from_biobank_stored_samples() add_sample(M_baseline_samples, baseline_tests[1], '9999') M_second_update = self.dao.get(M_baseline_samples.participantId) # sleep 1 sec to make lastModified different time.sleep(1) self.dao.update_from_biobank_stored_samples() self.assertNotEqual(M_first_update.lastModified, M_second_update.lastModified) self.assertEquals(get_p_baseline_last_modified(), p_baseline_last_modified2) def test_update_from_samples_changed_tests(self): baseline_tests = ["1PST8", "2PST8"] config.override_setting(config.BASELINE_SAMPLE_TEST_CODES, baseline_tests) self.dao.update_from_biobank_stored_samples() # safe noop participant = self._insert(Participant(participantId=1, biobankId=11)) self.assertEquals(self.dao.get(participant.participantId).numBaselineSamplesArrived, 0) sample_dao = BiobankStoredSampleDao() def add_sample(test_code, sample_id): TIME = datetime.datetime(2018, 3, 2) sample_dao.insert(BiobankStoredSample( biobankStoredSampleId=sample_id, biobankId=participant.biobankId, biobankOrderIdentifier='KIT', test=test_code, confirmed=TIME)) add_sample(baseline_tests[0], '11111') add_sample(baseline_tests[1], '22223') self.dao.update_from_biobank_stored_samples() summary = self.dao.get(participant.participantId) init_last_modified = summary.lastModified self.assertEquals(summary.numBaselineSamplesArrived, 2) # sleep 1 sec to make lastModified different time.sleep(1) # Simulate removal of one of the baseline tests from config.json. baseline_tests.pop() config.override_setting(config.BASELINE_SAMPLE_TEST_CODES, baseline_tests) self.dao.update_from_biobank_stored_samples() summary = self.dao.get(participant.participantId) self.assertEquals(summary.numBaselineSamplesArrived, 1) self.assertNotEqual(init_last_modified, summary.lastModified) def test_only_update_dna_sample(self): dna_tests = ["1ED10", "1SAL2"] config.override_setting(config.DNA_SAMPLE_TEST_CODES, dna_tests) self.dao.update_from_biobank_stored_samples() # safe noop p_dna_samples = self._insert(Participant(participantId=1, biobankId=11)) self.assertEquals(self.dao.get(p_dna_samples.participantId).samplesToIsolateDNA, None) self.assertEquals( self.dao.get(p_dna_samples.participantId).enrollmentStatusCoreStoredSampleTime, None) self.assertEquals( self.dao.get(p_dna_samples.participantId).enrollmentStatusCoreOrderedSampleTime, None) sample_dao = BiobankStoredSampleDao() def add_sample(participant, test_code, sample_id, confirmed_time): sample_dao.insert(BiobankStoredSample( biobankStoredSampleId=sample_id, biobankId=participant.biobankId, biobankOrderIdentifier='KIT', test=test_code, confirmed=confirmed_time)) confirmed_time_0 = datetime.datetime(2018, 3, 1) add_sample(p_dna_samples, dna_tests[0], '11111', confirmed_time_0) self.dao.update_from_biobank_stored_samples() self.assertEquals(self.dao.get(p_dna_samples.participantId).samplesToIsolateDNA, SampleStatus.RECEIVED) # only update dna sample will not update enrollmentStatusCoreStoredSampleTime self.assertEquals( self.dao.get(p_dna_samples.participantId).enrollmentStatusCoreStoredSampleTime, None) self.assertEquals( self.dao.get(p_dna_samples.participantId).enrollmentStatusCoreOrderedSampleTime, None) def test_calculate_enrollment_status(self): self.assertEquals(EnrollmentStatus.FULL_PARTICIPANT, self.dao.calculate_enrollment_status(True, NUM_BASELINE_PPI_MODULES, PhysicalMeasurementsStatus.COMPLETED, SampleStatus.RECEIVED)) self.assertEquals(EnrollmentStatus.MEMBER, self.dao.calculate_enrollment_status(True, NUM_BASELINE_PPI_MODULES - 1, PhysicalMeasurementsStatus.COMPLETED, SampleStatus.RECEIVED)) self.assertEquals(EnrollmentStatus.MEMBER, self.dao.calculate_enrollment_status(True, NUM_BASELINE_PPI_MODULES, PhysicalMeasurementsStatus.UNSET, SampleStatus.RECEIVED)) self.assertEquals(EnrollmentStatus.MEMBER, self.dao.calculate_enrollment_status(True, NUM_BASELINE_PPI_MODULES, PhysicalMeasurementsStatus.COMPLETED, SampleStatus.UNSET)) self.assertEquals(EnrollmentStatus.INTERESTED, self.dao.calculate_enrollment_status(False, NUM_BASELINE_PPI_MODULES, PhysicalMeasurementsStatus.COMPLETED, SampleStatus.RECEIVED)) def testUpdateEnrollmentStatus(self): ehr_consent_time = datetime.datetime(2018, 3, 1) summary = ParticipantSummary( participantId=1, biobankId=2, consentForStudyEnrollment=QuestionnaireStatus.SUBMITTED, consentForElectronicHealthRecords=QuestionnaireStatus.SUBMITTED, consentForElectronicHealthRecordsTime=ehr_consent_time, enrollmentStatus=EnrollmentStatus.INTERESTED) self.dao.update_enrollment_status(summary) self.assertEquals(EnrollmentStatus.MEMBER, summary.enrollmentStatus) self.assertEquals(ehr_consent_time, summary.enrollmentStatusMemberTime) def testUpdateEnrollmentStatusLastModified(self): """DA-631: enrollment_status update should update last_modified.""" participant = self._insert(Participant(participantId=6, biobankId=66)) # collect current modified and enrollment status summary = self.dao.get(participant.participantId) test_dt = datetime.datetime(2018, 11, 1) def reset_summary(): # change summary so enrollment status will be changed from INTERESTED to MEMBER. summary.enrollmentStatus = EnrollmentStatus.INTERESTED summary.lastModified = test_dt summary.consentForStudyEnrollment = QuestionnaireStatus.SUBMITTED summary.consentForElectronicHealthRecords = QuestionnaireStatus.SUBMITTED summary.physicalMeasurementsStatus = PhysicalMeasurementsStatus.COMPLETED summary.samplesToIsolateDNA = SampleStatus.RECEIVED self.dao.update(summary) ## Test Step 1: Validate update_from_biobank_stored_samples() changes lastModified. reset_summary() # Update and reload summary record self.dao.update_from_biobank_stored_samples(participant_id=participant.participantId) summary = self.dao.get(participant.participantId) # Test that status has changed and lastModified is also different self.assertEquals(EnrollmentStatus.MEMBER, summary.enrollmentStatus) self.assertNotEqual(test_dt, summary.lastModified) ## Test Step 2: Validate that update_enrollment_status() changes the lastModified property. reset_summary() summary = self.dao.get(participant.participantId) self.assertEqual(test_dt, summary.lastModified) # update_enrollment_status() does not touch the db, it only modifies object properties. self.dao.update_enrollment_status(summary) self.assertEquals(EnrollmentStatus.MEMBER, summary.enrollmentStatus) self.assertNotEqual(test_dt, summary.lastModified) def testNumberDistinctVisitsCounts(self): self.participant = self._insert(Participant(participantId=7, biobankId=77)) # insert biobank order order = self.order_dao.insert(self._make_biobank_order()) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 1) cancel_request = cancel_biobank_order() # cancel biobank order self.order_dao.update_with_patch(order.biobankOrderId, cancel_request, order.version) summary = self.dao.get(self.participant.participantId) # distinct count should be 0 self.assertEquals(summary.numberDistinctVisits, 0) self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId, TIME_1.isoformat())) # insert physical measurement measurement = self.measurement_dao.insert(self._make_physical_measurements()) summary = self.dao.get(self.participant.participantId) # count should be 1 self.assertEquals(summary.numberDistinctVisits, 1) # cancel the measurement cancel_measurement = get_restore_or_cancel_info() with self.measurement_dao.session() as session: self.measurement_dao.update_with_patch(measurement.physicalMeasurementsId, session, cancel_measurement) summary = self.dao.get(self.participant.participantId) # count should be 0 self.assertEquals(summary.numberDistinctVisits, 0) with clock.FakeClock(TIME_1): self.order_dao.insert(self._make_biobank_order(biobankOrderId='2', identifiers=[ BiobankOrderIdentifier(system='b', value='d')], samples=[BiobankOrderedSample( biobankOrderId='2', test=BIOBANK_TESTS[0], description='description', processingRequired=True)])) with clock.FakeClock(TIME_2): self.measurement_dao.insert(self._make_physical_measurements( physicalMeasurementsId=2)) summary = self.dao.get(self.participant.participantId) # A PM on another day should add a new distinct count. self.assertEquals(summary.numberDistinctVisits, 2) with clock.FakeClock(TIME_3): self.order_dao.insert(self._make_biobank_order(biobankOrderId='3', identifiers=[ BiobankOrderIdentifier(system='s', value='s')], samples=[BiobankOrderedSample( biobankOrderId ='3', finalized=TIME_3, test=BIOBANK_TESTS[1], description='another description', processingRequired=False)])) # a physical measurement on same day as biobank order does not add distinct visit. self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=6)) # another biobank order on the same day should also not add a distinct visit self.order_dao.insert(self._make_biobank_order(biobankOrderId='7', identifiers=[ BiobankOrderIdentifier(system='x', value='x')], samples=[BiobankOrderedSample( biobankOrderId ='7', finalized=TIME_3, test=BIOBANK_TESTS[1], description='another description', processingRequired=False)])) summary = self.dao.get(self.participant.participantId) # 1 from each of TIME_1 TIME_2 TIME_3 self.assertEquals(summary.numberDistinctVisits, 3) def test_qa_scenarios_for_pmb_visits(self): """ PDR at https://docs.google.com/document/d/1sL54f-I91RvhjIprrdbwD8TlR9Jq91MX2ELf1EtJdxc/edit#heading=h.bqo8kt3igsrw<Paste> """ self.participant = self._insert(Participant(participantId=6, biobankId=66)) # test scenario 1 with clock.FakeClock(TIME_4): self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId, TIME_4.isoformat())) self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=666, participantId=self.participant.participantId, finalized=TIME_4)) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 1) with clock.FakeClock(TIME_5): self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId, TIME_5.isoformat())) self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=669, finalized=TIME_5)) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 2) # test scenario 2 with clock.FakeClock(TIME_6): self.participant = self._insert(Participant(participantId=9, biobankId=13)) self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId, TIME_6.isoformat())) self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=8, finalized=TIME_6)) self.order_dao.insert(self._make_biobank_order(biobankOrderId='2', identifiers=[ BiobankOrderIdentifier(system='b', value='d')], samples=[BiobankOrderedSample( biobankOrderId='2', finalized=TIME_7, test=BIOBANK_TESTS[0], description='description', processingRequired=True)])) summary = self.dao.get(self.participant.participantId) # distinct count should be 2 self.assertEquals(summary.numberDistinctVisits, 2) # test scenario 3 with clock.FakeClock(TIME_6): self.participant = self._insert(Participant(participantId=66, biobankId=42)) self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId, TIME_6.isoformat())) self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=12, createdSiteId=2, finalized=TIME_6)) self.order_dao.insert(self._make_biobank_order(biobankOrderId='18', finalizedSiteId=1, identifiers=[ BiobankOrderIdentifier(system='x', value='y')], samples=[BiobankOrderedSample( biobankOrderId='18', finalized=TIME_6, test=BIOBANK_TESTS[0], description='description', processingRequired=True)])) summary = self.dao.get(self.participant.participantId) # distinct count should be 1 self.assertEquals(summary.numberDistinctVisits, 1) # test scenario 4 with clock.FakeClock(TIME_8): self.participant = self._insert(Participant(participantId=6613, biobankId=142)) self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId, TIME_8.isoformat())) self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=129, finalized=TIME_8)) order = self.order_dao.insert(self._make_biobank_order(biobankOrderId='999', identifiers=[ BiobankOrderIdentifier(system='s', value='s')], samples=[BiobankOrderedSample( biobankOrderId='999', finalized=TIME_8, test=BIOBANK_TESTS[1], description='description', processingRequired=True)])) summary = self.dao.get(self.participant.participantId) # distinct count should be 1 self.assertEquals(summary.numberDistinctVisits, 1) # change finalized time, recalculating count with self.order_dao.session() as session: existing_order = copy.deepcopy(order) order.samples[0].finalized = TIME_9 self.order_dao._do_update(session, order, existing_order) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 1) # change test, should not change count. with self.order_dao.session() as session: existing_order = copy.deepcopy(order) order.samples[0].test = BIOBANK_TESTS[0] self.order_dao._do_update(session, order, existing_order) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 1) # test scenario 5 with clock.FakeClock(TIME_12): self.participant = self._insert(Participant(participantId=3000, biobankId=2019)) self.order_dao.insert(self._make_biobank_order(biobankOrderId='700', identifiers=[ BiobankOrderIdentifier(system='n', value='s')], samples=[BiobankOrderedSample( biobankOrderId='700', finalized=TIME_10, test=BIOBANK_TESTS[1], description='description', processingRequired=True)])) summary = self.dao.get(self.participant.participantId) # distinct count should be 1 self.assertEquals(summary.numberDistinctVisits, 1) other_order = self.order_dao.insert(self._make_biobank_order(biobankOrderId='701', identifiers=[ BiobankOrderIdentifier(system='n', value='t')], samples=[BiobankOrderedSample( biobankOrderId='701', finalized=TIME_11, test=BIOBANK_TESTS[1], description='description', processingRequired=True)])) summary = self.dao.get(self.participant.participantId) # distinct count should be 2 self.assertEquals(summary.numberDistinctVisits, 2) order = self.order_dao.insert(self._make_biobank_order(biobankOrderId='702', identifiers=[ BiobankOrderIdentifier(system='n', value='u')], samples=[BiobankOrderedSample( biobankOrderId='702', finalized=TIME_12, test=BIOBANK_TESTS[1], description='description', processingRequired=True)])) summary = self.dao.get(self.participant.participantId) # distinct count should be 3 self.assertEquals(summary.numberDistinctVisits, 3) self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId, TIME_12.isoformat())) self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=120, finalized=TIME_12)) summary = self.dao.get(self.participant.participantId) # distinct count should be 3 self.assertEquals(summary.numberDistinctVisits, 3) cancel_request = cancel_biobank_order() # cancel biobank order with PM on same day self.order_dao.update_with_patch(order.biobankOrderId, cancel_request, order.version) summary = self.dao.get(self.participant.participantId) # distinct count should be 3 (the PM on same day still counts) self.assertEquals(summary.numberDistinctVisits, 3) self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId, TIME_1.isoformat())) self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=150, finalized=TIME_1)) summary = self.dao.get(self.participant.participantId) # distinct count should be 4 self.assertEquals(summary.numberDistinctVisits, 4) # cancel order with pm on different day self.order_dao.update_with_patch(other_order.biobankOrderId, cancel_request, order.version) summary = self.dao.get(self.participant.participantId) # distinct count should be 3 self.assertEquals(summary.numberDistinctVisits, 3) def test_pm_restore_cancel_biobank_restore_cancel(self): self.participant = self._insert(Participant(participantId=9, biobankId=13)) self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId, TIME_4.isoformat())) measurement = self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=669, finalized=TIME_4)) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 1) with clock.FakeClock(TIME_5): order = self.order_dao.insert(self._make_biobank_order(biobankOrderId='2', identifiers=[ BiobankOrderIdentifier(system='b', value='d')], samples=[BiobankOrderedSample( biobankOrderId='2', finalized=TIME_5, test=BIOBANK_TESTS[0], description='description', processingRequired=True)])) with clock.FakeClock(TIME_7): summary = self.dao.get(self.participant.participantId) # distinct count should be 2 self.assertEquals(summary.numberDistinctVisits, 2) # cancel the measurement cancel_measurement = get_restore_or_cancel_info() with self.measurement_dao.session() as session: self.measurement_dao.update_with_patch(measurement.physicalMeasurementsId, session, cancel_measurement) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 1) with clock.FakeClock(TIME_7): restore_measurement = get_restore_or_cancel_info(status='restored') with self.measurement_dao.session() as session: self.measurement_dao.update_with_patch(measurement.physicalMeasurementsId, session, restore_measurement) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 2) cancel_request = cancel_biobank_order() order = self.order_dao.update_with_patch(order.biobankOrderId, cancel_request, order.version) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 1) restore_order = get_restore_or_cancel_info(status='restored') restore_order['amendedReason'] = 'some reason' self.order_dao.update_with_patch(order.biobankOrderId, restore_order, order.version) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 2) def test_amending_biobank_order_distinct_visit_count(self): self.participant = self._insert(Participant(participantId=9, biobankId=13)) with clock.FakeClock(TIME_5): order = self.order_dao.insert(self._make_biobank_order(biobankOrderId='2', identifiers=[ BiobankOrderIdentifier(system='b', value='d')], samples=[BiobankOrderedSample( biobankOrderId='2', finalized=TIME_5, test=BIOBANK_TESTS[0], description='description', processingRequired=True)])) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 1) with clock.FakeClock(TIME_7): amend_order = self._get_amended_info(order) with self.order_dao.session() as session: self.order_dao._do_update(session, amend_order, order) # Shouldn't change on a simple amendment (unless finalized time on samples change) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 1) with clock.FakeClock(TIME_7_5): cancel_request = cancel_biobank_order() order = self.order_dao.update_with_patch(order.biobankOrderId, cancel_request, order.version) # A cancelled order (even after amending) should reduce count (unless some other valid order on same day) summary = self.dao.get(self.participant.participantId) self.assertEquals(summary.numberDistinctVisits, 0) @staticmethod def _get_amended_info(order): amendment = dict(amendedReason='I had to change something', amendedInfo={ "author": { "system": "https://www.pmi-ops.org/healthpro-username", "value": "*****@*****.**" }, "site": { "system": "https://www.pmi-ops.org/site-id", "value": "hpo-site-monroeville" } }) order.amendedReason = amendment['amendedReason'] order.amendedInfo = amendment['amendedInfo'] return order def _make_biobank_order(self, **kwargs): """Makes a new BiobankOrder (same values every time) with valid/complete defaults. Kwargs pass through to BiobankOrder constructor, overriding defaults. """ for k, default_value in ( ('biobankOrderId', '1'), ('created', clock.CLOCK.now()), ('participantId', self.participant.participantId), ('sourceSiteId', 1), ('sourceUsername', '*****@*****.**'), ('collectedSiteId', 1), ('collectedUsername', '*****@*****.**'), ('processedSiteId', 1), ('processedUsername', '*****@*****.**'), ('finalizedSiteId', 2), ('finalizedUsername', '*****@*****.**'), ('identifiers', [BiobankOrderIdentifier(system='a', value='c')]), ('samples', [BiobankOrderedSample( biobankOrderId='1', test=BIOBANK_TESTS[0], description='description', finalized=TIME_1, processingRequired=True)])): if k not in kwargs: kwargs[k] = default_value return BiobankOrder(**kwargs) def _make_physical_measurements(self, **kwargs): """Makes a new PhysicalMeasurements (same values every time) with valid/complete defaults. Kwargs pass through to PM constructor, overriding defaults. """ for k, default_value in ( ('physicalMeasurementsId', 1), ('participantId', self.participant.participantId), ('resource', self.measurement_json), ('createdSiteId', 1), ('finalized', TIME_3), ('finalizedSiteId', 2)): if k not in kwargs: kwargs[k] = default_value return PhysicalMeasurements(**kwargs)
class BiobankOrderDaoTest(SqlTestBase): _A_TEST = BIOBANK_TESTS[0] def setUp(self): super(BiobankOrderDaoTest, self).setUp() self.participant = Participant(participantId=123, biobankId=555) ParticipantDao().insert(self.participant) self.dao = BiobankOrderDao() def _make_biobank_order(self, **kwargs): """Makes a new BiobankOrder (same values every time) with valid/complete defaults. Kwargs pass through to BiobankOrder constructor, overriding defaults. """ for k, default_value in (('biobankOrderId', '1'), ('created', clock.CLOCK.now()), ('participantId', self.participant.participantId), ('sourceSiteId', 1), ('sourceUsername', '*****@*****.**'), ('collectedSiteId', 1), ('collectedUsername', '*****@*****.**'), ('processedSiteId', 1), ('processedUsername', '*****@*****.**'), ('finalizedSiteId', 1), ('finalizedUsername', '*****@*****.**'), ('identifiers', [ BiobankOrderIdentifier(system='a', value='c') ]), ('samples', [ BiobankOrderedSample( biobankOrderId='1', test=self._A_TEST, description='description', processingRequired=True) ])): if k not in kwargs: kwargs[k] = default_value return BiobankOrder(**kwargs) def test_bad_participant(self): with self.assertRaises(BadRequest): self.dao.insert(self._make_biobank_order(participantId=999)) def test_from_json(self): ParticipantSummaryDao().insert( self.participant_summary(self.participant)) order_json = load_biobank_order_json(self.participant.participantId) order = BiobankOrderDao().from_client_json( order_json, self.participant.participantId) self.assertEquals(1, order.sourceSiteId) self.assertEquals('*****@*****.**', order.sourceUsername) self.assertEquals(1, order.collectedSiteId) self.assertEquals('*****@*****.**', order.collectedUsername) self.assertEquals(1, order.processedSiteId) self.assertEquals('*****@*****.**', order.processedUsername) self.assertEquals(1, order.finalizedSiteId) self.assertEquals('*****@*****.**', order.finalizedUsername) def test_to_json(self): order = self._make_biobank_order() order_json = BiobankOrderDao().to_client_json(order) expected_order_json = load_biobank_order_json( self.participant.participantId) for key in ('createdInfo', 'collectedInfo', 'processedInfo', 'finalizedInfo'): self.assertEquals(expected_order_json[key], order_json.get(key)) def test_duplicate_insert_ok(self): ParticipantSummaryDao().insert( self.participant_summary(self.participant)) order_1 = self.dao.insert(self._make_biobank_order()) order_2 = self.dao.insert(self._make_biobank_order()) self.assertEquals(order_1.asdict(), order_2.asdict()) def test_same_id_different_identifier_not_ok(self): ParticipantSummaryDao().insert( self.participant_summary(self.participant)) self.dao.insert( self._make_biobank_order( identifiers=[BiobankOrderIdentifier(system='a', value='b')])) with self.assertRaises(Conflict): self.dao.insert( self._make_biobank_order(identifiers=[ BiobankOrderIdentifier(system='a', value='c') ])) def test_reject_used_identifier(self): ParticipantSummaryDao().insert( self.participant_summary(self.participant)) self.dao.insert( self._make_biobank_order( biobankOrderId='1', identifiers=[BiobankOrderIdentifier(system='a', value='b')])) with self.assertRaises(BadRequest): self.dao.insert( self._make_biobank_order(biobankOrderId='2', identifiers=[ BiobankOrderIdentifier(system='a', value='b') ])) def test_order_for_withdrawn_participant_fails(self): self.participant.withdrawalStatus = WithdrawalStatus.NO_USE ParticipantDao().update(self.participant) ParticipantSummaryDao().insert( self.participant_summary(self.participant)) with self.assertRaises(Forbidden): self.dao.insert( self._make_biobank_order( participantId=self.participant.participantId)) def test_get_for_withdrawn_participant_fails(self): ParticipantSummaryDao().insert( self.participant_summary(self.participant)) self.dao.insert( self._make_biobank_order( biobankOrderId='1', participantId=self.participant.participantId)) self.participant.version += 1 self.participant.withdrawalStatus = WithdrawalStatus.NO_USE ParticipantDao().update(self.participant) with self.assertRaises(Forbidden): self.dao.get(1) def test_store_invalid_test(self): with self.assertRaises(BadRequest): self.dao.insert( self._make_biobank_order(samples=[ BiobankOrderedSample(test='InvalidTestName', processingRequired=True, description=u'tested it') ]))