def test_generate_samples(self):
participant_id = self.send_post('Participant', {})['participantId']
self.send_consent(participant_id)
self.send_post(
'Participant/%s/BiobankOrder' % participant_id,
load_biobank_order_json(from_client_participant_id(participant_id)))
# Sanity check that the orders were created correctly.
bo_dao = BiobankOrderDao()
self.assertEquals(1, bo_dao.count())
order = bo_dao.get_all()[0]
self.assertEquals(16, len(bo_dao.get_with_children(order.biobankOrderId).samples))
self.send_post('DataGen', {'create_biobank_samples': True, 'samples_missing_fraction': 0.0})
upsert_from_latest_csv() # Run the (usually offline) Biobank CSV import job.
self.assertEquals(16, BiobankStoredSampleDao().count())
ps = ParticipantSummaryDao().get(from_client_participant_id(participant_id))
self.assertEquals(SampleStatus.RECEIVED, ps.samplesToIsolateDNA)
self.assertEquals(13, ps.numBaselineSamplesArrived)
def test_update_from_samples_changed_tests(self):
baseline_tests = ["1PST8", "2PST8"]
config.override_setting(config.BASELINE_SAMPLE_TEST_CODES,
baseline_tests)
self.dao.update_from_biobank_stored_samples() # safe noop
participant = self._insert(Participant(participantId=1, biobankId=11))
self.assertEquals(
self.dao.get(participant.participantId).numBaselineSamplesArrived,
0)
sample_dao = BiobankStoredSampleDao()
def add_sample(test_code, sample_id):
TIME = datetime.datetime(2018, 3, 2)
sample_dao.insert(
BiobankStoredSample(biobankStoredSampleId=sample_id,
biobankId=participant.biobankId,
biobankOrderIdentifier='KIT',
test=test_code,
confirmed=TIME))
add_sample(baseline_tests[0], '11111')
add_sample(baseline_tests[1], '22223')
self.dao.update_from_biobank_stored_samples()
summary = self.dao.get(participant.participantId)
init_last_modified = summary.lastModified
self.assertEquals(summary.numBaselineSamplesArrived, 2)
# sleep 1 sec to make lastModified different
time.sleep(1)
# Simulate removal of one of the baseline tests from config.json.
baseline_tests.pop()
config.override_setting(config.BASELINE_SAMPLE_TEST_CODES,
baseline_tests)
self.dao.update_from_biobank_stored_samples()
summary = self.dao.get(participant.participantId)
self.assertEquals(summary.numBaselineSamplesArrived, 1)
self.assertNotEqual(init_last_modified, summary.lastModified)
def test_update_from_samples(self):
baseline_tests = ['BASELINE1', 'BASELINE2']
config.override_setting(config.BASELINE_SAMPLE_TEST_CODES,
baseline_tests)
self.dao.update_from_biobank_stored_samples() # safe noop
p_baseline_samples = self._insert(
Participant(participantId=1, biobankId=11))
p_mixed_samples = self._insert(
Participant(participantId=2, biobankId=22))
p_no_samples = self._insert(Participant(participantId=3, biobankId=33))
self.assertEquals(
self.dao.get(
p_baseline_samples.participantId).numBaselineSamplesArrived, 0)
sample_dao = BiobankStoredSampleDao()
def add_sample(participant, test_code, sample_id):
sample_dao.insert(
BiobankStoredSample(biobankStoredSampleId=sample_id,
biobankId=participant.biobankId,
test=test_code))
add_sample(p_baseline_samples, baseline_tests[0], '11111')
add_sample(p_baseline_samples, baseline_tests[1], '22223')
add_sample(p_mixed_samples, baseline_tests[0], '11112')
add_sample(p_mixed_samples, 'NOT1', '44441')
self.dao.update_from_biobank_stored_samples()
self.assertEquals(
self.dao.get(
p_baseline_samples.participantId).numBaselineSamplesArrived, 2)
self.assertEquals(
self.dao.get(
p_mixed_samples.participantId).numBaselineSamplesArrived, 1)
self.assertEquals(
self.dao.get(p_no_samples.participantId).numBaselineSamplesArrived,
0)
def test_update_from_samples(self):
# baseline_tests = ['BASELINE1', 'BASELINE2']
baseline_tests = ["1PST8", "2PST8"]
config.override_setting(config.BASELINE_SAMPLE_TEST_CODES,
baseline_tests)
self.dao.update_from_biobank_stored_samples() # safe noop
p_baseline_samples = self._insert(
Participant(participantId=1, biobankId=11))
p_mixed_samples = self._insert(
Participant(participantId=2, biobankId=22))
p_no_samples = self._insert(Participant(participantId=3, biobankId=33))
p_unconfirmed = self._insert(Participant(participantId=4,
biobankId=44))
self.assertEquals(
self.dao.get(
p_baseline_samples.participantId).numBaselineSamplesArrived, 0)
def get_p_baseline_last_modified():
return self.dao.get(p_baseline_samples.participantId).lastModified
p_baseline_last_modified1 = get_p_baseline_last_modified()
sample_dao = BiobankStoredSampleDao()
def add_sample(participant, test_code, sample_id):
TIME = datetime.datetime(2018, 3, 2)
sample_dao.insert(
BiobankStoredSample(biobankStoredSampleId=sample_id,
biobankId=participant.biobankId,
biobankOrderIdentifier='KIT',
test=test_code,
confirmed=TIME))
add_sample(p_baseline_samples, baseline_tests[0], '11111')
add_sample(p_baseline_samples, baseline_tests[1], '22223')
add_sample(p_mixed_samples, baseline_tests[0], '11112')
add_sample(p_mixed_samples, 'NOT1', '44441')
# add unconfirmed sample
sample_dao.insert(
BiobankStoredSample(biobankStoredSampleId=55555,
biobankId=p_unconfirmed.biobankId,
biobankOrderIdentifier='KIT',
test=baseline_tests[1],
confirmed=None))
# sleep 1 sec to make lastModified different
time.sleep(1)
self.dao.update_from_biobank_stored_samples()
p_baseline_last_modified2 = get_p_baseline_last_modified()
self.assertNotEquals(p_baseline_last_modified2,
p_baseline_last_modified1)
self.assertEquals(
self.dao.get(
p_baseline_samples.participantId).numBaselineSamplesArrived, 2)
self.assertEquals(
self.dao.get(
p_mixed_samples.participantId).numBaselineSamplesArrived, 1)
self.assertEquals(
self.dao.get(p_no_samples.participantId).numBaselineSamplesArrived,
0)
self.assertEquals(
self.dao.get(
p_unconfirmed.participantId).numBaselineSamplesArrived, 0)
M_baseline_samples = self._insert(
Participant(participantId=9, biobankId=99))
add_sample(M_baseline_samples, baseline_tests[0], '999')
M_first_update = self.dao.get(M_baseline_samples.participantId)
# sleep 1 sec to make lastModified different
time.sleep(1)
self.dao.update_from_biobank_stored_samples()
add_sample(M_baseline_samples, baseline_tests[1], '9999')
M_second_update = self.dao.get(M_baseline_samples.participantId)
# sleep 1 sec to make lastModified different
time.sleep(1)
self.dao.update_from_biobank_stored_samples()
self.assertNotEqual(M_first_update.lastModified,
M_second_update.lastModified)
self.assertEquals(get_p_baseline_last_modified(),
p_baseline_last_modified2)
def setUp(self):
super(BiobankStoredSampleDaoTest, self).setUp()
self.participant = Participant(participantId=123, biobankId=555)
ParticipantDao().insert(self.participant)
self.dao = BiobankStoredSampleDao()
def setUp(self):
super(MySqlReconciliationTest, self).setUp(use_mysql=True)
self.participant_dao = ParticipantDao()
self.summary_dao = ParticipantSummaryDao()
self.order_dao = BiobankOrderDao()
self.sample_dao = BiobankStoredSampleDao()
class MySqlReconciliationTest(FlaskTestBase):
"""Biobank samples pipeline tests requiring slower MySQL (not SQLite)."""
def setUp(self):
super(MySqlReconciliationTest, self).setUp(use_mysql=True)
self.participant_dao = ParticipantDao()
self.summary_dao = ParticipantSummaryDao()
self.order_dao = BiobankOrderDao()
self.sample_dao = BiobankStoredSampleDao()
def _withdraw(self, participant, withdrawal_time):
with FakeClock(withdrawal_time):
participant.withdrawalStatus = WithdrawalStatus.NO_USE
self.participant_dao.update(participant)
def _insert_participant(self, race_codes=[]):
participant = self.participant_dao.insert(Participant())
# satisfies the consent requirement
self.summary_dao.insert(self.participant_summary(participant))
if race_codes:
self._submit_race_questionnaire_response(
to_client_participant_id(participant.participantId),
race_codes)
return participant
def _insert_order(self,
participant,
order_id,
tests,
order_time,
finalized_tests=None,
kit_id=None,
tracking_number=None):
order = BiobankOrder(biobankOrderId=order_id,
participantId=participant.participantId,
sourceSiteId=1,
finalizedSiteId=1,
finalizedUsername='******',
created=order_time,
samples=[])
id_1 = BiobankOrderIdentifier(
system="https://orders.mayomedicallaboratories.com",
value=order_id)
id_2 = BiobankOrderIdentifier(system="https://www.pmi-ops.org",
value='O%s' % order_id)
order.identifiers.append(id_1)
order.identifiers.append(id_2)
if kit_id:
order.identifiers.append(
BiobankOrderIdentifier(system=_KIT_ID_SYSTEM, value=kit_id))
if tracking_number:
order.identifiers.append(
BiobankOrderIdentifier(system=_TRACKING_NUMBER_SYSTEM,
value=tracking_number))
for test_code in tests:
finalized_time = order_time
if finalized_tests and not test_code in finalized_tests:
finalized_time = None
order.samples.append(
BiobankOrderedSample(biobankOrderId=order.biobankOrderId,
test=test_code,
description=u'test',
processingRequired=False,
collected=order_time,
processed=order_time,
finalized=finalized_time))
return self.order_dao.insert(order)
def _insert_samples(self, participant, tests, sample_ids, confirmed_time,
created_time):
for test_code, sample_id in zip(tests, sample_ids):
self.sample_dao.insert(
BiobankStoredSample(biobankStoredSampleId=sample_id,
biobankId=participant.biobankId,
test=test_code,
confirmed=confirmed_time,
created=created_time))
def _submit_race_questionnaire_response(self, participant_id,
race_answers):
code_answers = []
for answer in race_answers:
_add_code_answer(code_answers, "race", answer)
qr = make_questionnaire_response_json(participant_id,
self._questionnaire_id,
code_answers=code_answers)
self.send_post('Participant/%s/QuestionnaireResponse' % participant_id,
qr)
def test_reconciliation_query(self):
self.setup_codes([RACE_QUESTION_CODE], CodeType.QUESTION)
self.setup_codes([RACE_AIAN_CODE, RACE_WHITE_CODE], CodeType.ANSWER)
self._questionnaire_id = self.create_questionnaire(
'questionnaire3.json')
# MySQL and Python sub-second rounding differs, so trim micros from generated times.
order_time = clock.CLOCK.now().replace(microsecond=0)
old_order_time = order_time - datetime.timedelta(days=10)
within_24_hours = order_time + datetime.timedelta(hours=23)
old_within_24_hours = old_order_time + datetime.timedelta(hours=23)
late_time = order_time + datetime.timedelta(hours=25)
old_late_time = old_order_time + datetime.timedelta(hours=25)
file_time = order_time + datetime.timedelta(
hours=23) + datetime.timedelta(minutes=59)
two_days_ago = file_time - datetime.timedelta(days=2)
# On time, recent order and samples; shows up in rx
p_on_time = self._insert_participant()
# Extra samples ordered now aren't considered missing or late.
self._insert_order(p_on_time,
'GoodOrder',
BIOBANK_TESTS[:4],
order_time,
finalized_tests=BIOBANK_TESTS[:3],
kit_id='kit1',
tracking_number='t1')
self._insert_samples(p_on_time, BIOBANK_TESTS[:2],
['GoodSample1', 'GoodSample2'], within_24_hours,
within_24_hours - datetime.timedelta(hours=1))
# On time order and samples from 10 days ago; shows up in rx
p_old_on_time = self._insert_participant(race_codes=[RACE_AIAN_CODE])
# Old missing samples from 10 days ago don't show up in missing or late.
self._insert_order(p_old_on_time,
'OldGoodOrder',
BIOBANK_TESTS[:3],
old_order_time,
kit_id='kit2')
self._insert_samples(p_old_on_time, BIOBANK_TESTS[:2],
['OldGoodSample1', 'OldGoodSample2'],
old_within_24_hours,
old_within_24_hours - datetime.timedelta(hours=1))
# Late, recent order and samples; shows up in rx and late. (But not missing, as it hasn't been
# 24 hours since the order.)
p_late_and_missing = self._insert_participant()
# Extra missing sample doesn't show up as missing as it hasn't been 24 hours yet.
o_late_and_missing = self._insert_order(p_late_and_missing,
'SlowOrder', BIOBANK_TESTS[:3],
order_time)
self._insert_samples(p_late_and_missing, [BIOBANK_TESTS[0]],
['LateSample'], late_time,
late_time - datetime.timedelta(minutes=59))
# Late order and samples from 10 days ago; shows up in rx (but not missing, as it was too
# long ago.
p_old_late_and_missing = self._insert_participant()
self._insert_order(p_old_late_and_missing, 'OldSlowOrder',
BIOBANK_TESTS[:2], old_order_time)
self._insert_samples(p_old_late_and_missing, [BIOBANK_TESTS[0]],
['OldLateSample'], old_late_time,
old_late_time - datetime.timedelta(minutes=59))
# Order with missing sample from 2 days ago; shows up in rx and missing.
p_two_days_missing = self._insert_participant()
# The third test doesn't wind up in missing, as it was never finalized.
self._insert_order(p_two_days_missing,
'TwoDaysMissingOrder',
BIOBANK_TESTS[:3],
two_days_ago,
finalized_tests=BIOBANK_TESTS[:2])
# Recent samples with no matching order; shows up in missing.
p_extra = self._insert_participant(race_codes=[RACE_WHITE_CODE])
self._insert_samples(p_extra, [BIOBANK_TESTS[-1]],
['NobodyOrderedThisSample'], order_time,
order_time - datetime.timedelta(minutes=59))
# Old samples with no matching order; shows up in rx.
p_old_extra = self._insert_participant(race_codes=[RACE_AIAN_CODE])
self._insert_samples(p_old_extra, [BIOBANK_TESTS[-1]],
['OldNobodyOrderedThisSample'], old_order_time,
old_order_time - datetime.timedelta(hours=1))
# Withdrawn participants don't show up in any reports except withdrawal report.
p_withdrawn_old_on_time = self._insert_participant(
race_codes=[RACE_AIAN_CODE])
# This updates the version of the participant and its HPO ID.
self._insert_order(p_withdrawn_old_on_time, 'OldWithdrawnGoodOrder',
BIOBANK_TESTS[:2], old_order_time)
p_withdrawn_old_on_time = self.participant_dao.get(
p_withdrawn_old_on_time.participantId)
self._insert_samples(
p_withdrawn_old_on_time, BIOBANK_TESTS[:2],
['OldWithdrawnGoodSample1', 'OldWithdrawnGoodSample2'],
old_within_24_hours,
old_within_24_hours - datetime.timedelta(hours=1))
self._withdraw(p_withdrawn_old_on_time, within_24_hours)
p_withdrawn_late_and_missing = self._insert_participant()
self._insert_order(p_withdrawn_late_and_missing, 'WithdrawnSlowOrder',
BIOBANK_TESTS[:2], order_time)
self._insert_samples(p_withdrawn_late_and_missing, [BIOBANK_TESTS[0]],
['WithdrawnLateSample'], late_time,
late_time - datetime.timedelta(minutes=59))
p_withdrawn_late_and_missing = (self.participant_dao.get(
p_withdrawn_late_and_missing.participantId))
self._withdraw(p_withdrawn_late_and_missing, within_24_hours)
p_withdrawn_old_late_and_missing = self._insert_participant()
self._insert_order(p_withdrawn_old_late_and_missing,
'WithdrawnOldSlowOrder', BIOBANK_TESTS[:2],
old_order_time)
self._insert_samples(p_withdrawn_old_late_and_missing,
[BIOBANK_TESTS[0]], ['WithdrawnOldLateSample'],
old_late_time,
old_late_time - datetime.timedelta(minutes=59))
p_withdrawn_old_late_and_missing = (self.participant_dao.get(
p_withdrawn_old_late_and_missing.participantId))
self._withdraw(p_withdrawn_old_late_and_missing, old_late_time)
p_withdrawn_extra = self._insert_participant(
race_codes=[RACE_WHITE_CODE])
self._insert_samples(p_withdrawn_extra, [BIOBANK_TESTS[-1]],
['WithdrawnNobodyOrderedThisSample'], order_time,
order_time - datetime.timedelta(hours=1))
self._withdraw(p_withdrawn_extra, within_24_hours)
p_withdrawn_old_extra = self._insert_participant(
race_codes=[RACE_AIAN_CODE])
self._insert_samples(p_withdrawn_old_extra, [BIOBANK_TESTS[-1]],
['WithdrawnOldNobodyOrderedThisSample'],
old_order_time,
old_order_time - datetime.timedelta(hours=1))
self._withdraw(p_withdrawn_old_extra, within_24_hours)
p_withdrawn_race_change = self._insert_participant(
race_codes=[RACE_AIAN_CODE])
p_withdrawn_race_change_id = to_client_participant_id(
p_withdrawn_race_change.participantId)
self._submit_race_questionnaire_response(p_withdrawn_race_change_id,
[RACE_WHITE_CODE])
self._withdraw(p_withdrawn_race_change, within_24_hours)
# for the same participant/test, 3 orders sent and only 2 samples received. Shows up in both
# missing (we are missing one sample) and late (the two samples that were received were after
# 24 hours.)
p_repeated = self._insert_participant()
for repetition in xrange(3):
self._insert_order(
p_repeated, 'RepeatedOrder%d' % repetition, [BIOBANK_TESTS[0]],
two_days_ago + datetime.timedelta(hours=repetition))
if repetition != 2:
self._insert_samples(
p_repeated, [BIOBANK_TESTS[0]],
['RepeatedSample%d' % repetition],
within_24_hours + datetime.timedelta(hours=repetition),
within_24_hours + datetime.timedelta(hours=repetition - 1))
received, late, missing, withdrawals = 'rx.csv', 'late.csv', 'missing.csv', 'withdrawals.csv'
exporter = InMemorySqlExporter(self)
biobank_samples_pipeline._query_and_write_reports(
exporter, file_time, received, late, missing, withdrawals)
exporter.assertFilesEqual((received, late, missing, withdrawals))
# sent-and-received: 4 on-time, 2 late, none of the missing/extra/repeated ones;
# includes orders/samples from more than 7 days ago
exporter.assertRowCount(received, 6)
exporter.assertColumnNamesEqual(received, _CSV_COLUMN_NAMES)
row = exporter.assertHasRow(
received, {
'biobank_id': to_client_biobank_id(p_on_time.biobankId),
'sent_test': BIOBANK_TESTS[0],
'received_test': BIOBANK_TESTS[0]
})
# Also check the values of all remaining fields on one row.
self.assertEquals(row['source_site_name'],
'Monroeville Urgent Care Center')
self.assertEquals(row['source_site_consortium'], 'Pittsburgh')
self.assertEquals(row['source_site_mayolink_client_number'], '7035769')
self.assertEquals(row['source_site_hpo'], 'PITT')
self.assertEquals(row['source_site_hpo_type'], 'HPO')
self.assertEquals(row['finalized_site_name'],
'Monroeville Urgent Care Center')
self.assertEquals(row['finalized_site_consortium'], 'Pittsburgh')
self.assertEquals(row['finalized_site_mayolink_client_number'],
'7035769')
self.assertEquals(row['finalized_site_hpo'], 'PITT')
self.assertEquals(row['finalized_site_hpo_type'], 'HPO')
self.assertEquals(row['finalized_username'], '*****@*****.**')
self.assertEquals(row['sent_finalized_time'],
database_utils.format_datetime(order_time))
self.assertEquals(row['sent_collection_time'],
database_utils.format_datetime(order_time))
self.assertEquals(row['sent_processed_time'],
database_utils.format_datetime(order_time))
self.assertEquals(row['received_time'],
database_utils.format_datetime(within_24_hours))
self.assertEquals(
row['Sample Family Create Date'],
database_utils.format_datetime(within_24_hours -
datetime.timedelta(hours=1)))
self.assertEquals(row['sent_count'], '1')
self.assertEquals(row['received_count'], '1')
self.assertEquals(row['sent_order_id'], 'OGoodOrder')
self.assertEquals(row['received_sample_id'], 'GoodSample1')
self.assertEquals(row['biospecimen_kit_id'], 'kit1')
self.assertEquals(row['fedex_tracking_number'], 't1')
# the other sent-and-received rows
exporter.assertHasRow(
received, {
'biobank_id': to_client_biobank_id(p_on_time.biobankId),
'sent_test': BIOBANK_TESTS[1]
})
exporter.assertHasRow(
received, {
'biobank_id': to_client_biobank_id(
p_late_and_missing.biobankId),
'sent_test': BIOBANK_TESTS[0]
})
exporter.assertHasRow(
received, {
'biobank_id': to_client_biobank_id(p_old_on_time.biobankId),
'sent_test': BIOBANK_TESTS[0]
})
exporter.assertHasRow(
received, {
'biobank_id': to_client_biobank_id(p_old_on_time.biobankId),
'sent_test': BIOBANK_TESTS[1]
})
exporter.assertHasRow(
received, {
'biobank_id':
to_client_biobank_id(p_old_late_and_missing.biobankId),
'sent_test':
BIOBANK_TESTS[0]
})
# sent-and-received: 2 late; don't include orders/samples from more than 7 days ago
exporter.assertRowCount(late, 2)
exporter.assertColumnNamesEqual(late, _CSV_COLUMN_NAMES)
exporter.assertHasRow(
late, {
'biobank_id': to_client_biobank_id(
p_late_and_missing.biobankId),
'sent_order_id': 'O%s' % o_late_and_missing.biobankOrderId,
'elapsed_hours': '24'
})
exporter.assertHasRow(
late, {
'biobank_id': to_client_biobank_id(p_repeated.biobankId),
'elapsed_hours': '45'
})
# orders/samples where something went wrong; don't include orders/samples from more than 7
# days ago, or where 24 hours hasn't elapsed yet.
exporter.assertRowCount(missing, 4)
exporter.assertColumnNamesEqual(missing, _CSV_COLUMN_NAMES)
# sample received, nothing ordered
exporter.assertHasRow(
missing, {
'biobank_id': to_client_biobank_id(p_extra.biobankId),
'sent_order_id': ''
})
# order received, no sample
exporter.assertHasRow(
missing, {
'biobank_id': to_client_biobank_id(
p_two_days_missing.biobankId),
'sent_order_id': 'OTwoDaysMissingOrder',
'sent_test': BIOBANK_TESTS[0]
})
exporter.assertHasRow(
missing, {
'biobank_id': to_client_biobank_id(
p_two_days_missing.biobankId),
'sent_order_id': 'OTwoDaysMissingOrder',
'sent_test': BIOBANK_TESTS[1]
})
# 3 orders sent, only 2 received
multi_sample_row = exporter.assertHasRow(
missing, {
'biobank_id': to_client_biobank_id(p_repeated.biobankId),
'sent_count': '3',
'received_count': '2'
})
# Also verify the comma-joined fields of the row with multiple orders/samples.
self.assertItemsEqual(
multi_sample_row['sent_order_id'].split(','),
['ORepeatedOrder1', 'ORepeatedOrder0', 'ORepeatedOrder2'])
self.assertItemsEqual(
multi_sample_row['received_sample_id'].split(','),
['RepeatedSample0', 'RepeatedSample1'])
# We don't include the old withdrawal.
exporter.assertRowCount(withdrawals, 5)
exporter.assertHasRow(
withdrawals, {
'biobank_id':
to_client_biobank_id(p_withdrawn_old_on_time.biobankId),
'withdrawal_time':
database_utils.format_datetime(within_24_hours),
'is_native_american':
'Y'
})
exporter.assertHasRow(
withdrawals, {
'biobank_id':
to_client_biobank_id(p_withdrawn_late_and_missing.biobankId),
'withdrawal_time':
database_utils.format_datetime(within_24_hours),
'is_native_american':
'N'
})
exporter.assertHasRow(
withdrawals, {
'biobank_id': to_client_biobank_id(
p_withdrawn_extra.biobankId),
'withdrawal_time':
database_utils.format_datetime(within_24_hours),
'is_native_american': 'N'
})
exporter.assertHasRow(
withdrawals, {
'biobank_id': to_client_biobank_id(
p_withdrawn_old_extra.biobankId),
'withdrawal_time':
database_utils.format_datetime(within_24_hours),
'is_native_american': 'Y'
})
exporter.assertHasRow(
withdrawals, {
'biobank_id':
to_client_biobank_id(p_withdrawn_race_change.biobankId),
'withdrawal_time':
database_utils.format_datetime(within_24_hours),
'is_native_american':
'N'
})
def _create_data(self):
HPODao().insert(HPO(hpoId=PITT_HPO_ID + 1, name='AZ_TUCSON_2'))
HPODao().insert(HPO(hpoId=PITT_HPO_ID + 4, name='TEST'))
SqlTestBase.setup_codes(
ANSWER_FIELD_TO_QUESTION_CODE.values() + [EHR_CONSENT_QUESTION_CODE],
code_type=CodeType.QUESTION)
SqlTestBase.setup_codes(
FIELD_TO_QUESTIONNAIRE_MODULE_CODE.values(), code_type=CodeType.MODULE)
# Import codes for white and female, but not male or black.
SqlTestBase.setup_codes([
RACE_WHITE_CODE, CONSENT_PERMISSION_YES_CODE, RACE_NONE_OF_THESE_CODE,
PMI_PREFER_NOT_TO_ANSWER_CODE, CONSENT_PERMISSION_NO_CODE, 'female', 'PIIState_VA',
PMI_SKIP_CODE
], code_type=CodeType.ANSWER)
participant_dao = ParticipantDao()
questionnaire_id = self.create_questionnaire('questionnaire3.json')
questionnaire_id_2 = self.create_questionnaire('questionnaire4.json')
questionnaire_id_3 = self.create_questionnaire('all_consents_questionnaire.json')
pl_tucson = make_primary_provider_link_for_name('AZ_TUCSON')
pl_test = make_primary_provider_link_for_name('TEST')
pl_pitt = make_primary_provider_link_for_name('PITT')
with FakeClock(TIME):
participant = Participant(participantId=1, biobankId=2, providerLink=pl_tucson)
participant_dao.insert(participant)
self.send_consent('P1', email='*****@*****.**')
# Participant 2 starts out unpaired; later gets paired automatically when their physical
# measurements come in.
participant2 = Participant(participantId=2, biobankId=3)
participant_dao.insert(participant2)
self.send_consent('P2', email='*****@*****.**')
# Test HPO affiliation; this test participant is ignored.
participant3 = Participant(participantId=3, biobankId=4, providerLink=pl_test)
participant_dao.insert(participant3)
self.send_consent('P3', email='*****@*****.**')
# example.com e-mail; this test participant is ignored, too.
participant4 = Participant(participantId=4, biobankId=5, providerLink=pl_pitt)
participant_dao.insert(participant4)
self.send_consent('P4', email='*****@*****.**')
participant5 = Participant(participantId=5, biobankId=6, providerLink=pl_tucson)
participant_dao.insert(participant5)
self.send_consent('P5', email='*****@*****.**')
with FakeClock(TIME_2):
# FIXME: The test passes, but the following "update" doesn't actually make much sense. The
# providerlink is not changed but the HPO ID actually is (at this point in time
# `participant.hpoId` evaluates to 4, which is the value given in `unit_test_util.AZ_HPO_ID`).
# The original intent of the test is not clear.
# This update to participant has no effect, as the HPO ID didn't change.
participant = self._participant_with_defaults(
participantId=1, version=1, biobankId=2,
hpoId=3, # <<<< Had to add hpoId here, default is UNSET_HPO_ID
providerLink=pl_tucson
)
participant_dao.update(participant)
self.submit_questionnaire_response('P1', questionnaire_id,
race_code=RACE_WHITE_CODE,
gender_code='male',
state=PMI_SKIP_CODE,
date_of_birth=datetime.date(1980, 1, 2))
self.submit_questionnaire_response('P2', questionnaire_id,
race_code=RACE_NONE_OF_THESE_CODE,
gender_code=None,
state=None,
date_of_birth=None)
self.submit_questionnaire_response('P5', questionnaire_id,
race_code=PMI_SKIP_CODE,
gender_code=PMI_SKIP_CODE,
state=None,
date_of_birth=None)
with FakeClock(TIME_3):
# Re-pair the original participant
participant.version = 2
participant.providerLink = pl_pitt
participant_dao.update(participant)
self.send_post('Participant/P2/PhysicalMeasurements', load_measurement_json(2))
self.send_post('Participant/P2/BiobankOrder', load_biobank_order_json(2))
self.submit_questionnaire_response('P1', questionnaire_id,
race_code='black',
gender_code='female',
state=None,
date_of_birth=datetime.date(1980, 1, 3))
self.submit_questionnaire_response('P2', questionnaire_id,
race_code=None,
gender_code=PMI_PREFER_NOT_TO_ANSWER_CODE,
state=None,
date_of_birth=None)
self.submit_questionnaire_response('P2', questionnaire_id_2,
race_code=None,
gender_code=None,
state='PIIState_VA',
date_of_birth=None)
self.submit_consent_questionnaire_response('P1', questionnaire_id_3,
CONSENT_PERMISSION_NO_CODE)
self.submit_consent_questionnaire_response('P2', questionnaire_id_3,
CONSENT_PERMISSION_YES_CODE)
sample_dao = BiobankStoredSampleDao()
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='abc',
biobankId=2,
biobankOrderIdentifier='KIT',
test='test',
confirmed=TIME_2))
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='def',
biobankId=3,
biobankOrderIdentifier='KIT',
test='1SAL',
confirmed=TIME_2))
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='xyz',
biobankId=4,
biobankOrderIdentifier='KIT',
test='1SAL',
confirmed=TIME_2))
def _create_data(self):
HPODao().insert(HPO(hpoId=PITT_HPO_ID + 1, name='AZ_TUCSON'))
HPODao().insert(HPO(hpoId=PITT_HPO_ID + 2, name='TEST'))
SqlTestBase.setup_codes(
ANSWER_FIELD_TO_QUESTION_CODE.values() + [EHR_CONSENT_QUESTION_CODE],
code_type=CodeType.QUESTION)
SqlTestBase.setup_codes(
FIELD_TO_QUESTIONNAIRE_MODULE_CODE.values(), code_type=CodeType.MODULE)
# Import codes for white and female, but not male or black.
SqlTestBase.setup_codes(
[
RACE_WHITE_CODE, CONSENT_PERMISSION_YES_CODE,
CONSENT_PERMISSION_NO_CODE, 'female', 'PIIState_VA'
],
code_type=CodeType.ANSWER)
participant_dao = ParticipantDao()
questionnaire_id = self.create_questionnaire('questionnaire3.json')
questionnaire_id_2 = self.create_questionnaire('questionnaire4.json')
questionnaire_id_3 = self.create_questionnaire(
'all_consents_questionnaire.json')
with FakeClock(TIME):
participant = Participant(
participantId=1,
biobankId=2,
providerLink=make_primary_provider_link_for_name('AZ_TUCSON'))
participant_dao.insert(participant)
self.send_consent('P1', email='*****@*****.**')
with FakeClock(TIME):
# Participant 2 starts out unpaired; later gets paired automatically when their physical
# measurements come in.
participant2 = Participant(
participantId=2,
biobankId=3)
participant_dao.insert(participant2)
self.send_consent('P2', email='*****@*****.**')
with FakeClock(TIME):
# Test HPO affiliation; this test participant is ignored.
participant3 = Participant(participantId=3, biobankId=4,
providerLink=make_primary_provider_link_for_name('TEST'))
participant_dao.insert(participant3)
self.send_consent('P3', email='*****@*****.**')
# example.com e-mail; this test participant is ignored, too.
participant4 = Participant(participantId=4, biobankId=5,
providerLink=make_primary_provider_link_for_name('PITT'))
participant_dao.insert(participant4)
self.send_consent('P4', email='*****@*****.**')
with FakeClock(TIME_2):
# This update to participant has no effect, as the HPO ID didn't change.
participant = self._participant_with_defaults(
participantId=1, version=1, biobankId=2,
providerLink=make_primary_provider_link_for_name('AZ_TUCSON'))
participant_dao.update(participant)
self.submit_questionnaire_response('P1', questionnaire_id,
RACE_WHITE_CODE, 'male', None,
datetime.date(1980, 1, 2))
self.submit_questionnaire_response('P2', questionnaire_id, None, None,
None, None)
with FakeClock(TIME_3):
participant = self._participant_with_defaults(
participantId=1,
version=2,
biobankId=2,
providerLink=make_primary_provider_link_for_name('PITT'))
participant_dao.update(participant)
self.send_post('Participant/P2/PhysicalMeasurements',
load_measurement_json(2))
self.send_post('Participant/P2/BiobankOrder', load_biobank_order_json(2))
self.submit_questionnaire_response('P1', questionnaire_id, 'black',
'female', None,
datetime.date(1980, 1, 3))
self.submit_questionnaire_response('P2', questionnaire_id_2, None, None,
'PIIState_VA', None)
self._submit_consent_questionnaire_response('P1', questionnaire_id_3,
CONSENT_PERMISSION_NO_CODE)
self._submit_consent_questionnaire_response('P2', questionnaire_id_3,
CONSENT_PERMISSION_YES_CODE)
sample_dao = BiobankStoredSampleDao()
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='abc',
biobankId=2,
test='test',
confirmed=TIME_2))
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='def',
biobankId=3,
test='1SAL',
confirmed=TIME_2))
def _create_data(self):
HPODao().insert(HPO(hpoId=AZ_HPO_ID + 1, name='AZ_TUCSON_2'))
HPODao().insert(HPO(hpoId=PITT_HPO_ID + 4, name='TEST'))
SqlTestBase.setup_codes(
ANSWER_FIELD_TO_QUESTION_CODE.values() + [EHR_CONSENT_QUESTION_CODE],
code_type=CodeType.QUESTION)
SqlTestBase.setup_codes(
FIELD_TO_QUESTIONNAIRE_MODULE_CODE.values(), code_type=CodeType.MODULE)
# Import codes for white and female, but not male or black.
SqlTestBase.setup_codes(
[
RACE_WHITE_CODE, CONSENT_PERMISSION_YES_CODE,
RACE_NONE_OF_THESE_CODE, PMI_PREFER_NOT_TO_ANSWER_CODE,
CONSENT_PERMISSION_NO_CODE, 'female', 'PIIState_VA'
],
code_type=CodeType.ANSWER)
participant_dao = ParticipantDao()
questionnaire_id = self.create_questionnaire('questionnaire3.json')
questionnaire_id_2 = self.create_questionnaire('questionnaire4.json')
questionnaire_id_3 = self.create_questionnaire(
'all_consents_questionnaire.json')
with FakeClock(TIME):
participant = self._participant_with_defaults(
participantId=1,
version=2,
biobankId=2,
providerLink=make_primary_provider_link_for_name('PITT'))
participant_dao.insert(participant)
self.send_consent('P1', email='*****@*****.**')
# Participant 2 starts out unpaired; later gets paired automatically when their physical
# measurements come in.
participant2 = Participant(participantId=2, biobankId=3)
participant_dao.insert(participant2)
self.send_consent('P2', email='*****@*****.**')
# Test HPO affiliation; this test participant is ignored.
participant3 = Participant(
participantId=3,
biobankId=4,
providerLink=make_primary_provider_link_for_name('TEST'))
participant_dao.insert(participant3)
self.send_consent('P3', email='*****@*****.**')
# example.com e-mail; this test participant is ignored, too.
participant4 = Participant(
participantId=4,
biobankId=5,
providerLink=make_primary_provider_link_for_name('PITT'))
participant_dao.insert(participant4)
self.send_consent('P4', email='*****@*****.**')
participant5 = Participant(
participantId=5,
biobankId=6,
providerLink=make_primary_provider_link_for_name('PITT'))
participant_dao.insert(participant5)
self.send_consent('P5', email='*****@*****.**')
# A withdrawn participant should be excluded from metrics.
participant6 = Participant(
participantId=6,
biobankId=7,
providerLink=make_primary_provider_link_for_name('PITT')
)
participant_dao.insert(participant6)
self.send_consent('P6', email='*****@*****.**')
participant6.withdrawalStatus=WithdrawalStatus.NO_USE
participant_dao.update(participant6)
self.send_post('Participant/P2/PhysicalMeasurements',
load_measurement_json(2))
self.send_post('Participant/P2/BiobankOrder', load_biobank_order_json(2))
self.submit_questionnaire_response('P1', questionnaire_id,
RACE_WHITE_CODE, 'female', None,
datetime.date(1980, 1, 2))
self.submit_questionnaire_response(
'P2', questionnaire_id, PMI_PREFER_NOT_TO_ANSWER_CODE, 'male', None,
datetime.date(1920, 1, 3))
self.submit_questionnaire_response('P2', questionnaire_id_2, None, None,
'PIIState_VA', None)
self.submit_questionnaire_response('P5', questionnaire_id, None, None,
None, datetime.date(1970, 1, 2))
self.submit_consent_questionnaire_response('P1', questionnaire_id_3,
CONSENT_PERMISSION_NO_CODE)
self.submit_consent_questionnaire_response('P2', questionnaire_id_3,
CONSENT_PERMISSION_YES_CODE)
sample_dao = BiobankStoredSampleDao()
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='abc',
biobankId=2,
biobankOrderIdentifier='KIT',
test='test',
confirmed=TIME))
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='def',
biobankId=3,
biobankOrderIdentifier='KIT',
test='1SAL',
confirmed=TIME))
# Required to update the HPO linkage (and test filtering for P3).
sample_dao.insert(
BiobankStoredSample(
biobankStoredSampleId='xyz',
biobankId=4,
biobankOrderIdentifier='KIT',
test='1SAL',
confirmed=TIME))
def _store_biobank_sample(self, participant, test_code):
BiobankStoredSampleDao().insert(BiobankStoredSample(
biobankStoredSampleId='s' + participant['participantId'] + test_code,
biobankId=participant['biobankId'][1:],
test=test_code,
confirmed=TIME_1))