def test_same_id_different_identifier_not_ok(self):
ParticipantSummaryDao().insert(self.participant_summary(self.participant))
self.dao.insert(self._make_biobank_order(
identifiers=[BiobankOrderIdentifier(system='a', value='b')]))
with self.assertRaises(Conflict):
self.dao.insert(self._make_biobank_order(
identifiers=[BiobankOrderIdentifier(system='a', value='c')]))
def _add_identifiers_and_main_id(self, order, resource):
order.identifiers = []
dv_user = resource.auth_user.split("@")[0]
for i in resource.identifier:
try:
if i['system'].lower() == DV_FHIR_URL + 'trackingid':
order.identifiers.append(
BiobankOrderIdentifier(
system=BiobankDVOrder._DV_ID_SYSTEM[dv_user] +
'/trackingId',
value=i['value']))
except AttributeError:
raise BadRequest(
'No identifier for system %r, required for primary key.' %
BiobankDVOrder._DV_ID_SYSTEM[dv_user])
for i in resource.basedOn:
try:
if i['identifier']['system'].lower(
) == DV_FHIR_URL + 'orderid':
order.identifiers.append(
BiobankOrderIdentifier(
system=BiobankDVOrder._DV_ID_SYSTEM[dv_user],
value=i['identifier']['value']))
except AttributeError:
raise BadRequest(
'No identifier for system %r, required for primary key.' %
BiobankDVOrder._DV_ID_SYSTEM[dv_user])
def test_reject_used_identifier(self):
ParticipantSummaryDao().insert(self.participant_summary(self.participant))
self.dao.insert(self._make_biobank_order(
biobankOrderId='1',
identifiers=[BiobankOrderIdentifier(system='a', value='b')]))
with self.assertRaises(BadRequest):
self.dao.insert(self._make_biobank_order(
biobankOrderId='2',
identifiers=[BiobankOrderIdentifier(system='a', value='b')]))
def _insert_order(self,
participant,
order_id,
tests,
order_time,
finalized_tests=None,
kit_id=None,
tracking_number=None,
collected_note=None,
processed_note=None,
finalized_note=None):
order = BiobankOrder(biobankOrderId=order_id,
participantId=participant.participantId,
sourceSiteId=1,
finalizedSiteId=2,
collectedSiteId=1,
finalizedUsername='******',
created=order_time,
collectedNote=collected_note,
processedNote=processed_note,
finalizedNote=finalized_note,
samples=[])
id_1 = BiobankOrderIdentifier(
system="https://orders.mayomedicallaboratories.com",
value=order_id)
id_2 = BiobankOrderIdentifier(system="https://www.pmi-ops.org",
value='O%s' % order_id)
order.identifiers.append(id_1)
order.identifiers.append(id_2)
if kit_id:
order.identifiers.append(
BiobankOrderIdentifier(system=_KIT_ID_SYSTEM, value=kit_id))
if tracking_number:
order.identifiers.append(
BiobankOrderIdentifier(system=_TRACKING_NUMBER_SYSTEM,
value=tracking_number))
for test_code in tests:
finalized_time = order_time
if finalized_tests and not test_code in finalized_tests:
finalized_time = None
order.samples.append(
BiobankOrderedSample(biobankOrderId=order.biobankOrderId,
test=test_code,
description=u'test',
processingRequired=False,
collected=order_time,
processed=order_time,
finalized=finalized_time))
return self.order_dao.insert(order)
def test_cancelled_order_removes_from_participant_summary(self):
ParticipantSummaryDao().insert(self.participant_summary(self.participant))
samples = [BiobankOrderedSample(
test=self._B_TEST, processingRequired=True, description=u'new sample')]
biobank_order_id = 2
with clock.FakeClock(self.TIME_1):
order_1 = self.dao.insert(self._make_biobank_order())
with clock.FakeClock(self.TIME_2):
self.dao.insert(self._make_biobank_order(samples=samples,
biobankOrderId=biobank_order_id,
identifiers=[
BiobankOrderIdentifier(system='z',
value='x')]))
cancelled_request = self._get_cancel_patch()
ps_dao = ParticipantSummaryDao().get(self.participant.participantId)
self.assertEqual(ps_dao.sampleOrderStatus1ED10, OrderStatus.FINALIZED)
self.assertEqual(ps_dao.sampleOrderStatus1ED10Time, self.TIME_2)
self.assertEqual(ps_dao.sampleOrderStatus2ED10, OrderStatus.CREATED)
self.assertEqual(ps_dao.sampleOrderStatus2ED10Time, self.TIME_2)
self.dao.update_with_patch(order_1.biobankOrderId, cancelled_request,
order_1.version)
ps_dao = ParticipantSummaryDao().get(self.participant.participantId)
self.assertEqual(ps_dao.sampleOrderStatus1ED10, None)
self.assertEqual(ps_dao.sampleOrderStatus1ED10Time, None)
# should not remove the other order
self.assertEqual(ps_dao.sampleOrderStatus2ED10, OrderStatus.CREATED)
self.assertEqual(ps_dao.sampleOrderStatus2ED10Time, self.TIME_2)
self.assertEqual(ps_dao.biospecimenCollectedSiteId, 1)
self.assertEqual(ps_dao.biospecimenFinalizedSiteId, 2)
self.assertEqual(ps_dao.biospecimenProcessedSiteId, 1)
self.assertEqual(ps_dao.biospecimenStatus, OrderStatus.FINALIZED)
def _make_biobank_order(self, **kwargs):
"""Makes a new BiobankOrder (same values every time) with valid/complete defaults.
Kwargs pass through to BiobankOrder constructor, overriding defaults.
"""
participantId = kwargs['participantId']
modified = datetime.datetime(2019, 03, 25, 15, 59, 30)
for k, default_value in (
('biobankOrderId', u'1'),
('created', clock.CLOCK.now()),
# ('participantId', self.participant.participantId),
('sourceSiteId', 1),
('sourceUsername', u'*****@*****.**'),
('collectedSiteId', 1),
('collectedUsername', u'*****@*****.**'),
('processedSiteId', 1),
('processedUsername', u'*****@*****.**'),
('finalizedSiteId', 2),
('finalizedUsername', u'*****@*****.**'),
('version', 1),
('identifiers', [BiobankOrderIdentifier(system=u'a', value=u'c')]),
('samples', [
BiobankOrderedSample(test=u'1SAL2',
description=u'description',
processingRequired=True)
]),
('dvOrders', [
BiobankDVOrder(participantId=participantId,
modified=modified,
version=1)
])):
if k not in kwargs:
kwargs[k] = default_value
return BiobankOrder(**kwargs)
def test_amending_biobank_order_distinct_visit_count(self):
self.participant = self._insert(Participant(participantId=9, biobankId=13))
with clock.FakeClock(TIME_5):
order = self.order_dao.insert(self._make_biobank_order(biobankOrderId='2', identifiers=[
BiobankOrderIdentifier(system='b', value='d')], samples=[BiobankOrderedSample(
biobankOrderId='2',
finalized=TIME_5,
test=BIOBANK_TESTS[0],
description='description',
processingRequired=True)]))
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 1)
with clock.FakeClock(TIME_7):
amend_order = self._get_amended_info(order)
with self.order_dao.session() as session:
self.order_dao._do_update(session, amend_order, order)
# Shouldn't change on a simple amendment (unless finalized time on samples change)
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 1)
with clock.FakeClock(TIME_7_5):
cancel_request = cancel_biobank_order()
order = self.order_dao.update_with_patch(order.biobankOrderId, cancel_request, order.version)
# A cancelled order (even after amending) should reduce count (unless some other valid order on same day)
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 0)
def _make_biobank_order(self, **kwargs):
"""Makes a new BiobankOrder (same values every time) with valid/complete defaults.
Kwargs pass through to BiobankOrder constructor, overriding defaults.
"""
for k, default_value in (('biobankOrderId', '1'), ('created',
clock.CLOCK.now()),
('participantId',
self.participant.participantId),
('sourceSiteId', 1), ('sourceUsername',
'*****@*****.**'),
('collectedSiteId', 1), ('collectedUsername',
'*****@*****.**'),
('processedSiteId', 1), ('processedUsername',
'*****@*****.**'),
('finalizedSiteId',
2), ('finalizedUsername',
'*****@*****.**'), ('identifiers', [
BiobankOrderIdentifier(system='a',
value='c')
]), ('samples', [
BiobankOrderedSample(
biobankOrderId='1',
test=BIOBANK_TESTS[0],
description=u'description',
finalized=self.TIME_1,
processingRequired=True)
])):
if k not in kwargs:
kwargs[k] = default_value
return BiobankOrder(**kwargs)
def test_read_from_csv_file(self):
participant = self.participant_dao.insert(Participant(participantId=123, biobankId=1234))
self.summary_dao.insert(self.participant_summary(participant))
bo = self._make_biobank_order(participantId=participant.participantId, biobankOrderId='123',
identifiers=[BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org', value=u'12345678')])
BiobankOrderDao().insert(bo)
participant2 = self.participant_dao.insert(Participant(participantId=124, biobankId=1235))
self.summary_dao.insert(self.participant_summary(participant2))
bo2 = self._make_biobank_order(participantId=participant2.participantId, biobankOrderId='124',
identifiers=[BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org', value=u'12345679')])
BiobankOrderDao().insert(bo2)
participant3 = self.participant_dao.insert(Participant(participantId=125, biobankId=1236))
self.summary_dao.insert(self.participant_summary(participant3))
bo3 = self._make_biobank_order(participantId=participant3.participantId, biobankOrderId='125',
identifiers=[BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org', value=u'12345680')])
BiobankOrderDao().insert(bo3)
samples_file = test_data.open_genomic_set_file('Genomic-Test-Set-test-1.csv')
input_filename = 'cloud%s.csv' % self._naive_utc_to_naive_central(clock.CLOCK.now()).strftime(
genomic_set_file_handler.INPUT_CSV_TIME_FORMAT)
self._write_cloud_csv(input_filename, samples_file)
genomic_set_file_handler.read_genomic_set_from_bucket()
set_dao = GenomicSetDao()
obj = set_dao.get_all()[0]
self.assertEqual(obj.genomicSetName, 'name_xxx')
self.assertEqual(obj.genomicSetCriteria, 'criteria_xxx')
self.assertEqual(obj.genomicSetVersion, 1)
member_dao = GenomicSetMemberDao()
items = member_dao.get_all()
for item in items:
self.assertIn(item.participantId, [123, 124, 125])
self.assertIn(item.biobankOrderId, ['123', '124', '125'])
self.assertIn(item.biobankId, ['1234', '1235', '1236'])
self.assertIn(item.biobankOrderClientId, ['12345678', '12345679', '12345680'])
self.assertEqual(item.genomicSetId, 1)
self.assertIn(item.genomeType, ['aou_wgs', 'aou_array'])
self.assertIn(item.nyFlag, [0, 1])
self.assertIn(item.sexAtBirth, ['F', 'M'])
def _add_identifiers_and_main_id(self, order, resource):
order.identifiers = []
for i in resource.identifier:
try:
if i['system'] == 'orderId':
order.identifiers.append(
BiobankOrderIdentifier(
system=BiobankDVOrder._VIBRENT_ID_SYSTEM,
value=i['value']))
if i['system'] == 'fulfillmentId':
order.identifiers.append(
BiobankOrderIdentifier(
system=BiobankDVOrder._VIBRENT_ID_SYSTEM +
'/fulfillmentId',
value=i['value']))
except AttributeError:
raise BadRequest(
'No identifier for system %r, required for primary key.' %
BiobankDVOrder._VIBRENT_ID_SYSTEM)
def test_pm_restore_cancel_biobank_restore_cancel(self):
self.participant = self._insert(Participant(participantId=9, biobankId=13))
self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId,
TIME_4.isoformat()))
measurement = self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=669,
finalized=TIME_4))
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 1)
with clock.FakeClock(TIME_5):
order = self.order_dao.insert(self._make_biobank_order(biobankOrderId='2', identifiers=[
BiobankOrderIdentifier(system='b', value='d')], samples=[BiobankOrderedSample(
biobankOrderId='2',
finalized=TIME_5,
test=BIOBANK_TESTS[0],
description='description',
processingRequired=True)]))
with clock.FakeClock(TIME_7):
summary = self.dao.get(self.participant.participantId)
# distinct count should be 2
self.assertEquals(summary.numberDistinctVisits, 2)
# cancel the measurement
cancel_measurement = get_restore_or_cancel_info()
with self.measurement_dao.session() as session:
self.measurement_dao.update_with_patch(measurement.physicalMeasurementsId, session,
cancel_measurement)
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 1)
with clock.FakeClock(TIME_7):
restore_measurement = get_restore_or_cancel_info(status='restored')
with self.measurement_dao.session() as session:
self.measurement_dao.update_with_patch(measurement.physicalMeasurementsId, session,
restore_measurement)
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 2)
cancel_request = cancel_biobank_order()
order = self.order_dao.update_with_patch(order.biobankOrderId, cancel_request, order.version)
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 1)
restore_order = get_restore_or_cancel_info(status='restored')
restore_order['amendedReason'] = 'some reason'
self.order_dao.update_with_patch(order.biobankOrderId, restore_order, order.version)
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 2)
def _add_identifiers_and_main_id(cls, order, resource):
found_main_id = False
for i in resource.identifier:
order.identifiers.append(
BiobankOrderIdentifier(system=i.system, value=i.value))
if i.system == BiobankOrder._MAIN_ID_SYSTEM:
order.biobankOrderId = i.value
found_main_id = True
if not found_main_id:
raise BadRequest(
'No identifier for system %r, required for primary key.' %
BiobankOrder._MAIN_ID_SYSTEM)
def test_schema_biobank_order_and_datetime_roundtrip(self):
bo_id = 1
now = isodate.parse_datetime('2016-01-04T10:28:50-04:00')
write_session = self.database.make_session()
p = self._create_participant(write_session)
site = Site(siteId=1,
siteName='site',
googleGroup='*****@*****.**',
mayolinkClientNumber=12345,
hpoId=1)
write_session.add(site)
write_session.commit()
bo = BiobankOrder(biobankOrderId=bo_id,
participantId=p.participantId,
created=now,
sourceSiteId=1,
version=1,
sourceUsername='******',
collectedSiteId=1,
collectedUsername='******',
processedSiteId=1,
processedUsername='******',
finalizedSiteId=1,
finalizedUsername='******',
collectedNote=r'written by ' + self.fake.last_name(),
processedNote=u'd',
finalizedNote=u'e',
logPosition=LogPosition())
bo.identifiers.append(BiobankOrderIdentifier(system='a', value='b'))
bo.samples.append(
BiobankOrderedSample(test='a',
description=u'a test invented by ' +
self.fake.first_name(),
processingRequired=True,
collected=now,
processed=now,
finalized=now))
write_session.add(bo)
write_session.commit()
read_session = self.database.make_session()
bo = read_session.query(BiobankOrder).get(bo_id)
self.assertEquals(
bo.created.isoformat(),
now.astimezone(tzutc()).replace(tzinfo=None).isoformat())
def test_create_genomic_set_result_file(self):
participant = self.participant_dao.insert(Participant(participantId=123, biobankId=123))
self.summary_dao.insert(self.participant_summary(participant))
bo = self._make_biobank_order(participantId=participant.participantId, biobankOrderId='123',
identifiers=[BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org', value=u'12345678')])
BiobankOrderDao().insert(bo)
participant2 = self.participant_dao.insert(Participant(participantId=124, biobankId=124))
self.summary_dao.insert(self.participant_summary(participant2))
bo2 = self._make_biobank_order(participantId=participant2.participantId, biobankOrderId='124',
identifiers=[BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org', value=u'12345679')])
BiobankOrderDao().insert(bo2)
participant3 = self.participant_dao.insert(Participant(participantId=125, biobankId=125))
self.summary_dao.insert(self.participant_summary(participant3))
bo3 = self._make_biobank_order(participantId=participant3.participantId, biobankOrderId='125',
identifiers=[BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org', value=u'12345680')])
BiobankOrderDao().insert(bo3)
genomic_set = self._create_fake_genomic_set('fake_genomic_set_name',
'fake_genomic_set_criteria',
'Genomic-Test-Set-v12019-04-05-00-30-10.CSV')
self._create_fake_genomic_member(genomic_set.id, participant.participantId, bo.biobankOrderId,
participant.biobankId, bo.identifiers[0].value,
validation_status=GenomicValidationStatus.VALID,
sex_at_birth='F', genome_type='aou_array', ny_flag='Y')
self._create_fake_genomic_member(genomic_set.id, participant2.participantId, bo2.biobankOrderId,
participant2.biobankId, bo2.identifiers[0].value,
validation_status=GenomicValidationStatus.INVALID_AGE,
sex_at_birth='M', genome_type='aou_array', ny_flag='N')
self._create_fake_genomic_member(genomic_set.id, participant3.participantId, bo3.biobankOrderId,
participant3.biobankId, bo3.identifiers[0].value,
validation_status=GenomicValidationStatus.INVALID_CONSENT,
sex_at_birth='F', genome_type='aou_wgs', ny_flag='Y')
genomic_set_file_handler.create_genomic_set_status_result_file(genomic_set.id)
expected_result_filename = 'Genomic-Test-Set-v12019-04-05-00-30-10-Validation-Result.CSV'
bucket_name = config.getSetting(config.GENOMIC_SET_BUCKET_NAME)
path = '/' + bucket_name + '/' + expected_result_filename
csv_file = cloudstorage_api.open(path)
csv_reader = csv.DictReader(csv_file, delimiter=',')
class ResultCsvColumns(object):
"""Names of CSV columns that we read from the genomic set upload."""
GENOMIC_SET_NAME = 'genomic_set_name'
GENOMIC_SET_CRITERIA = 'genomic_set_criteria'
PID = 'pid'
BIOBANK_ORDER_ID = 'biobank_order_id'
NY_FLAG = 'ny_flag'
SEX_AT_BIRTH = 'sex_at_birth'
GENOME_TYPE = 'genome_type'
STATUS = 'status'
INVALID_REASON = 'invalid_reason'
ALL = (GENOMIC_SET_NAME, GENOMIC_SET_CRITERIA, PID, BIOBANK_ORDER_ID, NY_FLAG, SEX_AT_BIRTH,
GENOME_TYPE, STATUS, INVALID_REASON)
missing_cols = set(ResultCsvColumns.ALL) - set(csv_reader.fieldnames)
self.assertEqual(len(missing_cols), 0)
rows = list(csv_reader)
self.assertEqual(len(rows), 3)
self.assertEqual(rows[0][ResultCsvColumns.GENOMIC_SET_NAME], 'fake_genomic_set_name')
self.assertEqual(rows[0][ResultCsvColumns.GENOMIC_SET_CRITERIA], 'fake_genomic_set_criteria')
self.assertEqual(rows[0][ResultCsvColumns.STATUS], 'valid')
self.assertEqual(rows[0][ResultCsvColumns.INVALID_REASON], '')
self.assertEqual(rows[0][ResultCsvColumns.PID], '123')
self.assertEqual(rows[0][ResultCsvColumns.BIOBANK_ORDER_ID], '123')
self.assertEqual(rows[0][ResultCsvColumns.NY_FLAG], 'Y')
self.assertEqual(rows[0][ResultCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[0][ResultCsvColumns.SEX_AT_BIRTH], 'F')
self.assertEqual(rows[1][ResultCsvColumns.GENOMIC_SET_NAME], 'fake_genomic_set_name')
self.assertEqual(rows[1][ResultCsvColumns.GENOMIC_SET_CRITERIA], 'fake_genomic_set_criteria')
self.assertEqual(rows[1][ResultCsvColumns.STATUS], 'invalid')
self.assertEqual(rows[1][ResultCsvColumns.INVALID_REASON], 'INVALID_AGE')
self.assertEqual(rows[1][ResultCsvColumns.PID], '124')
self.assertEqual(rows[1][ResultCsvColumns.BIOBANK_ORDER_ID], '124')
self.assertEqual(rows[1][ResultCsvColumns.NY_FLAG], 'N')
self.assertEqual(rows[1][ResultCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[1][ResultCsvColumns.SEX_AT_BIRTH], 'M')
self.assertEqual(rows[2][ResultCsvColumns.GENOMIC_SET_NAME], 'fake_genomic_set_name')
self.assertEqual(rows[2][ResultCsvColumns.GENOMIC_SET_CRITERIA], 'fake_genomic_set_criteria')
self.assertEqual(rows[2][ResultCsvColumns.STATUS], 'invalid')
self.assertEqual(rows[2][ResultCsvColumns.INVALID_REASON], 'INVALID_CONSENT')
self.assertEqual(rows[2][ResultCsvColumns.PID], '125')
self.assertEqual(rows[2][ResultCsvColumns.BIOBANK_ORDER_ID], '125')
self.assertEqual(rows[2][ResultCsvColumns.NY_FLAG], 'Y')
self.assertEqual(rows[2][ResultCsvColumns.GENOME_TYPE], 'aou_wgs')
self.assertEqual(rows[2][ResultCsvColumns.SEX_AT_BIRTH], 'F')
def test_end_to_end_valid_case(self):
participant = self._make_participant()
self._make_summary(participant)
self._make_biobank_order(participantId=participant.participantId,
biobankOrderId=participant.participantId,
identifiers=[
BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org',
value=u'12345678')
])
participant2 = self._make_participant()
self._make_summary(participant2)
self._make_biobank_order(participantId=participant2.participantId,
biobankOrderId=participant2.participantId,
identifiers=[
BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org',
value=u'12345679')
])
participant3 = self._make_participant()
self._make_summary(participant3)
self._make_biobank_order(participantId=participant3.participantId,
biobankOrderId=participant3.participantId,
identifiers=[
BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org',
value=u'12345680')
])
samples_file = test_data.open_genomic_set_file(
'Genomic-Test-Set-test-2.csv')
input_filename = 'Genomic-Test-Set-v1%s.csv' % self\
._naive_utc_to_naive_central(clock.CLOCK.now())\
.strftime(genomic_set_file_handler.INPUT_CSV_TIME_FORMAT)
self._write_cloud_csv(input_filename, samples_file)
manifest_result_file = test_data.open_genomic_set_file(
'Genomic-Manifest-Result-test.csv')
manifest_result_filename = 'Genomic-Manifest-Result-AoU-1-v1%s.csv' % self \
._naive_utc_to_naive_central(clock.CLOCK.now()) \
.strftime(genomic_set_file_handler.INPUT_CSV_TIME_FORMAT)
self._write_cloud_csv(manifest_result_filename,
manifest_result_file,
bucket=_FAKE_BIOBANK_SAMPLE_BUCKET,
folder=_FAKE_BUCKET_RESULT_FOLDER)
genomic_pipeline.process_genomic_water_line()
# verify result file
bucket_name = config.getSetting(config.GENOMIC_SET_BUCKET_NAME)
path = self._find_latest_genomic_set_csv(bucket_name,
'Validation-Result')
csv_file = cloudstorage_api.open(path)
csv_reader = csv.DictReader(csv_file, delimiter=',')
class ResultCsvColumns(object):
"""Names of CSV columns that we read from the genomic set upload."""
GENOMIC_SET_NAME = 'genomic_set_name'
GENOMIC_SET_CRITERIA = 'genomic_set_criteria'
PID = 'pid'
BIOBANK_ORDER_ID = 'biobank_order_id'
NY_FLAG = 'ny_flag'
SEX_AT_BIRTH = 'sex_at_birth'
GENOME_TYPE = 'genome_type'
STATUS = 'status'
INVALID_REASON = 'invalid_reason'
ALL = (GENOMIC_SET_NAME, GENOMIC_SET_CRITERIA, PID,
BIOBANK_ORDER_ID, NY_FLAG, SEX_AT_BIRTH, GENOME_TYPE,
STATUS, INVALID_REASON)
missing_cols = set(ResultCsvColumns.ALL) - set(csv_reader.fieldnames)
self.assertEqual(len(missing_cols), 0)
rows = list(csv_reader)
self.assertEqual(len(rows), 3)
self.assertEqual(rows[0][ResultCsvColumns.GENOMIC_SET_NAME],
'name_xxx')
self.assertEqual(rows[0][ResultCsvColumns.GENOMIC_SET_CRITERIA],
'criteria_xxx')
self.assertEqual(rows[0][ResultCsvColumns.STATUS], 'valid')
self.assertEqual(rows[0][ResultCsvColumns.INVALID_REASON], '')
self.assertEqual(rows[0][ResultCsvColumns.PID], '1')
self.assertEqual(rows[0][ResultCsvColumns.BIOBANK_ORDER_ID], '1')
self.assertEqual(rows[0][ResultCsvColumns.NY_FLAG], 'Y')
self.assertEqual(rows[0][ResultCsvColumns.GENOME_TYPE], 'aou_wgs')
self.assertEqual(rows[0][ResultCsvColumns.SEX_AT_BIRTH], 'M')
self.assertEqual(rows[1][ResultCsvColumns.GENOMIC_SET_NAME],
'name_xxx')
self.assertEqual(rows[1][ResultCsvColumns.GENOMIC_SET_CRITERIA],
'criteria_xxx')
self.assertEqual(rows[1][ResultCsvColumns.STATUS], 'valid')
self.assertEqual(rows[1][ResultCsvColumns.INVALID_REASON], '')
self.assertEqual(rows[1][ResultCsvColumns.PID], '2')
self.assertEqual(rows[1][ResultCsvColumns.BIOBANK_ORDER_ID], '2')
self.assertEqual(rows[1][ResultCsvColumns.NY_FLAG], 'N')
self.assertEqual(rows[1][ResultCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[1][ResultCsvColumns.SEX_AT_BIRTH], 'F')
self.assertEqual(rows[2][ResultCsvColumns.GENOMIC_SET_NAME],
'name_xxx')
self.assertEqual(rows[2][ResultCsvColumns.GENOMIC_SET_CRITERIA],
'criteria_xxx')
self.assertEqual(rows[2][ResultCsvColumns.STATUS], 'valid')
self.assertEqual(rows[2][ResultCsvColumns.INVALID_REASON], '')
self.assertEqual(rows[2][ResultCsvColumns.PID], '3')
self.assertEqual(rows[2][ResultCsvColumns.BIOBANK_ORDER_ID], '3')
self.assertEqual(rows[2][ResultCsvColumns.NY_FLAG], 'N')
self.assertEqual(rows[2][ResultCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[2][ResultCsvColumns.SEX_AT_BIRTH], 'M')
# verify manifest files
bucket_name = config.getSetting(config.BIOBANK_SAMPLES_BUCKET_NAME)
class ExpectedCsvColumns(object):
VALUE = 'value'
BIOBANK_ID = 'biobank_id'
SEX_AT_BIRTH = 'sex_at_birth'
GENOME_TYPE = 'genome_type'
NY_FLAG = 'ny_flag'
REQUEST_ID = 'request_id'
PACKAGE_ID = 'package_id'
ALL = (VALUE, SEX_AT_BIRTH, GENOME_TYPE, NY_FLAG, REQUEST_ID,
PACKAGE_ID)
path = self._find_latest_genomic_set_csv(bucket_name, 'Manifest')
csv_file = cloudstorage_api.open(path)
csv_reader = csv.DictReader(csv_file, delimiter=',')
missing_cols = set(ExpectedCsvColumns.ALL) - set(csv_reader.fieldnames)
self.assertEqual(len(missing_cols), 0)
rows = list(csv_reader)
self.assertEqual(rows[0][ExpectedCsvColumns.VALUE], '12345678')
self.assertEqual(rows[0][ExpectedCsvColumns.BIOBANK_ID], '1')
self.assertEqual(rows[0][ExpectedCsvColumns.SEX_AT_BIRTH], 'M')
self.assertEqual(rows[0][ExpectedCsvColumns.GENOME_TYPE], 'aou_wgs')
self.assertEqual(rows[0][ExpectedCsvColumns.NY_FLAG], 'Y')
self.assertEqual(rows[1][ExpectedCsvColumns.VALUE], '12345679')
self.assertEqual(rows[1][ExpectedCsvColumns.BIOBANK_ID], '2')
self.assertEqual(rows[1][ExpectedCsvColumns.SEX_AT_BIRTH], 'F')
self.assertEqual(rows[1][ExpectedCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[1][ExpectedCsvColumns.NY_FLAG], 'N')
self.assertEqual(rows[2][ExpectedCsvColumns.VALUE], '12345680')
self.assertEqual(rows[2][ExpectedCsvColumns.BIOBANK_ID], '3')
self.assertEqual(rows[2][ExpectedCsvColumns.SEX_AT_BIRTH], 'M')
self.assertEqual(rows[2][ExpectedCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[2][ExpectedCsvColumns.NY_FLAG], 'N')
# verify manifest result files
bucket_name = config.getSetting(config.BIOBANK_SAMPLES_BUCKET_NAME)
class ExpectedCsvColumns(object):
VALUE = 'value'
BIOBANK_ID = 'biobank_id'
SEX_AT_BIRTH = 'sex_at_birth'
GENOME_TYPE = 'genome_type'
NY_FLAG = 'ny_flag'
REQUEST_ID = 'request_id'
PACKAGE_ID = 'package_id'
ALL = (VALUE, SEX_AT_BIRTH, GENOME_TYPE, NY_FLAG, REQUEST_ID,
PACKAGE_ID)
path = self._find_latest_genomic_set_csv(bucket_name,
'Manifest-Result')
csv_file = cloudstorage_api.open(path)
csv_reader = csv.DictReader(csv_file, delimiter=',')
missing_cols = set(ExpectedCsvColumns.ALL) - set(csv_reader.fieldnames)
self.assertEqual(len(missing_cols), 0)
rows = list(csv_reader)
self.assertEqual(rows[0][ExpectedCsvColumns.VALUE], '12345678')
self.assertEqual(rows[0][ExpectedCsvColumns.BIOBANK_ID], '1')
self.assertEqual(rows[0][ExpectedCsvColumns.SEX_AT_BIRTH], 'M')
self.assertEqual(rows[0][ExpectedCsvColumns.GENOME_TYPE], 'aou_wgs')
self.assertEqual(rows[0][ExpectedCsvColumns.NY_FLAG], 'Y')
self.assertEqual(rows[0][ExpectedCsvColumns.PACKAGE_ID],
'PKG-XXXX-XXXX1')
self.assertEqual(rows[1][ExpectedCsvColumns.VALUE], '12345679')
self.assertEqual(rows[1][ExpectedCsvColumns.BIOBANK_ID], '2')
self.assertEqual(rows[1][ExpectedCsvColumns.SEX_AT_BIRTH], 'F')
self.assertEqual(rows[1][ExpectedCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[1][ExpectedCsvColumns.NY_FLAG], 'N')
self.assertEqual(rows[1][ExpectedCsvColumns.PACKAGE_ID],
'PKG-XXXX-XXXX2')
self.assertEqual(rows[2][ExpectedCsvColumns.VALUE], '12345680')
self.assertEqual(rows[2][ExpectedCsvColumns.BIOBANK_ID], '3')
self.assertEqual(rows[2][ExpectedCsvColumns.SEX_AT_BIRTH], 'M')
self.assertEqual(rows[2][ExpectedCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[2][ExpectedCsvColumns.NY_FLAG], 'N')
self.assertEqual(rows[2][ExpectedCsvColumns.PACKAGE_ID],
'PKG-XXXX-XXXX3')
# verify package id in database
member_dao = GenomicSetMemberDao()
members = member_dao.get_all()
for member in members:
self.assertIn(
member.packageId,
['PKG-XXXX-XXXX1', 'PKG-XXXX-XXXX2', 'PKG-XXXX-XXXX3'])
def test_end_to_end_invalid_case(self):
participant = self._make_participant()
self._make_summary(participant, dateOfBirth='2018-02-14')
self._make_biobank_order(participantId=participant.participantId,
biobankOrderId=participant.participantId,
identifiers=[
BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org',
value=u'12345678')
])
participant2 = self._make_participant()
self._make_summary(participant2,
consentForStudyEnrollmentTime=datetime.datetime(
1990, 1, 1))
self._make_biobank_order(participantId=participant2.participantId,
biobankOrderId=participant2.participantId,
identifiers=[
BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org',
value=u'12345679')
])
participant3 = self._make_participant()
self._make_summary(participant3, zipCode='')
self._make_biobank_order(participantId=participant3.participantId,
biobankOrderId=participant3.participantId,
identifiers=[
BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org',
value=u'12345680')
])
participant4 = self._make_participant()
self._make_summary(participant4)
self._make_biobank_order(
participantId=participant4.participantId,
biobankOrderId=participant4.participantId,
identifiers=[BiobankOrderIdentifier(system=u'c', value=u'e')])
samples_file = test_data.open_genomic_set_file(
'Genomic-Test-Set-test-3.csv')
input_filename = 'Genomic-Test-Set-v1%s.csv' % self\
._naive_utc_to_naive_central(clock.CLOCK.now())\
.strftime(genomic_set_file_handler.INPUT_CSV_TIME_FORMAT)
self._write_cloud_csv(input_filename, samples_file)
genomic_pipeline.process_genomic_water_line()
# verify result file
bucket_name = config.getSetting(config.GENOMIC_SET_BUCKET_NAME)
path = self._find_latest_genomic_set_csv(bucket_name,
'Validation-Result')
csv_file = cloudstorage_api.open(path)
csv_reader = csv.DictReader(csv_file, delimiter=',')
class ResultCsvColumns(object):
"""Names of CSV columns that we read from the genomic set upload."""
GENOMIC_SET_NAME = 'genomic_set_name'
GENOMIC_SET_CRITERIA = 'genomic_set_criteria'
PID = 'pid'
BIOBANK_ORDER_ID = 'biobank_order_id'
NY_FLAG = 'ny_flag'
SEX_AT_BIRTH = 'sex_at_birth'
GENOME_TYPE = 'genome_type'
STATUS = 'status'
INVALID_REASON = 'invalid_reason'
ALL = (GENOMIC_SET_NAME, GENOMIC_SET_CRITERIA, PID,
BIOBANK_ORDER_ID, NY_FLAG, SEX_AT_BIRTH, GENOME_TYPE,
STATUS, INVALID_REASON)
missing_cols = set(ResultCsvColumns.ALL) - set(csv_reader.fieldnames)
self.assertEqual(len(missing_cols), 0)
rows = list(csv_reader)
self.assertEqual(len(rows), 4)
self.assertEqual(rows[0][ResultCsvColumns.GENOMIC_SET_NAME],
'name_xxx')
self.assertEqual(rows[0][ResultCsvColumns.GENOMIC_SET_CRITERIA],
'criteria_xxx')
self.assertEqual(rows[0][ResultCsvColumns.STATUS], 'invalid')
self.assertEqual(rows[0][ResultCsvColumns.INVALID_REASON],
'INVALID_AGE')
self.assertEqual(rows[0][ResultCsvColumns.PID], '1')
self.assertEqual(rows[0][ResultCsvColumns.BIOBANK_ORDER_ID], '1')
self.assertEqual(rows[0][ResultCsvColumns.NY_FLAG], 'Y')
self.assertEqual(rows[0][ResultCsvColumns.GENOME_TYPE], 'aou_wgs')
self.assertEqual(rows[0][ResultCsvColumns.SEX_AT_BIRTH], 'M')
self.assertEqual(rows[1][ResultCsvColumns.GENOMIC_SET_NAME],
'name_xxx')
self.assertEqual(rows[1][ResultCsvColumns.GENOMIC_SET_CRITERIA],
'criteria_xxx')
self.assertEqual(rows[1][ResultCsvColumns.STATUS], 'invalid')
self.assertEqual(rows[1][ResultCsvColumns.INVALID_REASON],
'INVALID_CONSENT')
self.assertEqual(rows[1][ResultCsvColumns.PID], '2')
self.assertEqual(rows[1][ResultCsvColumns.BIOBANK_ORDER_ID], '2')
self.assertEqual(rows[1][ResultCsvColumns.NY_FLAG], 'N')
self.assertEqual(rows[1][ResultCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[1][ResultCsvColumns.SEX_AT_BIRTH], 'F')
self.assertEqual(rows[2][ResultCsvColumns.GENOMIC_SET_NAME],
'name_xxx')
self.assertEqual(rows[2][ResultCsvColumns.GENOMIC_SET_CRITERIA],
'criteria_xxx')
self.assertEqual(rows[2][ResultCsvColumns.STATUS], 'invalid')
self.assertEqual(rows[2][ResultCsvColumns.INVALID_REASON],
'INVALID_NY_ZIPCODE')
self.assertEqual(rows[2][ResultCsvColumns.PID], '3')
self.assertEqual(rows[2][ResultCsvColumns.BIOBANK_ORDER_ID], '3')
self.assertEqual(rows[2][ResultCsvColumns.NY_FLAG], 'N')
self.assertEqual(rows[2][ResultCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[2][ResultCsvColumns.SEX_AT_BIRTH], 'M')
self.assertEqual(rows[3][ResultCsvColumns.GENOMIC_SET_NAME],
'name_xxx')
self.assertEqual(rows[3][ResultCsvColumns.GENOMIC_SET_CRITERIA],
'criteria_xxx')
self.assertEqual(rows[3][ResultCsvColumns.STATUS], 'invalid')
self.assertEqual(rows[3][ResultCsvColumns.INVALID_REASON],
'INVALID_BIOBANK_ORDER_CLIENT_ID')
self.assertEqual(rows[3][ResultCsvColumns.PID], '4')
self.assertEqual(rows[3][ResultCsvColumns.BIOBANK_ORDER_ID], '4')
self.assertEqual(rows[3][ResultCsvColumns.NY_FLAG], 'Y')
self.assertEqual(rows[3][ResultCsvColumns.GENOME_TYPE], 'aou_wgs')
self.assertEqual(rows[3][ResultCsvColumns.SEX_AT_BIRTH], 'F')
def testNumberDistinctVisitsCounts(self):
self.participant = self._insert(Participant(participantId=7, biobankId=77))
# insert biobank order
order = self.order_dao.insert(self._make_biobank_order())
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 1)
cancel_request = cancel_biobank_order()
# cancel biobank order
self.order_dao.update_with_patch(order.biobankOrderId, cancel_request, order.version)
summary = self.dao.get(self.participant.participantId)
# distinct count should be 0
self.assertEquals(summary.numberDistinctVisits, 0)
self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId,
TIME_1.isoformat()))
# insert physical measurement
measurement = self.measurement_dao.insert(self._make_physical_measurements())
summary = self.dao.get(self.participant.participantId)
# count should be 1
self.assertEquals(summary.numberDistinctVisits, 1)
# cancel the measurement
cancel_measurement = get_restore_or_cancel_info()
with self.measurement_dao.session() as session:
self.measurement_dao.update_with_patch(measurement.physicalMeasurementsId, session,
cancel_measurement)
summary = self.dao.get(self.participant.participantId)
# count should be 0
self.assertEquals(summary.numberDistinctVisits, 0)
with clock.FakeClock(TIME_1):
self.order_dao.insert(self._make_biobank_order(biobankOrderId='2', identifiers=[
BiobankOrderIdentifier(system='b', value='d')], samples=[BiobankOrderedSample(
biobankOrderId='2',
test=BIOBANK_TESTS[0],
description='description',
processingRequired=True)]))
with clock.FakeClock(TIME_2):
self.measurement_dao.insert(self._make_physical_measurements(
physicalMeasurementsId=2))
summary = self.dao.get(self.participant.participantId)
# A PM on another day should add a new distinct count.
self.assertEquals(summary.numberDistinctVisits, 2)
with clock.FakeClock(TIME_3):
self.order_dao.insert(self._make_biobank_order(biobankOrderId='3', identifiers=[
BiobankOrderIdentifier(system='s', value='s')], samples=[BiobankOrderedSample(
biobankOrderId ='3',
finalized=TIME_3,
test=BIOBANK_TESTS[1],
description='another description',
processingRequired=False)]))
# a physical measurement on same day as biobank order does not add distinct visit.
self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=6))
# another biobank order on the same day should also not add a distinct visit
self.order_dao.insert(self._make_biobank_order(biobankOrderId='7', identifiers=[
BiobankOrderIdentifier(system='x', value='x')], samples=[BiobankOrderedSample(
biobankOrderId ='7',
finalized=TIME_3,
test=BIOBANK_TESTS[1],
description='another description',
processingRequired=False)]))
summary = self.dao.get(self.participant.participantId)
# 1 from each of TIME_1 TIME_2 TIME_3
self.assertEquals(summary.numberDistinctVisits, 3)
def test_qa_scenarios_for_pmb_visits(self):
""" PDR at https://docs.google.com/document/d/1sL54f-I91RvhjIprrdbwD8TlR9Jq91MX2ELf1EtJdxc/edit#heading=h.bqo8kt3igsrw<Paste> """
self.participant = self._insert(Participant(participantId=6, biobankId=66))
# test scenario 1
with clock.FakeClock(TIME_4):
self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId,
TIME_4.isoformat()))
self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=666,
participantId=self.participant.participantId,
finalized=TIME_4))
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 1)
with clock.FakeClock(TIME_5):
self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId,
TIME_5.isoformat()))
self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=669,
finalized=TIME_5))
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 2)
# test scenario 2
with clock.FakeClock(TIME_6):
self.participant = self._insert(Participant(participantId=9, biobankId=13))
self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId,
TIME_6.isoformat()))
self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=8,
finalized=TIME_6))
self.order_dao.insert(self._make_biobank_order(biobankOrderId='2', identifiers=[
BiobankOrderIdentifier(system='b', value='d')], samples=[BiobankOrderedSample(
biobankOrderId='2',
finalized=TIME_7,
test=BIOBANK_TESTS[0],
description='description',
processingRequired=True)]))
summary = self.dao.get(self.participant.participantId)
# distinct count should be 2
self.assertEquals(summary.numberDistinctVisits, 2)
# test scenario 3
with clock.FakeClock(TIME_6):
self.participant = self._insert(Participant(participantId=66, biobankId=42))
self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId,
TIME_6.isoformat()))
self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=12,
createdSiteId=2,
finalized=TIME_6))
self.order_dao.insert(self._make_biobank_order(biobankOrderId='18', finalizedSiteId=1, identifiers=[
BiobankOrderIdentifier(system='x', value='y')], samples=[BiobankOrderedSample(
biobankOrderId='18',
finalized=TIME_6,
test=BIOBANK_TESTS[0],
description='description',
processingRequired=True)]))
summary = self.dao.get(self.participant.participantId)
# distinct count should be 1
self.assertEquals(summary.numberDistinctVisits, 1)
# test scenario 4
with clock.FakeClock(TIME_8):
self.participant = self._insert(Participant(participantId=6613, biobankId=142))
self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId,
TIME_8.isoformat()))
self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=129,
finalized=TIME_8))
order = self.order_dao.insert(self._make_biobank_order(biobankOrderId='999', identifiers=[
BiobankOrderIdentifier(system='s', value='s')], samples=[BiobankOrderedSample(
biobankOrderId='999',
finalized=TIME_8,
test=BIOBANK_TESTS[1],
description='description',
processingRequired=True)]))
summary = self.dao.get(self.participant.participantId)
# distinct count should be 1
self.assertEquals(summary.numberDistinctVisits, 1)
# change finalized time, recalculating count
with self.order_dao.session() as session:
existing_order = copy.deepcopy(order)
order.samples[0].finalized = TIME_9
self.order_dao._do_update(session, order, existing_order)
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 1)
# change test, should not change count.
with self.order_dao.session() as session:
existing_order = copy.deepcopy(order)
order.samples[0].test = BIOBANK_TESTS[0]
self.order_dao._do_update(session, order, existing_order)
summary = self.dao.get(self.participant.participantId)
self.assertEquals(summary.numberDistinctVisits, 1)
# test scenario 5
with clock.FakeClock(TIME_12):
self.participant = self._insert(Participant(participantId=3000, biobankId=2019))
self.order_dao.insert(self._make_biobank_order(biobankOrderId='700', identifiers=[
BiobankOrderIdentifier(system='n', value='s')], samples=[BiobankOrderedSample(
biobankOrderId='700',
finalized=TIME_10,
test=BIOBANK_TESTS[1],
description='description',
processingRequired=True)]))
summary = self.dao.get(self.participant.participantId)
# distinct count should be 1
self.assertEquals(summary.numberDistinctVisits, 1)
other_order = self.order_dao.insert(self._make_biobank_order(biobankOrderId='701', identifiers=[
BiobankOrderIdentifier(system='n', value='t')], samples=[BiobankOrderedSample(
biobankOrderId='701',
finalized=TIME_11,
test=BIOBANK_TESTS[1],
description='description',
processingRequired=True)]))
summary = self.dao.get(self.participant.participantId)
# distinct count should be 2
self.assertEquals(summary.numberDistinctVisits, 2)
order = self.order_dao.insert(self._make_biobank_order(biobankOrderId='702', identifiers=[
BiobankOrderIdentifier(system='n', value='u')], samples=[BiobankOrderedSample(
biobankOrderId='702',
finalized=TIME_12,
test=BIOBANK_TESTS[1],
description='description',
processingRequired=True)]))
summary = self.dao.get(self.participant.participantId)
# distinct count should be 3
self.assertEquals(summary.numberDistinctVisits, 3)
self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId,
TIME_12.isoformat()))
self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=120,
finalized=TIME_12))
summary = self.dao.get(self.participant.participantId)
# distinct count should be 3
self.assertEquals(summary.numberDistinctVisits, 3)
cancel_request = cancel_biobank_order()
# cancel biobank order with PM on same day
self.order_dao.update_with_patch(order.biobankOrderId, cancel_request, order.version)
summary = self.dao.get(self.participant.participantId)
# distinct count should be 3 (the PM on same day still counts)
self.assertEquals(summary.numberDistinctVisits, 3)
self.measurement_json = json.dumps(load_measurement_json(self.participant.participantId,
TIME_1.isoformat()))
self.measurement_dao.insert(self._make_physical_measurements(physicalMeasurementsId=150,
finalized=TIME_1))
summary = self.dao.get(self.participant.participantId)
# distinct count should be 4
self.assertEquals(summary.numberDistinctVisits, 4)
# cancel order with pm on different day
self.order_dao.update_with_patch(other_order.biobankOrderId, cancel_request, order.version)
summary = self.dao.get(self.participant.participantId)
# distinct count should be 3
self.assertEquals(summary.numberDistinctVisits, 3)
def test_create_and_upload_biobank_manifest_file(self):
participant = self.participant_dao.insert(Participant(participantId=123, biobankId=123))
self.summary_dao.insert(self.participant_summary(participant))
bo = self._make_biobank_order(participantId=participant.participantId, biobankOrderId='123',
identifiers=[BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org', value=u'12345678')])
BiobankOrderDao().insert(bo)
participant2 = self.participant_dao.insert(Participant(participantId=124, biobankId=124))
self.summary_dao.insert(self.participant_summary(participant2))
bo2 = self._make_biobank_order(participantId=participant2.participantId, biobankOrderId='124',
identifiers=[BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org', value=u'12345679')])
BiobankOrderDao().insert(bo2)
participant3 = self.participant_dao.insert(Participant(participantId=125, biobankId=125))
self.summary_dao.insert(self.participant_summary(participant3))
bo3 = self._make_biobank_order(participantId=participant3.participantId, biobankOrderId='125',
identifiers=[BiobankOrderIdentifier(
system=u'https://www.pmi-ops.org', value=u'12345680')])
BiobankOrderDao().insert(bo3)
genomic_set = self._create_fake_genomic_set('fake_genomic_set_name',
'fake_genomic_set_criteria',
'Genomic-Test-Set-v12019-04-05-00-30-10.CSV')
self._create_fake_genomic_member(genomic_set.id, participant.participantId, bo.biobankOrderId,
participant.biobankId, bo.identifiers[0].value,
validation_status=GenomicValidationStatus.VALID,
sex_at_birth='F', genome_type='aou_array', ny_flag='Y')
self._create_fake_genomic_member(genomic_set.id, participant2.participantId, bo2.biobankOrderId,
participant2.biobankId, bo2.identifiers[0].value,
validation_status=GenomicValidationStatus.INVALID_AGE,
sex_at_birth='M', genome_type='aou_array', ny_flag='N')
self._create_fake_genomic_member(genomic_set.id, participant3.participantId, bo3.biobankOrderId,
participant3.biobankId, bo3.identifiers[0].value,
validation_status=GenomicValidationStatus.INVALID_CONSENT,
sex_at_birth='F', genome_type='aou_wgs', ny_flag='Y')
now = clock.CLOCK.now()
genomic_biobank_menifest_handler\
.create_and_upload_genomic_biobank_manifest_file(genomic_set.id, now)
bucket_name = config.getSetting(config.BIOBANK_SAMPLES_BUCKET_NAME)
# convert UTC to CDT
now_cdt_str = _UTC.localize(now).astimezone(_US_CENTRAL).replace(tzinfo=None) \
.strftime(_OUTPUT_CSV_TIME_FORMAT)
class ExpectedCsvColumns(object):
VALUE = 'value'
BIOBANK_ID = 'biobank_id'
SEX_AT_BIRTH = 'sex_at_birth'
GENOME_TYPE = 'genome_type'
NY_FLAG = 'ny_flag'
REQUEST_ID = 'request_id'
PACKAGE_ID = 'package_id'
ALL = (VALUE, SEX_AT_BIRTH, GENOME_TYPE, NY_FLAG, REQUEST_ID, PACKAGE_ID)
expected_result_filename = 'rdr_fake_sub_folder/Genomic-Manifest-AoU-1-v1' + \
now_cdt_str + '.CSV'
path = '/' + bucket_name + '/' + expected_result_filename
csv_file = cloudstorage_api.open(path)
csv_reader = csv.DictReader(csv_file, delimiter=',')
missing_cols = set(ExpectedCsvColumns.ALL) - set(csv_reader.fieldnames)
self.assertEqual(len(missing_cols), 0)
rows = list(csv_reader)
self.assertEqual(rows[0][ExpectedCsvColumns.VALUE], '12345678')
self.assertEqual(rows[0][ExpectedCsvColumns.BIOBANK_ID], '123')
self.assertEqual(rows[0][ExpectedCsvColumns.SEX_AT_BIRTH], 'F')
self.assertEqual(rows[0][ExpectedCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[0][ExpectedCsvColumns.NY_FLAG], 'Y')
self.assertEqual(rows[1][ExpectedCsvColumns.VALUE], '12345679')
self.assertEqual(rows[1][ExpectedCsvColumns.BIOBANK_ID], '124')
self.assertEqual(rows[1][ExpectedCsvColumns.SEX_AT_BIRTH], 'M')
self.assertEqual(rows[1][ExpectedCsvColumns.GENOME_TYPE], 'aou_array')
self.assertEqual(rows[1][ExpectedCsvColumns.NY_FLAG], 'N')
self.assertEqual(rows[2][ExpectedCsvColumns.VALUE], '12345680')
self.assertEqual(rows[2][ExpectedCsvColumns.BIOBANK_ID], '125')
self.assertEqual(rows[2][ExpectedCsvColumns.SEX_AT_BIRTH], 'F')
self.assertEqual(rows[2][ExpectedCsvColumns.GENOME_TYPE], 'aou_wgs')
self.assertEqual(rows[2][ExpectedCsvColumns.NY_FLAG], 'Y')
