def generate_samples(fraction_missing): """Creates fake sample CSV data in GCS. Args: fraction_missing: This many samples which exist as BiobankStoredSamples will not have rows generated in the fake CSV. """ bucket_name = config.getSetting(config.BIOBANK_SAMPLES_BUCKET_NAME) now = clock.CLOCK.now() file_name = '/%s/fake_%s.csv' % (bucket_name, now.strftime(INPUT_CSV_TIME_FORMAT)) num_rows = 0 sample_id_start = random.randint(1000000, 10000000) with cloudstorage_api.open(file_name, mode='w') as dest: writer = csv.writer(dest, delimiter="\t") writer.writerow(_HEADERS) biobank_order_dao = BiobankOrderDao() with biobank_order_dao.session() as session: rows = biobank_order_dao.get_ordered_samples_sample( session, 1 - fraction_missing, _BATCH_SIZE) for biobank_id, collected_time, test in rows: if collected_time is None: logging.warning( 'biobank_id=%s test=%s skipped (collected=%s)', biobank_id, test, collected_time) continue minutes_delta = random.randint( 0, _MAX_MINUTES_BETWEEN_SAMPLE_COLLECTED_AND_CONFIRMED) confirmed_time = collected_time + datetime.timedelta( minutes=minutes_delta) writer.writerow([ sample_id_start + num_rows, None, # no parent confirmed_time.strftime(_TIME_FORMAT), to_client_biobank_id(biobank_id), test, confirmed_time.strftime(_TIME_FORMAT), 'KIT' ]) # reuse confirmed time as created time num_rows += 1 participant_dao = ParticipantDao() with participant_dao.session() as session: rows = participant_dao.get_biobank_ids_sample( session, _PARTICIPANTS_WITH_ORPHAN_SAMPLES, _BATCH_SIZE) for biobank_id, sign_up_time in rows: minutes_delta = random.randint( 0, _MAX_MINUTES_BETWEEN_PARTICIPANT_CREATED_AND_CONFIRMED) confirmed_time = sign_up_time + datetime.timedelta( minutes=minutes_delta) tests = random.sample(BIOBANK_TESTS, random.randint(1, len(BIOBANK_TESTS))) for test in tests: writer.writerow([ sample_id_start + num_rows, None, confirmed_time.strftime(_TIME_FORMAT), to_client_biobank_id(biobank_id), test, confirmed_time.strftime(_TIME_FORMAT), 'KIT' ]) num_rows += 1 logging.info("Generated %d samples in %s.", num_rows, file_name)
def open_biobank_samples( biobank_id1, biobank_id2, biobank_id3, test1=None, test2=None, test3=None): """Returns an readable stream for the biobank samples CSV.""" with open(data_path('biobank_samples_1.csv')) as f: csv_str = f.read() % { 'biobank_id1': to_client_biobank_id(biobank_id1), 'biobank_id2': to_client_biobank_id(biobank_id2), 'biobank_id3': to_client_biobank_id(biobank_id3), 'test1': test1 or random.choice(BIOBANK_TESTS), 'test2': test2 or random.choice(BIOBANK_TESTS), 'test3': test3 or random.choice(BIOBANK_TESTS), } return StringIO.StringIO(csv_str)
def to_client_json(self, model): client_json = { 'participantId': to_client_participant_id(model.participantId), 'externalId': model.externalId, 'hpoId': model.hpoId, 'awardee': model.hpoId, 'organization': model.organizationId, 'siteId': model.siteId, 'biobankId': to_client_biobank_id(model.biobankId), 'lastModified': model.lastModified.isoformat(), 'signUpTime': model.signUpTime.isoformat(), 'providerLink': json.loads(model.providerLink), 'withdrawalStatus': model.withdrawalStatus, 'withdrawalReason': model.withdrawalReason, 'withdrawalReasonJustification': model.withdrawalReasonJustification, 'withdrawalTime': model.withdrawalTime, 'withdrawalAuthored': model.withdrawalAuthored, 'suspensionStatus': model.suspensionStatus, 'suspensionTime': model.suspensionTime } format_json_hpo(client_json, self.hpo_dao, 'hpoId'), format_json_org(client_json, self.organization_dao, 'organization'), format_json_site(client_json, self.site_dao, 'site'), format_json_enum(client_json, 'withdrawalStatus') format_json_enum(client_json, 'withdrawalReason') format_json_enum(client_json, 'suspensionStatus') format_json_date(client_json, 'withdrawalTime') format_json_date(client_json, 'suspensionTime') client_json['awardee'] = client_json['hpoId'] if 'siteId' in client_json: del client_json['siteId'] return client_json
def _new_row(sample_id, biobank_id, test, confirmed_time): row = [] disposed_time = confirmed_time + datetime.timedelta(minutes=random.randint( 0, _MAX_MINUTES_BETWEEN_SAMPLE_CONFIRMED_AND_DISPOSED)) for col in CsvColumns.ALL: if col == CsvColumns.SAMPLE_ID: row.append(sample_id) elif col == CsvColumns.PARENT_ID: row.append(None) elif col == CsvColumns.CONFIRMED_DATE: row.append(confirmed_time.strftime(_TIME_FORMAT)) elif col == CsvColumns.EXTERNAL_PARTICIPANT_ID: row.append(to_client_biobank_id(biobank_id)) elif col == CsvColumns.BIOBANK_ORDER_IDENTIFIER: row.append('KIT') elif col == CsvColumns.TEST_CODE: row.append(test) elif col == CsvColumns.CREATE_DATE: row.append(confirmed_time.strftime(_TIME_FORMAT)) elif col == CsvColumns.STATUS: # TODO: Do we want a distribution of statuses here? row.append('consumed') elif col == CsvColumns.DISPOSAL_DATE: row.append(disposed_time.strftime(_TIME_FORMAT)) elif col == CsvColumns.SAMPLE_FAMILY: # TODO: Is there a need for a more realistic value here? row.append('family_id') else: raise ValueError( "unsupported biobank CSV column: '{}'".format(col)) return row
def open_biobank_samples(biobank_ids, tests): """ Returns a string representing the biobank samples CSV file. The number of records returned is equal to the number of biobank_ids passed. :param biobank_ids: list of biobank ids. :param tests: list of tests :return: StringIO object """ nids = len(biobank_ids) # get the same number of sample lines as biobank_ids, plus header line. lines = open(data_path('biobank_samples_1.csv')).readlines()[:nids + 1] csv_str = lines[0] # include header line every time. for x in range(0, nids): # if we don't have a test code for this index, use a random one. try: test_code = tests[x] except IndexError: test_code = random.choice(BIOBANK_TESTS) csv_str += lines[x + 1].format(biobank_id=to_client_biobank_id( biobank_ids[x]), test=test_code) return csv_str
def to_client_json(self, model): result = model.asdict() # Participants that withdrew more than 48 hours ago should have fields other than # WITHDRAWN_PARTICIPANT_FIELDS cleared. if (model.withdrawalStatus == WithdrawalStatus.NO_USE and (model.withdrawalTime is None or model.withdrawalTime < clock.CLOCK.now() - WITHDRAWN_PARTICIPANT_VISIBILITY_TIME)): result = {k: result.get(k) for k in WITHDRAWN_PARTICIPANT_FIELDS} elif model.withdrawalStatus != WithdrawalStatus.NO_USE and \ model.suspensionStatus == SuspensionStatus.NO_CONTACT: for i in SUSPENDED_PARTICIPANT_FIELDS: result[i] = UNSET result['participantId'] = to_client_participant_id(model.participantId) biobank_id = result.get('biobankId') if biobank_id: result['biobankId'] = to_client_biobank_id(biobank_id) date_of_birth = result.get('dateOfBirth') if date_of_birth: result['ageRange'] = get_bucketed_age(date_of_birth, clock.CLOCK.now()) else: result['ageRange'] = UNSET if result.get('primaryLanguage') is None: result['primaryLanguage'] = UNSET if 'organizationId' in result: result['organization'] = result['organizationId'] del result['organizationId'] format_json_org(result, self.organization_dao, 'organization') format_json_hpo(result, self.hpo_dao, 'hpoId') result['awardee'] = result['hpoId'] _initialize_field_type_sets() for fieldname in _DATE_FIELDS: format_json_date(result, fieldname) for fieldname in _CODE_FIELDS: format_json_code(result, self.code_dao, fieldname) for fieldname in _ENUM_FIELDS: format_json_enum(result, fieldname) for fieldname in _SITE_FIELDS: format_json_site(result, self.site_dao, fieldname) if (model.withdrawalStatus == WithdrawalStatus.NO_USE or model.suspensionStatus == SuspensionStatus.NO_CONTACT): result['recontactMethod'] = 'NO_CONTACT' # Strip None values. result = {k: v for k, v in result.iteritems() if v is not None} return result
def test_sample_from_row(self): samples_file = test_data.open_biobank_samples(112, 222, 333) reader = csv.DictReader(samples_file, delimiter='\t') row = reader.next() sample = biobank_samples_pipeline._create_sample_from_row( row, get_biobank_id_prefix()) self.assertIsNotNone(sample) cols = biobank_samples_pipeline._Columns self.assertEquals(sample.biobankStoredSampleId, row[cols.SAMPLE_ID]) self.assertEquals(to_client_biobank_id(sample.biobankId), row[cols.EXTERNAL_PARTICIPANT_ID]) self.assertEquals(sample.test, row[cols.TEST_CODE]) confirmed_date = self._naive_utc_to_naive_central(sample.confirmed) self.assertEquals( confirmed_date.strftime( biobank_samples_pipeline._INPUT_TIMESTAMP_FORMAT), row[cols.CONFIRMED_DATE]) received_date = self._naive_utc_to_naive_central(sample.created) self.assertEquals( received_date.strftime( biobank_samples_pipeline._INPUT_TIMESTAMP_FORMAT), row[cols.CREATE_DATE])
def _filter_order_fields(self, resource, pid): fhir_resource = SimpleFhirR4Reader(resource) summary = ParticipantSummaryDao().get(pid) if not summary: raise BadRequest('No summary for participant id: {}'.format(pid)) code_dict = summary.asdict() format_json_code(code_dict, self.code_dao, 'genderIdentityId') format_json_code(code_dict, self.code_dao, 'stateId') if 'genderIdentity' in code_dict and code_dict['genderIdentity']: if code_dict['genderIdentity'] == 'GenderIdentity_Woman': gender_val = 'F' elif code_dict['genderIdentity'] == 'GenderIdentity_Man': gender_val = 'M' else: gender_val = 'U' else: gender_val = 'U' order_id = int(fhir_resource.basedOn[0].identifier.value) with self.session() as session: result = session.query(BiobankDVOrder.barcode).filter( BiobankDVOrder.order_id == order_id).first() barcode = None if not result else result if isinstance( result, str) else result.barcode # MayoLink api has strong opinions on what should be sent and the order of elements. Dont touch. order = { 'order': { 'collected': fhir_resource.occurrenceDateTime, 'account': '', 'number': barcode, 'patient': { 'medical_record_number': str(to_client_biobank_id(summary.biobankId)), 'first_name': '*', 'last_name': str(to_client_biobank_id(summary.biobankId)), 'middle_name': '', 'birth_date': '3/3/1933', 'gender': gender_val, 'address1': summary.streetAddress, 'address2': summary.streetAddress2, 'city': summary.city, 'state': code_dict['state'], 'postal_code': str(summary.zipCode), 'phone': str(summary.phoneNumber), 'account_number': None, 'race': summary.race, 'ethnic_group': None }, 'physician': { 'name': 'None', # must be a string value, not None. 'phone': None, 'npi': None }, 'report_notes': fhir_resource.extension.get(url=DV_ORDER_URL).valueString, 'tests': { 'test': { 'code': '1SAL2', 'name': 'PMI Saliva, FDA Kit', 'comments': None } }, 'comments': 'Salivary Kit Order, direct from participant' } } return order
def test_reconciliation_query(self): self.setup_codes([RACE_QUESTION_CODE], CodeType.QUESTION) self.setup_codes([RACE_AIAN_CODE, RACE_WHITE_CODE], CodeType.ANSWER) self._questionnaire_id = self.create_questionnaire( 'questionnaire3.json') # MySQL and Python sub-second rounding differs, so trim micros from generated times. order_time = clock.CLOCK.now().replace(microsecond=0) old_order_time = order_time - datetime.timedelta(days=10) within_24_hours = order_time + datetime.timedelta(hours=23) old_within_24_hours = old_order_time + datetime.timedelta(hours=23) late_time = order_time + datetime.timedelta(hours=25) old_late_time = old_order_time + datetime.timedelta(hours=25) file_time = order_time + datetime.timedelta( hours=23) + datetime.timedelta(minutes=59) two_days_ago = file_time - datetime.timedelta(days=2) # On time, recent order and samples; shows up in rx p_on_time = self._insert_participant() # Extra samples ordered now aren't considered missing or late. self._insert_order(p_on_time, 'GoodOrder', BIOBANK_TESTS[:4], order_time, finalized_tests=BIOBANK_TESTS[:3], kit_id='kit1', tracking_number='t1', collected_note=u'\u2013foo', processed_note='bar', finalized_note='baz') self._insert_samples(p_on_time, BIOBANK_TESTS[:2], ['GoodSample1', 'GoodSample2'], 'OGoodOrder', within_24_hours, within_24_hours - datetime.timedelta(hours=1)) # On time order and samples from 10 days ago; shows up in rx p_old_on_time = self._insert_participant(race_codes=[RACE_AIAN_CODE]) # Old missing samples from 10 days ago don't show up in missing or late. self._insert_order(p_old_on_time, 'OldGoodOrder', BIOBANK_TESTS[:3], old_order_time, kit_id='kit2') self._insert_samples(p_old_on_time, BIOBANK_TESTS[:2], ['OldGoodSample1', 'OldGoodSample2'], 'OOldGoodOrder', old_within_24_hours, old_within_24_hours - datetime.timedelta(hours=1)) # Late, recent order and samples; shows up in rx and late. (But not missing, as it hasn't been # 36 hours since the order.) p_late_and_missing = self._insert_participant() # Extra missing sample doesn't show up as missing as it hasn't been 24 hours yet. o_late_and_missing = self._insert_order(p_late_and_missing, 'SlowOrder', BIOBANK_TESTS[:3], order_time) self._insert_samples(p_late_and_missing, [BIOBANK_TESTS[0]], ['LateSample'], 'OSlowOrder', late_time, late_time - datetime.timedelta(minutes=59)) # ordered sample not finalized with stored sample should be in missing. p_not_finalized = self._insert_participant() self._insert_order(p_not_finalized, 'UnfinalizedOrder', BIOBANK_TESTS[:2], order_time, finalized_tests=BIOBANK_TESTS[:1]) self._insert_samples(p_not_finalized, [BIOBANK_TESTS[1]], ['missing_order'], 'OUnfinalizedOrder', order_time, order_time - datetime.timedelta(hours=1)) # Late order and samples from 10 days ago; shows up in rx (but not missing, as it was too # long ago. p_old_late_and_missing = self._insert_participant() self._insert_order(p_old_late_and_missing, 'OldSlowOrder', BIOBANK_TESTS[:2], old_order_time) self._insert_samples(p_old_late_and_missing, [BIOBANK_TESTS[0]], ['OldLateSample'], 'OOldSlowOrder', old_late_time, old_late_time - datetime.timedelta(minutes=59)) # Order with missing sample from 2 days ago; shows up in missing. p_two_days_missing = self._insert_participant() # The third test doesn't wind up in missing, as it was never finalized. self._insert_order(p_two_days_missing, 'TwoDaysMissingOrder', BIOBANK_TESTS[:3], two_days_ago, finalized_tests=BIOBANK_TESTS[:2]) # Recent samples with no matching order; shows up in missing. p_extra = self._insert_participant(race_codes=[RACE_WHITE_CODE]) self._insert_samples(p_extra, [BIOBANK_TESTS[-1]], ['NobodyOrderedThisSample'], 'OExtraOrderNotSent', order_time, order_time - datetime.timedelta(minutes=59)) # Old samples with no matching order; Does not show up. p_old_extra = self._insert_participant(race_codes=[RACE_AIAN_CODE]) self._insert_samples(p_old_extra, [BIOBANK_TESTS[-1]], ['OldNobodyOrderedThisSample'], 'OOldExtrOrderNotSent', old_order_time, old_order_time - datetime.timedelta(hours=1)) # Withdrawn participants don't show up in any reports except withdrawal report. p_withdrawn_old_on_time = self._insert_participant( race_codes=[RACE_AIAN_CODE]) # This updates the version of the participant and its HPO ID. self._insert_order(p_withdrawn_old_on_time, 'OldWithdrawnGoodOrder', BIOBANK_TESTS[:2], old_order_time) p_withdrawn_old_on_time = self.participant_dao.get( p_withdrawn_old_on_time.participantId) self._insert_samples( p_withdrawn_old_on_time, BIOBANK_TESTS[:2], ['OldWithdrawnGoodSample1', 'OldWithdrawnGoodSample2'], 'OOldWithdrawnGoodOrder', old_within_24_hours, old_within_24_hours - datetime.timedelta(hours=1)) self._withdraw(p_withdrawn_old_on_time, within_24_hours) p_withdrawn_late_and_missing = self._insert_participant() self._insert_order(p_withdrawn_late_and_missing, 'WithdrawnSlowOrder', BIOBANK_TESTS[:2], order_time) self._insert_samples(p_withdrawn_late_and_missing, [BIOBANK_TESTS[0]], ['WithdrawnLateSample'], 'OWithdrawnSlowOrder', late_time, late_time - datetime.timedelta(minutes=59)) p_withdrawn_late_and_missing = (self.participant_dao.get( p_withdrawn_late_and_missing.participantId)) self._withdraw(p_withdrawn_late_and_missing, within_24_hours) p_withdrawn_old_late_and_missing = self._insert_participant() self._insert_order(p_withdrawn_old_late_and_missing, 'WithdrawnOldSlowOrder', BIOBANK_TESTS[:2], old_order_time) self._insert_samples(p_withdrawn_old_late_and_missing, [BIOBANK_TESTS[0]], ['WithdrawnOldLateSample'], 'OWithdrawnOldSlowOrder', old_late_time, old_late_time - datetime.timedelta(minutes=59)) p_withdrawn_old_late_and_missing = (self.participant_dao.get( p_withdrawn_old_late_and_missing.participantId)) self._withdraw(p_withdrawn_old_late_and_missing, old_late_time) p_withdrawn_extra = self._insert_participant( race_codes=[RACE_WHITE_CODE]) self._insert_samples(p_withdrawn_extra, [BIOBANK_TESTS[-1]], ['WithdrawnNobodyOrderedThisSample'], 'OWithdrawnOldSlowOrder', order_time, order_time - datetime.timedelta(hours=1)) self._withdraw(p_withdrawn_extra, within_24_hours) p_withdrawn_old_extra = self._insert_participant( race_codes=[RACE_AIAN_CODE]) self._insert_samples(p_withdrawn_old_extra, [BIOBANK_TESTS[-1]], ['WithdrawnOldNobodyOrderedThisSample'], 'OwithdrawnOldSlowOrder', old_order_time, old_order_time - datetime.timedelta(hours=1)) self._withdraw(p_withdrawn_old_extra, within_24_hours) p_withdrawn_race_change = self._insert_participant( race_codes=[RACE_AIAN_CODE]) p_withdrawn_race_change_id = to_client_participant_id( p_withdrawn_race_change.participantId) self._submit_race_questionnaire_response(p_withdrawn_race_change_id, [RACE_WHITE_CODE]) self._withdraw(p_withdrawn_race_change, within_24_hours) # for the same participant/test, 3 orders sent and only 2 samples received. Shows up in both # missing (we are missing one sample) and late (the two samples that were received were after # 24 hours.) p_repeated = self._insert_participant() for repetition in xrange(3): self._insert_order( p_repeated, 'RepeatedOrder%d' % repetition, [BIOBANK_TESTS[0]], two_days_ago + datetime.timedelta(hours=repetition)) if repetition != 2: self._insert_samples( p_repeated, [BIOBANK_TESTS[0]], ['RepeatedSample%d' % repetition], 'ORepeatedOrder%d' % repetition, within_24_hours + datetime.timedelta(hours=repetition), within_24_hours + datetime.timedelta(hours=repetition - 1)) received, late, missing, withdrawals = 'rx.csv', 'late.csv', 'missing.csv', 'withdrawals.csv' exporter = InMemorySqlExporter(self) biobank_samples_pipeline._query_and_write_reports( exporter, file_time, received, late, missing, withdrawals) exporter.assertFilesEqual((received, late, missing, withdrawals)) # sent-and-received: 4 on-time, 2 late, none of the missing/extra/repeated ones; # includes orders/samples from more than 7 days ago exporter.assertRowCount(received, 8) exporter.assertColumnNamesEqual(received, _CSV_COLUMN_NAMES) row = exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_on_time.biobankId), 'sent_test': BIOBANK_TESTS[0], 'received_test': BIOBANK_TESTS[0] }) # p_repeated has 2 received and 2 late. exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_repeated.biobankId), 'sent_test': BIOBANK_TESTS[0], 'received_test': BIOBANK_TESTS[0], 'sent_order_id': 'ORepeatedOrder1' }) exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_repeated.biobankId), 'sent_test': BIOBANK_TESTS[0], 'received_test': BIOBANK_TESTS[0], 'sent_order_id': 'ORepeatedOrder0' }) exporter.assertHasRow( late, { 'biobank_id': to_client_biobank_id(p_repeated.biobankId), 'sent_test': BIOBANK_TESTS[0], 'received_test': BIOBANK_TESTS[0], 'sent_order_id': 'ORepeatedOrder0' }) exporter.assertHasRow( late, { 'biobank_id': to_client_biobank_id(p_repeated.biobankId), 'sent_test': BIOBANK_TESTS[0], 'received_test': BIOBANK_TESTS[0], 'sent_order_id': 'ORepeatedOrder1' }) exporter.assertHasRow( missing, { 'biobank_id': to_client_biobank_id(p_not_finalized.biobankId), 'sent_order_id': 'OUnfinalizedOrder' }) # Also check the values of all remaining fields on one row. self.assertEquals(row['source_site_name'], 'Monroeville Urgent Care Center') self.assertEquals(row['source_site_mayolink_client_number'], '7035769') self.assertEquals(row['source_site_hpo'], 'PITT') self.assertEquals(row['source_site_hpo_type'], 'HPO') self.assertEquals(row['finalized_site_name'], 'Phoenix Urgent Care Center') self.assertEquals(row['finalized_site_mayolink_client_number'], '7035770') self.assertEquals(row['finalized_site_hpo'], 'PITT') self.assertEquals(row['finalized_site_hpo_type'], 'HPO') self.assertEquals(row['finalized_username'], '*****@*****.**') self.assertEquals(row['sent_finalized_time'], database_utils.format_datetime(order_time)) self.assertEquals(row['sent_collection_time'], database_utils.format_datetime(order_time)) self.assertEquals(row['sent_processed_time'], database_utils.format_datetime(order_time)) self.assertEquals(row['received_time'], database_utils.format_datetime(within_24_hours)) self.assertEquals( row['Sample Family Create Date'], database_utils.format_datetime(within_24_hours - datetime.timedelta(hours=1))) self.assertEquals(row['sent_count'], '1') self.assertEquals(row['received_count'], '1') self.assertEquals(row['sent_order_id'], 'OGoodOrder') self.assertEquals(row['received_sample_id'], 'GoodSample1') self.assertEquals(row['biospecimen_kit_id'], 'kit1') self.assertEquals(row['fedex_tracking_number'], 't1') self.assertEquals(row['is_native_american'], 'N') self.assertEquals(row['notes_collected'], u'\u2013foo') self.assertEquals(row['notes_processed'], 'bar') self.assertEquals(row['notes_finalized'], 'baz') self.assertEquals(row['sent_order_id'], 'OGoodOrder') # the other sent-and-received rows exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_on_time.biobankId), 'sent_test': BIOBANK_TESTS[1] }) exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id( p_late_and_missing.biobankId), 'sent_test': BIOBANK_TESTS[0] }) exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_old_on_time.biobankId), 'sent_test': BIOBANK_TESTS[0], 'is_native_american': 'Y' }) exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_old_on_time.biobankId), 'sent_test': BIOBANK_TESTS[1], 'is_native_american': 'Y' }) exporter.assertHasRow( received, { 'biobank_id': to_client_biobank_id(p_old_late_and_missing.biobankId), 'sent_test': BIOBANK_TESTS[0], 'is_native_american': 'N' }) # sent-and-received: 2 late; don't include orders/samples from more than 7 days ago exporter.assertRowCount(late, 3) exporter.assertColumnNamesEqual(late, _CSV_COLUMN_NAMES) exporter.assertHasRow( late, { 'biobank_id': to_client_biobank_id( p_late_and_missing.biobankId), 'sent_order_id': 'O%s' % o_late_and_missing.biobankOrderId, 'elapsed_hours': '24', 'is_native_american': 'N' }) exporter.assertHasRow( late, { 'biobank_id': to_client_biobank_id(p_repeated.biobankId), 'elapsed_hours': '46' }) # orders/samples where something went wrong; don't include orders/samples from more than 7 # days ago, or where 24 hours hasn't elapsed yet. exporter.assertRowCount(missing, 5) exporter.assertColumnNamesEqual(missing, _CSV_COLUMN_NAMES) # sample received, nothing ordered exporter.assertHasRow( missing, { 'biobank_id': to_client_biobank_id(p_extra.biobankId), 'sent_order_id': 'OExtraOrderNotSent' }) # order received, no sample exporter.assertHasRow( missing, { 'biobank_id': to_client_biobank_id( p_two_days_missing.biobankId), 'sent_order_id': 'OTwoDaysMissingOrder', 'sent_test': BIOBANK_TESTS[0], 'is_native_american': 'N' }) exporter.assertHasRow( missing, { 'biobank_id': to_client_biobank_id( p_two_days_missing.biobankId), 'sent_order_id': 'OTwoDaysMissingOrder', 'sent_test': BIOBANK_TESTS[1] }) # 3 orders sent, only 2 received multi_sample_row = exporter.assertHasRow( missing, { 'biobank_id': to_client_biobank_id(p_repeated.biobankId), 'sent_count': '1', 'received_count': '0' }) # Also verify the comma-joined fields of the row with multiple orders/samples. self.assertItemsEqual(multi_sample_row['sent_order_id'].split(','), ['ORepeatedOrder2']) # We don't include the old withdrawal. exporter.assertRowCount(withdrawals, 5) exporter.assertHasRow( withdrawals, { 'biobank_id': to_client_biobank_id(p_withdrawn_old_on_time.biobankId), 'withdrawal_time': database_utils.format_datetime(within_24_hours), 'is_native_american': 'Y' }) exporter.assertHasRow( withdrawals, { 'biobank_id': to_client_biobank_id(p_withdrawn_late_and_missing.biobankId), 'withdrawal_time': database_utils.format_datetime(within_24_hours), 'is_native_american': 'N' }) exporter.assertHasRow( withdrawals, { 'biobank_id': to_client_biobank_id( p_withdrawn_extra.biobankId), 'withdrawal_time': database_utils.format_datetime(within_24_hours), 'is_native_american': 'N' }) exporter.assertHasRow( withdrawals, { 'biobank_id': to_client_biobank_id( p_withdrawn_old_extra.biobankId), 'withdrawal_time': database_utils.format_datetime(within_24_hours), 'is_native_american': 'Y' }) exporter.assertHasRow( withdrawals, { 'biobank_id': to_client_biobank_id(p_withdrawn_race_change.biobankId), 'withdrawal_time': database_utils.format_datetime(within_24_hours), 'is_native_american': 'N' })