def map_answers(reader): """Emit (participantId, date|<metric>.<answer>) for each answer. Metric names are taken from the field name in code_constants. Code and string answers are accepted. Incoming rows are expected to be sorted by participant ID, start time, and question code, such that repeated answers for the same question are next to each other. """ last_participant_id = None last_start_time = None race_code_values = [] code_dao = CodeDao() for participant_id, start_time, question_code, answer_code, answer_string in reader: # Multiple race answer values for the participant at a single time # are combined into a single race enum. if race_code_values and (last_participant_id != participant_id or last_start_time != start_time or question_code != RACE_QUESTION_CODE): race_codes = [code_dao.get_code(PPI_SYSTEM, value) for value in race_code_values] race = get_race(race_codes) yield(last_participant_id, make_tuple(last_start_time, make_metric(RACE_METRIC, str(race)))) race_code_values = [] last_participant_id = participant_id last_start_time = start_time if question_code == RACE_QUESTION_CODE: race_code_values.append(answer_code) continue if question_code == EHR_CONSENT_QUESTION_CODE: metric = EHR_CONSENT_ANSWER_METRIC answer_value = answer_code else: question_field = QUESTION_CODE_TO_FIELD[question_code] metric = transform_participant_summary_field(question_field[0]) if question_field[1] == FieldType.CODE: answer_value = answer_code if metric == 'state': state_val = answer_code[len(answer_code) - 2:] census_region = census_regions.get(state_val) or UNSET yield(participant_id, make_tuple(start_time, make_metric(CENSUS_REGION_METRIC, census_region))) elif question_field[1] == FieldType.STRING: answer_value = answer_string else: raise AssertionError("Invalid field type: %s" % question_field[1]) yield(participant_id, make_tuple(start_time, make_metric(metric, answer_value))) # Emit race for the last participant if we saved some values for it. if race_code_values: race_codes = [code_dao.get_code(PPI_SYSTEM, value) for value in race_code_values] race = get_race(race_codes) yield(last_participant_id, make_tuple(last_start_time, make_metric(RACE_METRIC, str(race))))
def _query_and_write_reports(exporter, now, report_type, path_received, path_missing, path_modified, path_withdrawals): """Runs the reconciliation MySQL queries and writes result rows to the given CSV writers. Note that due to syntax differences, the query runs on MySQL only (not SQLite in unit tests). """ report_cover_range = 10 if report_type == 'monthly': report_cover_range = 60 # Gets all sample/order pairs where everything arrived, within the past n days. received_predicate = lambda result: (result[_RECEIVED_TEST_INDEX] and result[_SENT_COUNT_INDEX] <= result[_RECEIVED_COUNT_INDEX] and in_past_n_days(result, now, report_cover_range)) # Gets samples or orders where something has gone missing within the past n days, and if an order # was placed, it was placed at least 36 hours ago. missing_predicate = lambda result: ((result[_SENT_COUNT_INDEX] != result[_RECEIVED_COUNT_INDEX] or (result[_SENT_FINALIZED_INDEX] and not result[_RECEIVED_TEST_INDEX])) and in_past_n_days(result, now, report_cover_range, ordered_before=now - _THIRTY_SIX_HOURS_AGO)) # Gets samples or orders where something has modified within the past n days. modified_predicate = lambda result: (result[_EDITED_CANCELLED_RESTORED_STATUS_FLAG_INDEX] and in_past_n_days(result, now, report_cover_range)) code_dao = CodeDao() race_question_code = code_dao.get_code(PPI_SYSTEM, RACE_QUESTION_CODE) native_american_race_code = code_dao.get_code(PPI_SYSTEM, RACE_AIAN_CODE) # break into three steps to avoid OOM issue report_paths = [path_received, path_missing, path_modified] report_predicates = [received_predicate, missing_predicate, modified_predicate] for report_path, report_predicate in zip(report_paths, report_predicates): with exporter.open_writer(report_path, report_predicate) as report_writer: exporter.run_export_with_writer(report_writer, replace_isodate(_RECONCILIATION_REPORT_SQL), {'race_question_code_id': race_question_code.codeId, 'native_american_race_code_id': native_american_race_code.codeId, 'biobank_id_prefix': get_biobank_id_prefix(), 'pmi_ops_system': _PMI_OPS_SYSTEM, 'kit_id_system': _KIT_ID_SYSTEM, 'tracking_number_system': _TRACKING_NUMBER_SYSTEM, 'n_days_ago': now - datetime.timedelta( days=(report_cover_range + 1))}) # Now generate the withdrawal report, within the past n days. exporter.run_export(path_withdrawals, replace_isodate(_WITHDRAWAL_REPORT_SQL), {'race_question_code_id': race_question_code.codeId, 'native_american_race_code_id': native_american_race_code.codeId, 'n_days_ago': now - datetime.timedelta(days=report_cover_range), 'biobank_id_prefix': get_biobank_id_prefix()})
def test_participant_race_answers(self): with FakeClock(TIME_1): participant_id = self.create_participant() self.send_consent(participant_id) questionnaire_id = self.create_questionnaire('questionnaire_the_basics.json') with open(data_path('questionnaire_the_basics_resp_multiple_race.json')) as f: resource = json.load(f) resource['subject']['reference'] = \ resource['subject']['reference'].format(participant_id=participant_id) resource['questionnaire']['reference'] = \ resource['questionnaire']['reference'].format(questionnaire_id=questionnaire_id) with FakeClock(TIME_2): resource['authored'] = TIME_2.isoformat() self.send_post(_questionnaire_response_url(participant_id), resource) code_dao = CodeDao() code1 = code_dao.get_code('http://terminology.pmi-ops.org/CodeSystem/ppi', 'WhatRaceEthnicity_White') code2 = code_dao.get_code('http://terminology.pmi-ops.org/CodeSystem/ppi', 'WhatRaceEthnicity_Hispanic') participant_race_answers_dao = ParticipantRaceAnswersDao() answers = participant_race_answers_dao.get_all() self.assertEqual(len(answers), 2) for answer in answers: self.assertIn(answer.codeId, [code1.codeId, code2.codeId]) # resubmit the answers, old value should be removed with open(data_path('questionnaire_the_basics_resp_multiple_race_2.json')) as f: resource = json.load(f) resource['subject']['reference'] = \ resource['subject']['reference'].format(participant_id=participant_id) resource['questionnaire']['reference'] = \ resource['questionnaire']['reference'].format(questionnaire_id=questionnaire_id) with FakeClock(TIME_2): resource['authored'] = TIME_2.isoformat() self.send_post(_questionnaire_response_url(participant_id), resource) code_dao = CodeDao() code1 = code_dao.get_code('http://terminology.pmi-ops.org/CodeSystem/ppi', 'WhatRaceEthnicity_NHPI') code2 = code_dao.get_code('http://terminology.pmi-ops.org/CodeSystem/ppi', 'PMI_PreferNotToAnswer') answers = participant_race_answers_dao.get_all() self.assertEqual(len(answers), 2) for answer in answers: self.assertIn(answer.codeId, [code1.codeId, code2.codeId])
def _query_and_write_reports(exporter, now, path_received, path_late, path_missing, path_withdrawals): """Runs the reconciliation MySQL queries and writes result rows to the given CSV writers. Note that due to syntax differences, the query runs on MySQL only (not SQLite in unit tests). """ # Gets all sample/order pairs where everything arrived, regardless of timing. received_predicate = lambda result: (result[_RECEIVED_TEST_INDEX] and result[_SENT_COUNT_INDEX] == result[ _RECEIVED_COUNT_INDEX]) # Gets orders for which the samples arrived, but they arrived late, within the past 7 days. late_predicate = lambda result: (result[_ELAPSED_HOURS_INDEX] and int( result[_ELAPSED_HOURS_INDEX]) >= 24 and in_past_week(result, now)) # Gets samples or orders where something has gone missing within the past 7 days, and if an order # was placed, it was placed at least 36 hours ago. missing_predicate = lambda result: ( (result[_SENT_COUNT_INDEX] != result[_RECEIVED_COUNT_INDEX] or (result[ _SENT_FINALIZED_INDEX] and not result[_RECEIVED_TEST_INDEX])) and in_past_week(result, now, ordered_before=now - _THIRTY_SIX_HOURS_AGO)) code_dao = CodeDao() race_question_code = code_dao.get_code(PPI_SYSTEM, RACE_QUESTION_CODE) native_american_race_code = code_dao.get_code(PPI_SYSTEM, RACE_AIAN_CODE) # Open three files and a database session; run the reconciliation query and pipe the output # to the files, using per-file predicates to filter out results. with exporter.open_writer(path_received, received_predicate) as received_writer, \ exporter.open_writer(path_late, late_predicate) as late_writer, \ exporter.open_writer(path_missing, missing_predicate) as missing_writer, \ database_factory.get_database().session() as session: writer = CompositeSqlExportWriter( [received_writer, late_writer, missing_writer]) exporter.run_export_with_session( writer, session, replace_isodate(_RECONCILIATION_REPORT_SQL), { 'race_question_code_id': race_question_code.codeId, 'native_american_race_code_id': native_american_race_code.codeId, 'biobank_id_prefix': get_biobank_id_prefix(), 'pmi_ops_system': _PMI_OPS_SYSTEM, 'kit_id_system': _KIT_ID_SYSTEM, 'tracking_number_system': _TRACKING_NUMBER_SYSTEM }) # Now generate the withdrawal report. exporter.run_export( path_withdrawals, replace_isodate(_WITHDRAWAL_REPORT_SQL), { 'race_question_code_id': race_question_code.codeId, 'native_american_race_code_id': native_american_race_code.codeId, 'seven_days_ago': now - datetime.timedelta(days=7), 'biobank_id_prefix': get_biobank_id_prefix() })
def _get_validation_result(email, codes_to_answers): result = _ValidationResult() summaries = ParticipantSummaryDao().get_by_email(email) if not summaries: result.add_error('No ParticipantSummary found for %r.' % email) return result if len(summaries) > 1: result.add_error('%d ParticipantSummary values found for %r.' % (len(summaries), email)) return result participant_id = summaries[0].participantId code_dao = CodeDao() qra_dao = QuestionnaireResponseAnswerDao() with qra_dao.session() as session: for code_string, answer_string in codes_to_answers.iteritems(): result.tests_count += 1 question_code = code_dao.get_code(PPI_SYSTEM, code_string) if not question_code: result.add_error( 'Could not find question code %r, skipping answer %r.' % (code_string, answer_string)) continue if question_code.codeType != CodeType.QUESTION: result.add_error( 'Code %r type is %s, not QUESTION; skipping.' % (code_string, question_code.codeType)) continue qras = qra_dao.get_current_answers_for_concepts( session, participant_id, [question_code.codeId]) qra_values = set() for qra in qras: try: qra_values.add( _get_value_for_qra(qra, question_code, code_dao, session)) except ValueError as e: result.add_error(e.message) continue if answer_string: expected_values = set( _boolean_to_lower(v.strip()) for v in answer_string.split('|')) else: expected_values = set() if expected_values != qra_values: result.add_error( '%s: Expected %s, found %s.' % (question_code.value, _format_values(expected_values), _format_values(qra_values))) return result
def _setup_questionnaires(self): """Locates questionnaires and verifies that they have the appropriate questions in them.""" questionnaire_dao = QuestionnaireDao() code_dao = CodeDao() question_code_to_questionnaire_id = {} self._questionnaire_to_questions = collections.defaultdict(list) self._question_code_to_answer_codes = {} # Populate maps of questionnaire ID/version to [(question_code, link ID)] and # question code to answer codes. for concept in _QUESTIONNAIRE_CONCEPTS: code = code_dao.get_code(PPI_SYSTEM, concept) if code is None: raise BadRequest( 'Code missing: %s; import data and clear cache.' % concept) questionnaire = questionnaire_dao.get_latest_questionnaire_with_concept( code.codeId) if questionnaire is None: raise BadRequest( 'Questionnaire for code %s missing; import data.' % concept) questionnaire_id_and_version = (questionnaire.questionnaireId, questionnaire.version) if concept == CONSENT_FOR_STUDY_ENROLLMENT_MODULE: self._consent_questionnaire_id_and_version = questionnaire_id_and_version elif concept == THE_BASICS_PPI_MODULE: self._the_basics_questionnaire_id_and_version = questionnaire_id_and_version questions = self._questionnaire_to_questions[ questionnaire_id_and_version] if questions: # We already handled this questionnaire. continue for question in questionnaire.questions: question_code = code_dao.get(question.codeId) if (question_code.value in _QUESTION_CODES) or (question_code.value in self._answer_specs): question_code_to_questionnaire_id[ question_code.value] = questionnaire.questionnaireId questions.append((question_code.value, question.linkId)) if question_code.value in _QUESTION_CODES: answer_codes = self._get_answer_codes(question_code) all_codes = (answer_codes + _CONSTANT_CODES ) if answer_codes else _CONSTANT_CODES self._question_code_to_answer_codes[ question_code.value] = all_codes # Log warnings for any question codes not found in the questionnaires. for code_value in _QUESTION_CODES + self._answer_specs.keys(): questionnaire_id = question_code_to_questionnaire_id.get( code_value) if not questionnaire_id: logging.warning( 'Question for code %s missing; import questionnaires', code_value)
def _get_answer_sql(num_shards, shard_number): code_dao = CodeDao() code_ids = [] question_codes = list(ANSWER_FIELD_TO_QUESTION_CODE.values()) question_codes.append(RACE_QUESTION_CODE) question_codes.append(EHR_CONSENT_QUESTION_CODE) for code_value in question_codes: code = code_dao.get_code(PPI_SYSTEM, code_value) code_ids.append(str(code.codeId)) params = _get_params(num_shards, shard_number) params['unmapped'] = UNMAPPED return replace_isodate(_ANSWER_QUERY.format((','.join(code_ids)))), params
def _update_participant_summary(self, session, questionnaire_response, code_ids, questions, questionnaire_history, resource_json): """Updates the participant summary based on questions answered and modules completed in the questionnaire response. If no participant summary exists already, only a response to the study enrollment consent questionnaire can be submitted, and it must include first and last name and e-mail address. """ # Block on other threads modifying the participant or participant summary. participant = ParticipantDao().get_for_update( session, questionnaire_response.participantId) if participant is None: raise BadRequest('Participant with ID %d is not found.' % questionnaire_response.participantId) participant_summary = participant.participantSummary code_ids.extend( [concept.codeId for concept in questionnaire_history.concepts]) code_dao = CodeDao() something_changed = False # If no participant summary exists, make sure this is the study enrollment consent. if not participant_summary: consent_code = code_dao.get_code( PPI_SYSTEM, CONSENT_FOR_STUDY_ENROLLMENT_MODULE) if not consent_code: raise BadRequest( 'No study enrollment consent code found; import codebook.') if not consent_code.codeId in code_ids: raise BadRequest( "Can't submit order for participant %s without consent" % questionnaire_response.participantId) raise_if_withdrawn(participant) participant_summary = ParticipantDao.create_summary_for_participant( participant) something_changed = True else: raise_if_withdrawn(participant_summary) # Fetch the codes for all questions and concepts codes = code_dao.get_with_ids(code_ids) code_map = { code.codeId: code for code in codes if code.system == PPI_SYSTEM } question_map = { question.questionnaireQuestionId: question for question in questions } race_code_ids = [] ehr_consent = False dvehr_consent = QuestionnaireStatus.SUBMITTED_NO_CONSENT # Set summary fields for answers that have questions with codes found in QUESTION_CODE_TO_FIELD for answer in questionnaire_response.answers: question = question_map.get(answer.questionId) if question: code = code_map.get(question.codeId) if code: summary_field = QUESTION_CODE_TO_FIELD.get(code.value) if summary_field: if something_changed: self._update_field(participant_summary, summary_field[0], summary_field[1], answer) else: something_changed = self._update_field( participant_summary, summary_field[0], summary_field[1], answer) elif code.value == RACE_QUESTION_CODE: race_code_ids.append(answer.valueCodeId) elif code.value == DVEHR_SHARING_QUESTION_CODE: code = code_dao.get(answer.valueCodeId) if code and code.value == DVEHRSHARING_CONSENT_CODE_YES: dvehr_consent = QuestionnaireStatus.SUBMITTED elif code and code.value == DVEHRSHARING_CONSENT_CODE_NOT_SURE: dvehr_consent = QuestionnaireStatus.SUBMITTED_NOT_SURE elif code.value == EHR_CONSENT_QUESTION_CODE: code = code_dao.get(answer.valueCodeId) if code and code.value == CONSENT_PERMISSION_YES_CODE: ehr_consent = True elif code.value == CABOR_SIGNATURE_QUESTION_CODE: if answer.valueUri or answer.valueString: # TODO: validate the URI? [DA-326] if not participant_summary.consentForCABoR: participant_summary.consentForCABoR = True participant_summary.consentForCABoRTime = questionnaire_response.created something_changed = True # If race was provided in the response in one or more answers, set the new value. if race_code_ids: race_codes = [code_dao.get(code_id) for code_id in race_code_ids] race = get_race(race_codes) if race != participant_summary.race: participant_summary.race = race something_changed = True # Set summary fields to SUBMITTED for questionnaire concepts that are found in # QUESTIONNAIRE_MODULE_CODE_TO_FIELD module_changed = False for concept in questionnaire_history.concepts: code = code_map.get(concept.codeId) if code: summary_field = QUESTIONNAIRE_MODULE_CODE_TO_FIELD.get( code.value) if summary_field: new_status = QuestionnaireStatus.SUBMITTED if code.value == CONSENT_FOR_ELECTRONIC_HEALTH_RECORDS_MODULE and not ehr_consent: new_status = QuestionnaireStatus.SUBMITTED_NO_CONSENT elif code.value == CONSENT_FOR_DVEHR_MODULE: new_status = dvehr_consent elif code.value == CONSENT_FOR_STUDY_ENROLLMENT_MODULE: # set language of consent to participant summary for extension in resource_json.get('extension', []): if extension.get('url') == _LANGUAGE_EXTENSION and \ extension.get('valueCode') in LANGUAGE_OF_CONSENT: if participant_summary.primaryLanguage != extension.get( 'valueCode'): participant_summary.primaryLanguage = extension.get( 'valueCode') something_changed = True break elif extension.get('url') == _LANGUAGE_EXTENSION and \ extension.get('valueCode') not in LANGUAGE_OF_CONSENT: logging.warn( 'consent language %s not recognized.' % extension.get('valueCode')) if getattr(participant_summary, summary_field) != new_status: setattr(participant_summary, summary_field, new_status) setattr(participant_summary, summary_field + 'Time', questionnaire_response.created) something_changed = True module_changed = True if module_changed: participant_summary.numCompletedBaselinePPIModules = \ count_completed_baseline_ppi_modules(participant_summary) participant_summary.numCompletedPPIModules = \ count_completed_ppi_modules(participant_summary) if something_changed: first_last = (participant_summary.firstName, participant_summary.lastName) email_phone = (participant_summary.email, participant_summary.loginPhoneNumber) if not all(first_last): raise BadRequest( 'First name (%s), and last name (%s) required for consenting.' % tuple([ 'present' if part else 'missing' for part in first_last ])) if not any(email_phone): raise BadRequest( 'Email address (%s), or phone number (%s) required for consenting.' % tuple([ 'present' if part else 'missing' for part in email_phone ])) ParticipantSummaryDao().update_enrollment_status( participant_summary) participant_summary.lastModified = clock.CLOCK.now() session.merge(participant_summary) # switch account to test account if the phone number is start with 444 # this is a requirement from PTSC if participant_summary.loginPhoneNumber is not None and \ participant_summary.loginPhoneNumber.startswith(TEST_LOGIN_PHONE_NUMBER_PREFIX): ParticipantDao().switch_to_test_account( session, participant_summary.participantId)
def _update_participant_summary( self, session, questionnaire_response, code_ids, questions, questionnaire_history): """Updates the participant summary based on questions answered and modules completed in the questionnaire response. If no participant summary exists already, only a response to the study enrollment consent questionnaire can be submitted, and it must include first and last name and e-mail address. """ # Block on other threads modifying the participant or participant summary. participant = ParticipantDao().get_for_update(session, questionnaire_response.participantId) if participant is None: raise BadRequest('Participant with ID %d is not found.' % questionnaire_response.participantId) participant_summary = participant.participantSummary code_ids.extend([concept.codeId for concept in questionnaire_history.concepts]) code_dao = CodeDao() something_changed = False # If no participant summary exists, make sure this is the study enrollment consent. if not participant_summary: consent_code = code_dao.get_code(PPI_SYSTEM, CONSENT_FOR_STUDY_ENROLLMENT_MODULE) if not consent_code: raise BadRequest('No study enrollment consent code found; import codebook.') if not consent_code.codeId in code_ids: raise BadRequest("Can't submit order for participant %s without consent" % questionnaire_response.participantId) raise_if_withdrawn(participant) participant_summary = ParticipantDao.create_summary_for_participant(participant) something_changed = True else: raise_if_withdrawn(participant_summary) # Fetch the codes for all questions and concepts codes = code_dao.get_with_ids(code_ids) code_map = {code.codeId: code for code in codes if code.system == PPI_SYSTEM} question_map = {question.questionnaireQuestionId: question for question in questions} race_code_ids = [] ehr_consent = False # Set summary fields for answers that have questions with codes found in QUESTION_CODE_TO_FIELD for answer in questionnaire_response.answers: question = question_map.get(answer.questionId) if question: code = code_map.get(question.codeId) if code: summary_field = QUESTION_CODE_TO_FIELD.get(code.value) if summary_field: something_changed = self._update_field(participant_summary, summary_field[0], summary_field[1], answer) elif code.value == RACE_QUESTION_CODE: race_code_ids.append(answer.valueCodeId) elif code.value == EHR_CONSENT_QUESTION_CODE: code = code_dao.get(answer.valueCodeId) if code and code.value == CONSENT_PERMISSION_YES_CODE: ehr_consent = True elif code.value == CABOR_SIGNATURE_QUESTION_CODE: if answer.valueUri or answer.valueString: # TODO: validate the URI? [DA-326] if not participant_summary.consentForCABoR: participant_summary.consentForCABoR = True participant_summary.consentForCABoRTime = questionnaire_response.created something_changed = True # If race was provided in the response in one or more answers, set the new value. if race_code_ids: race_codes = [code_dao.get(code_id) for code_id in race_code_ids] race = get_race(race_codes) if race != participant_summary.race: participant_summary.race = race something_changed = True # Set summary fields to SUBMITTED for questionnaire concepts that are found in # QUESTIONNAIRE_MODULE_CODE_TO_FIELD module_changed = False for concept in questionnaire_history.concepts: code = code_map.get(concept.codeId) if code: summary_field = QUESTIONNAIRE_MODULE_CODE_TO_FIELD.get(code.value) if summary_field: new_status = QuestionnaireStatus.SUBMITTED if code.value == CONSENT_FOR_ELECTRONIC_HEALTH_RECORDS_MODULE and not ehr_consent: new_status = QuestionnaireStatus.SUBMITTED_NO_CONSENT if getattr(participant_summary, summary_field) != new_status: setattr(participant_summary, summary_field, new_status) setattr(participant_summary, summary_field + 'Time', questionnaire_response.created) something_changed = True module_changed = True if module_changed: participant_summary.numCompletedBaselinePPIModules = \ count_completed_baseline_ppi_modules(participant_summary) participant_summary.numCompletedPPIModules = \ count_completed_ppi_modules(participant_summary) if something_changed: first_last_email = ( participant_summary.firstName, participant_summary.lastName, participant_summary.email) if not all(first_last_email): raise BadRequest( 'First name (%s), last name (%s), and email address (%s) required for consenting.' % tuple(['present' if part else 'missing' for part in first_last_email])) ParticipantSummaryDao().update_enrollment_status(participant_summary) participant_summary.lastModified = clock.CLOCK.now() session.merge(participant_summary)
def test_insert(self): participant_id = self.create_participant() questionnaire_id = self.create_questionnaire('questionnaire1.json') with open(data_path('questionnaire_response3.json')) as fd: resource = json.load(fd) # Sending response with the dummy participant id in the file is an error self.send_post(_questionnaire_response_url('{participant_id}'), resource, expected_status=httplib.NOT_FOUND) # Fixing participant id but not the questionnaire id is also an error resource['subject']['reference'] = \ resource['subject']['reference'].format(participant_id=participant_id) self.send_post(_questionnaire_response_url(participant_id), resource, expected_status=httplib.BAD_REQUEST) # Fix the reference resource['questionnaire']['reference'] = \ resource['questionnaire']['reference'].format(questionnaire_id=questionnaire_id) # Sending the response before the consent is an error. self.send_post(_questionnaire_response_url(participant_id), resource, expected_status=httplib.BAD_REQUEST) # After consent, the post succeeds self.send_consent(participant_id) response = self.send_post(_questionnaire_response_url(participant_id), resource) resource['id'] = response['id'] # The resource gets rewritten to include the version resource['questionnaire'][ 'reference'] = 'Questionnaire/%s/_history/1' % questionnaire_id self.assertJsonResponseMatches(resource, response) # Do a get to fetch the questionnaire get_response = self.send_get( _questionnaire_response_url(participant_id) + "/" + response['id']) self.assertJsonResponseMatches(resource, get_response) code_dao = CodeDao() # Ensure we didn't create codes in the extra system self.assertIsNone(code_dao.get_code(PPI_EXTRA_SYSTEM, 'IgnoreThis')) name_of_child = code_dao.get_code("sys", "nameOfChild") birth_weight = code_dao.get_code("sys", "birthWeight") birth_length = code_dao.get_code("sys", "birthLength") vitamin_k_dose_1 = code_dao.get_code("sys", "vitaminKDose1") vitamin_k_dose_2 = code_dao.get_code("sys", "vitaminKDose2") hep_b_given = code_dao.get_code("sys", "hepBgiven") abnormalities_at_birth = code_dao.get_code("sys", "abnormalitiesAtBirth") answer_dao = QuestionnaireResponseAnswerDao() with answer_dao.session() as session: code_ids = [ code.codeId for code in [ name_of_child, birth_weight, birth_length, vitamin_k_dose_1, vitamin_k_dose_2, hep_b_given, abnormalities_at_birth ] ] current_answers = answer_dao.get_current_answers_for_concepts(session,\ from_client_participant_id(participant_id), code_ids) self.assertEquals(7, len(current_answers)) questionnaire = QuestionnaireDao().get_with_children(questionnaire_id) question_id_to_answer = { answer.questionId: answer for answer in current_answers } code_id_to_answer = { question.codeId: question_id_to_answer.get(question.questionnaireQuestionId) for question in questionnaire.questions } self.assertEquals("Cathy Jones", code_id_to_answer[name_of_child.codeId].valueString) self.assertEquals(3.25, code_id_to_answer[birth_weight.codeId].valueDecimal) self.assertEquals(44.3, code_id_to_answer[birth_length.codeId].valueDecimal) self.assertEquals(44, code_id_to_answer[birth_length.codeId].valueInteger) self.assertEquals(True, code_id_to_answer[hep_b_given.codeId].valueBoolean) self.assertEquals( 0, code_id_to_answer[abnormalities_at_birth.codeId].valueInteger) self.assertEquals(datetime.date(1972, 11, 30), code_id_to_answer[vitamin_k_dose_1.codeId].valueDate) self.assertEquals( datetime.datetime(1972, 11, 30, 12, 34, 42), code_id_to_answer[vitamin_k_dose_2.codeId].valueDateTime)
class ParticipantSummaryDao(UpdatableDao): def __init__(self): super(ParticipantSummaryDao, self).__init__(ParticipantSummary, order_by_ending=_ORDER_BY_ENDING) self.hpo_dao = HPODao() self.code_dao = CodeDao() def get_id(self, obj): return obj.participantId def get_by_email(self, email): with self.session() as session: return session.query(ParticipantSummary).filter( ParticipantSummary.email == email).all() def _validate_update(self, session, obj, existing_obj): """Participant summaries don't have a version value; drop it from validation logic.""" if not existing_obj: raise NotFound('%s with id %s does not exist' % (self.model_type.__name__, id)) def _has_withdrawn_filter(self, query): for field_filter in query.field_filters: if (field_filter.field_name == 'withdrawalStatus' and field_filter.value == WithdrawalStatus.NO_USE): return True if field_filter.field_name == 'withdrawalTime' and field_filter.value is not None: return True return False def _get_non_withdrawn_filter_field(self, query): """Returns the first field referenced in query filters which isn't in WITHDRAWN_PARTICIPANT_FIELDS.""" for field_filter in query.field_filters: if not field_filter.field_name in WITHDRAWN_PARTICIPANT_FIELDS: return field_filter.field_name return None def _initialize_query(self, session, query_def): non_withdrawn_field = self._get_non_withdrawn_filter_field(query_def) if self._has_withdrawn_filter(query_def): if non_withdrawn_field: raise BadRequest( "Can't query on %s for withdrawn participants" % non_withdrawn_field) # When querying for withdrawn participants, ensure that the only fields being filtered on or # ordered by are in WITHDRAWN_PARTICIPANT_FIELDS. return super(ParticipantSummaryDao, self)._initialize_query(session, query_def) else: query = super(ParticipantSummaryDao, self)._initialize_query(session, query_def) if non_withdrawn_field: # When querying on fields that aren't available for withdrawn participants, # ensure that we only return participants # who have not withdrawn or withdrew in the past 48 hours. withdrawn_visible_start = clock.CLOCK.now( ) - WITHDRAWN_PARTICIPANT_VISIBILITY_TIME return query.filter( or_( ParticipantSummary.withdrawalStatus != WithdrawalStatus.NO_USE, ParticipantSummary.withdrawalTime >= withdrawn_visible_start)) else: # When querying on fields that are available for withdrawn participants, return everybody; # withdrawn participants will have all but WITHDRAWN_PARTICIPANT_FIELDS cleared out 48 # hours after withdrawing. return query def _get_order_by_ending(self, query): if self._has_withdrawn_filter(query): return _WITHDRAWN_ORDER_BY_ENDING return self.order_by_ending def _add_order_by(self, query, order_by, field_names, fields): if order_by.field_name in _CODE_FILTER_FIELDS: return super(ParticipantSummaryDao, self)._add_order_by( query, OrderBy(order_by.field_name + 'Id', order_by.ascending), field_names, fields) return super(ParticipantSummaryDao, self)._add_order_by(query, order_by, field_names, fields) def make_query_filter(self, field_name, value): """Handle HPO and code values when parsing filter values.""" if field_name == 'hpoId': hpo = self.hpo_dao.get_by_name(value) if not hpo: raise BadRequest('No HPO found with name %s' % value) return super(ParticipantSummaryDao, self).make_query_filter(field_name, hpo.hpoId) if field_name in _CODE_FILTER_FIELDS: if value == UNSET: return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', None) # Note: we do not at present support querying for UNMAPPED code values. code = self.code_dao.get_code(PPI_SYSTEM, value) if not code: raise BadRequest('No code found: %s' % value) return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', code.codeId) return super(ParticipantSummaryDao, self).make_query_filter(field_name, value) def update_from_biobank_stored_samples(self, participant_id=None): """Rewrites sample-related summary data. Call this after updating BiobankStoredSamples. If participant_id is provided, only that participant will have their summary updated.""" baseline_tests_sql, baseline_tests_params = get_sql_and_params_for_array( config.getSettingList(config.BASELINE_SAMPLE_TEST_CODES), 'baseline') dna_tests_sql, dna_tests_params = get_sql_and_params_for_array( config.getSettingList(config.DNA_SAMPLE_TEST_CODES), 'dna') sample_sql, sample_params = _get_sample_sql_and_params() sql = """ UPDATE participant_summary SET num_baseline_samples_arrived = ( SELECT COUNT(*) FROM biobank_stored_sample WHERE biobank_stored_sample.biobank_id = participant_summary.biobank_id AND biobank_stored_sample.test IN %s ), samples_to_isolate_dna = ( CASE WHEN EXISTS(SELECT * FROM biobank_stored_sample WHERE biobank_stored_sample.biobank_id = participant_summary.biobank_id AND biobank_stored_sample.test IN %s) THEN :received ELSE :unset END ) %s""" % (baseline_tests_sql, dna_tests_sql, sample_sql) params = { 'received': int(SampleStatus.RECEIVED), 'unset': int(SampleStatus.UNSET) } params.update(baseline_tests_params) params.update(dna_tests_params) params.update(sample_params) enrollment_status_params = { 'submitted': int(QuestionnaireStatus.SUBMITTED), 'num_baseline_ppi_modules': self._get_num_baseline_ppi_modules(), 'completed': int(PhysicalMeasurementsStatus.COMPLETED), 'received': int(SampleStatus.RECEIVED), 'full_participant': int(EnrollmentStatus.FULL_PARTICIPANT), 'member': int(EnrollmentStatus.MEMBER), 'interested': int(EnrollmentStatus.INTERESTED) } enrollment_status_sql = _ENROLLMENT_STATUS_SQL # If participant_id is provided, add the participant ID filter to both update statements. if participant_id: sql += _PARTICIPANT_ID_FILTER params['participant_id'] = participant_id enrollment_status_sql += _PARTICIPANT_ID_FILTER enrollment_status_params['participant_id'] = participant_id with self.session() as session: session.execute(sql, params) session.execute(enrollment_status_sql, enrollment_status_params) def _get_num_baseline_ppi_modules(self): return len( config.getSettingList(config.BASELINE_PPI_QUESTIONNAIRE_FIELDS)) def update_enrollment_status(self, summary): """Updates the enrollment status field on the provided participant summary to the correct value based on the other fields on it. Called after a questionnaire response or physical measurements are submitted.""" consent = (summary.consentForStudyEnrollment == QuestionnaireStatus.SUBMITTED and summary.consentForElectronicHealthRecords == QuestionnaireStatus.SUBMITTED) enrollment_status = self.calculate_enrollment_status( consent, summary.numCompletedBaselinePPIModules, summary.physicalMeasurementsStatus, summary.samplesToIsolateDNA) summary.enrollment_status = enrollment_status def calculate_enrollment_status(self, consent_for_study_enrollment_and_ehr, num_completed_baseline_ppi_modules, physical_measurements_status, samples_to_isolate_dna): if consent_for_study_enrollment_and_ehr: if (num_completed_baseline_ppi_modules == self._get_num_baseline_ppi_modules() and physical_measurements_status == PhysicalMeasurementsStatus.COMPLETED and samples_to_isolate_dna == SampleStatus.RECEIVED): return EnrollmentStatus.FULL_PARTICIPANT return EnrollmentStatus.MEMBER return EnrollmentStatus.INTERESTED def to_client_json(self, model): result = model.asdict() # Participants that withdrew more than 48 hours ago should have fields other than # WITHDRAWN_PARTICIPANT_FIELDS cleared. if (model.withdrawalStatus == WithdrawalStatus.NO_USE and model.withdrawalTime < clock.CLOCK.now() - WITHDRAWN_PARTICIPANT_VISIBILITY_TIME): result = {k: result.get(k) for k in WITHDRAWN_PARTICIPANT_FIELDS} result['participantId'] = to_client_participant_id(model.participantId) biobank_id = result.get('biobankId') if biobank_id: result['biobankId'] = to_client_biobank_id(biobank_id) date_of_birth = result.get('dateOfBirth') if date_of_birth: result['ageRange'] = get_bucketed_age(date_of_birth, clock.CLOCK.now()) else: result['ageRange'] = UNSET format_json_hpo(result, self.hpo_dao, 'hpoId') _initialize_field_type_sets() for fieldname in _DATE_FIELDS: format_json_date(result, fieldname) for fieldname in _CODE_FIELDS: format_json_code(result, self.code_dao, fieldname) for fieldname in _ENUM_FIELDS: format_json_enum(result, fieldname) if (model.withdrawalStatus == WithdrawalStatus.NO_USE or model.suspensionStatus == SuspensionStatus.NO_CONTACT): result['recontactMethod'] = 'NO_CONTACT' # Strip None values. result = {k: v for k, v in result.iteritems() if v is not None} return result
class ParticipantSummaryDao(UpdatableDao): def __init__(self): super(ParticipantSummaryDao, self).__init__(ParticipantSummary, order_by_ending=_ORDER_BY_ENDING) self.hpo_dao = HPODao() self.code_dao = CodeDao() self.site_dao = SiteDao() self.organization_dao = OrganizationDao() def get_id(self, obj): return obj.participantId def _validate_update(self, session, obj, existing_obj): # pylint: disable=unused-argument """Participant summaries don't have a version value; drop it from validation logic.""" if not existing_obj: raise NotFound('%s with id %s does not exist' % (self.model_type.__name__, id)) def _has_withdrawn_filter(self, query): for field_filter in query.field_filters: if (field_filter.field_name == 'withdrawalStatus' and field_filter.value == WithdrawalStatus.NO_USE): return True if field_filter.field_name == 'withdrawalTime' and field_filter.value is not None: return True return False def _get_non_withdrawn_filter_field(self, query): """Returns the first field referenced in query filters which isn't in WITHDRAWN_PARTICIPANT_FIELDS.""" for field_filter in query.field_filters: if not field_filter.field_name in WITHDRAWN_PARTICIPANT_FIELDS: return field_filter.field_name return None def _initialize_query(self, session, query_def): non_withdrawn_field = self._get_non_withdrawn_filter_field(query_def) if self._has_withdrawn_filter(query_def): if non_withdrawn_field: raise BadRequest( "Can't query on %s for withdrawn participants" % non_withdrawn_field) # When querying for withdrawn participants, ensure that the only fields being filtered on or # ordered by are in WITHDRAWN_PARTICIPANT_FIELDS. return super(ParticipantSummaryDao, self)._initialize_query(session, query_def) else: query = super(ParticipantSummaryDao, self)._initialize_query(session, query_def) if non_withdrawn_field: # When querying on fields that aren't available for withdrawn participants, # ensure that we only return participants # who have not withdrawn or withdrew in the past 48 hours. withdrawn_visible_start = clock.CLOCK.now( ) - WITHDRAWN_PARTICIPANT_VISIBILITY_TIME return query.filter( or_( ParticipantSummary.withdrawalStatus != WithdrawalStatus.NO_USE, ParticipantSummary.withdrawalTime >= withdrawn_visible_start)) else: # When querying on fields that are available for withdrawn participants, return everybody; # withdrawn participants will have all but WITHDRAWN_PARTICIPANT_FIELDS cleared out 48 # hours after withdrawing. return query def _get_order_by_ending(self, query): if self._has_withdrawn_filter(query): return _WITHDRAWN_ORDER_BY_ENDING return self.order_by_ending def _add_order_by(self, query, order_by, field_names, fields): if order_by.field_name in _CODE_FILTER_FIELDS: return super(ParticipantSummaryDao, self)._add_order_by( query, OrderBy(order_by.field_name + 'Id', order_by.ascending), field_names, fields) return super(ParticipantSummaryDao, self)._add_order_by(query, order_by, field_names, fields) def make_query_filter(self, field_name, value): """Handle HPO and code values when parsing filter values.""" if field_name == 'biobankId': value = from_client_biobank_id(value, log_exception=True) if field_name == 'hpoId' or field_name == 'awardee': hpo = self.hpo_dao.get_by_name(value) if not hpo: raise BadRequest('No HPO found with name %s' % value) if field_name == 'awardee': field_name = 'hpoId' return super(ParticipantSummaryDao, self).make_query_filter(field_name, hpo.hpoId) if field_name == 'organization': organization = self.organization_dao.get_by_external_id(value) if not organization: raise BadRequest('No organization found with name %s' % value) return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', organization.organizationId) if field_name in _SITE_FIELDS: if value == UNSET: return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', None) site = self.site_dao.get_by_google_group(value) if not site: raise BadRequest('No site found with google group %s' % value) return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', site.siteId) if field_name in _CODE_FILTER_FIELDS: if value == UNSET: return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', None) # Note: we do not at present support querying for UNMAPPED code values. code = self.code_dao.get_code(PPI_SYSTEM, value) if not code: raise BadRequest('No code found: %s' % value) return super(ParticipantSummaryDao, self).make_query_filter(field_name + 'Id', code.codeId) return super(ParticipantSummaryDao, self).make_query_filter(field_name, value) def update_from_biobank_stored_samples(self, participant_id=None): """Rewrites sample-related summary data. Call this after updating BiobankStoredSamples. If participant_id is provided, only that participant will have their summary updated.""" now = clock.CLOCK.now() sample_sql, sample_params = _get_sample_sql_and_params(now) baseline_tests_sql, baseline_tests_params = _get_baseline_sql_and_params( ) dna_tests_sql, dna_tests_params = _get_dna_isolates_sql_and_params() sample_status_time_sql = _get_sample_status_time_sql_and_params() sample_status_time_params = {} counts_sql = """ UPDATE participant_summary SET num_baseline_samples_arrived = {baseline_tests_sql}, samples_to_isolate_dna = {dna_tests_sql}, last_modified = :now WHERE num_baseline_samples_arrived != {baseline_tests_sql} OR samples_to_isolate_dna != {dna_tests_sql} """.format(baseline_tests_sql=baseline_tests_sql, dna_tests_sql=dna_tests_sql) counts_params = {'now': now} counts_params.update(baseline_tests_params) counts_params.update(dna_tests_params) enrollment_status_sql = _ENROLLMENT_STATUS_SQL enrollment_status_params = { 'submitted': int(QuestionnaireStatus.SUBMITTED), 'unset': int(QuestionnaireStatus.UNSET), 'num_baseline_ppi_modules': self._get_num_baseline_ppi_modules(), 'completed': int(PhysicalMeasurementsStatus.COMPLETED), 'received': int(SampleStatus.RECEIVED), 'full_participant': int(EnrollmentStatus.FULL_PARTICIPANT), 'member': int(EnrollmentStatus.MEMBER), 'interested': int(EnrollmentStatus.INTERESTED), 'now': now } # If participant_id is provided, add the participant ID filter to all update statements. if participant_id: sample_sql += ' AND participant_id = :participant_id' sample_params['participant_id'] = participant_id counts_sql += ' AND participant_id = :participant_id' counts_params['participant_id'] = participant_id enrollment_status_sql += ' AND participant_id = :participant_id' enrollment_status_params['participant_id'] = participant_id sample_status_time_sql += ' AND a.participant_id = :participant_id' sample_status_time_params['participant_id'] = participant_id sample_sql = replace_null_safe_equals(sample_sql) counts_sql = replace_null_safe_equals(counts_sql) with self.session() as session: session.execute(sample_sql, sample_params) session.execute(counts_sql, counts_params) session.execute(enrollment_status_sql, enrollment_status_params) # TODO: Change this to the optimized sql in _update_dv_stored_samples() session.execute(sample_status_time_sql, sample_status_time_params) def _get_num_baseline_ppi_modules(self): return len( config.getSettingList(config.BASELINE_PPI_QUESTIONNAIRE_FIELDS)) def update_enrollment_status(self, summary): """Updates the enrollment status field on the provided participant summary to the correct value based on the other fields on it. Called after a questionnaire response or physical measurements are submitted.""" consent = (summary.consentForStudyEnrollment == QuestionnaireStatus.SUBMITTED and summary.consentForElectronicHealthRecords == QuestionnaireStatus.SUBMITTED) or \ (summary.consentForStudyEnrollment == QuestionnaireStatus.SUBMITTED and summary.consentForElectronicHealthRecords is None and summary.consentForDvElectronicHealthRecordsSharing == QuestionnaireStatus.SUBMITTED) enrollment_status = self.calculate_enrollment_status( consent, summary.numCompletedBaselinePPIModules, summary.physicalMeasurementsStatus, summary.samplesToIsolateDNA) summary.enrollmentStatusMemberTime = self.calculate_member_time( consent, summary) summary.enrollmentStatusCoreOrderedSampleTime = self.calculate_core_ordered_sample_time( consent, summary) summary.enrollmentStatusCoreStoredSampleTime = self.calculate_core_stored_sample_time( consent, summary) # Update last modified date if status changes if summary.enrollmentStatus != enrollment_status: summary.lastModified = clock.CLOCK.now() summary.enrollmentStatus = enrollment_status def calculate_enrollment_status(self, consent, num_completed_baseline_ppi_modules, physical_measurements_status, samples_to_isolate_dna): if consent: if (num_completed_baseline_ppi_modules == self._get_num_baseline_ppi_modules() and physical_measurements_status == PhysicalMeasurementsStatus.COMPLETED and samples_to_isolate_dna == SampleStatus.RECEIVED): return EnrollmentStatus.FULL_PARTICIPANT return EnrollmentStatus.MEMBER return EnrollmentStatus.INTERESTED def calculate_member_time(self, consent, participant_summary): if consent and participant_summary.enrollmentStatusMemberTime is not None: return participant_summary.enrollmentStatusMemberTime elif consent: if participant_summary.consentForElectronicHealthRecords is None and \ participant_summary.consentForDvElectronicHealthRecordsSharing == \ QuestionnaireStatus.SUBMITTED: return participant_summary.consentForDvElectronicHealthRecordsSharingTime return participant_summary.consentForElectronicHealthRecordsTime else: return None def calculate_core_stored_sample_time(self, consent, participant_summary): if consent and \ participant_summary.numCompletedBaselinePPIModules == \ self._get_num_baseline_ppi_modules() and \ participant_summary.physicalMeasurementsStatus == PhysicalMeasurementsStatus.COMPLETED and \ participant_summary.samplesToIsolateDNA == SampleStatus.RECEIVED: max_core_sample_time = self.calculate_max_core_sample_time( participant_summary, field_name_prefix='sampleStatus') if max_core_sample_time and participant_summary.enrollmentStatusCoreStoredSampleTime: return participant_summary.enrollmentStatusCoreStoredSampleTime else: return max_core_sample_time else: return None def calculate_core_ordered_sample_time(self, consent, participant_summary): if consent and \ participant_summary.numCompletedBaselinePPIModules == \ self._get_num_baseline_ppi_modules() and \ participant_summary.physicalMeasurementsStatus == PhysicalMeasurementsStatus.COMPLETED: max_core_sample_time = self.calculate_max_core_sample_time( participant_summary, field_name_prefix='sampleOrderStatus') if max_core_sample_time and participant_summary.enrollmentStatusCoreOrderedSampleTime: return participant_summary.enrollmentStatusCoreOrderedSampleTime else: return max_core_sample_time else: return None def calculate_max_core_sample_time(self, participant_summary, field_name_prefix='sampleStatus'): keys = [ field_name_prefix + '%sTime' % test for test in config.getSettingList(config.DNA_SAMPLE_TEST_CODES) ] sample_time_list = \ [v for k, v in participant_summary if k in keys and v is not None] sample_time = min(sample_time_list) if sample_time_list else None if sample_time is not None: return max([ sample_time, participant_summary.enrollmentStatusMemberTime, participant_summary.questionnaireOnTheBasicsTime, participant_summary.questionnaireOnLifestyleTime, participant_summary.questionnaireOnOverallHealthTime, participant_summary.physicalMeasurementsFinalizedTime ]) else: return None def calculate_distinct_visits(self, pid, finalized_time, id_, amendment=False): """ Participants may get PM or biobank samples on same day. This should be considered as a single visit in terms of program payment to participant. return Boolean: true if there has not been an order on same date.""" from dao.biobank_order_dao import BiobankOrderDao from dao.physical_measurements_dao import PhysicalMeasurementsDao day_has_order, day_has_measurement = False, False existing_orders = BiobankOrderDao().get_biobank_orders_for_participant( pid) ordered_samples = BiobankOrderDao( ).get_ordered_samples_for_participant(pid) existing_measurements = PhysicalMeasurementsDao( ).get_measuremnets_for_participant(pid) order_id_to_finalized_date = { sample.biobankOrderId: sample.finalized.date() for sample in ordered_samples if sample.finalized } if existing_orders and finalized_time: for order in existing_orders: order_finalized_date = order_id_to_finalized_date.get( order.biobankOrderId) if order_finalized_date == finalized_time.date() and order.biobankOrderId != id_ and \ order.orderStatus != BiobankOrderStatus.CANCELLED: day_has_order = True elif order.biobankOrderId == id_ and order.orderStatus == BiobankOrderStatus.AMENDED: day_has_order = True elif not finalized_time and amendment: day_has_order = True if existing_measurements and finalized_time: for measurement in existing_measurements: if not measurement.finalized: continue if measurement.finalized.date() == finalized_time.date() and measurement.physicalMeasurementsId\ != id_: day_has_measurement = True is_distinct_visit = not (day_has_order or day_has_measurement) return is_distinct_visit def to_client_json(self, model): result = model.asdict() # Participants that withdrew more than 48 hours ago should have fields other than # WITHDRAWN_PARTICIPANT_FIELDS cleared. if (model.withdrawalStatus == WithdrawalStatus.NO_USE and (model.withdrawalTime is None or model.withdrawalTime < clock.CLOCK.now() - WITHDRAWN_PARTICIPANT_VISIBILITY_TIME)): result = {k: result.get(k) for k in WITHDRAWN_PARTICIPANT_FIELDS} elif model.withdrawalStatus != WithdrawalStatus.NO_USE and \ model.suspensionStatus == SuspensionStatus.NO_CONTACT: for i in SUSPENDED_PARTICIPANT_FIELDS: result[i] = UNSET result['participantId'] = to_client_participant_id(model.participantId) biobank_id = result.get('biobankId') if biobank_id: result['biobankId'] = to_client_biobank_id(biobank_id) date_of_birth = result.get('dateOfBirth') if date_of_birth: result['ageRange'] = get_bucketed_age(date_of_birth, clock.CLOCK.now()) else: result['ageRange'] = UNSET if result.get('primaryLanguage') is None: result['primaryLanguage'] = UNSET if 'organizationId' in result: result['organization'] = result['organizationId'] del result['organizationId'] format_json_org(result, self.organization_dao, 'organization') format_json_hpo(result, self.hpo_dao, 'hpoId') result['awardee'] = result['hpoId'] _initialize_field_type_sets() for fieldname in _DATE_FIELDS: format_json_date(result, fieldname) for fieldname in _CODE_FIELDS: format_json_code(result, self.code_dao, fieldname) for fieldname in _ENUM_FIELDS: format_json_enum(result, fieldname) for fieldname in _SITE_FIELDS: format_json_site(result, self.site_dao, fieldname) if (model.withdrawalStatus == WithdrawalStatus.NO_USE or model.suspensionStatus == SuspensionStatus.NO_CONTACT): result['recontactMethod'] = 'NO_CONTACT' # Strip None values. result = {k: v for k, v in result.iteritems() if v is not None} return result def _decode_token(self, query_def, fields): """ If token exists in participant_summary api, decode and use lastModified to add a buffer of 60 seconds. This ensures when a _sync link is used no one is missed. This will return at a minimum, the last participant and any more that have been modified in the previous 60 seconds. Duplicate participants returned should be handled on the client side.""" decoded_vals = super(ParticipantSummaryDao, self)._decode_token(query_def, fields) if query_def.order_by and (query_def.order_by.field_name == 'lastModified' and query_def.always_return_token is True and query_def.backfill_sync is True): decoded_vals[0] = decoded_vals[0] - datetime.timedelta( seconds=config.LAST_MODIFIED_BUFFER_SECONDS) return decoded_vals @staticmethod def update_ehr_status(summary, update_time): summary.ehrStatus = EhrStatus.PRESENT if not summary.ehrReceiptTime: summary.ehrReceiptTime = update_time summary.ehrUpdateTime = update_time return summary