def test_participant_race_answers(self):
with FakeClock(TIME_1):
participant_id = self.create_participant()
self.send_consent(participant_id)
questionnaire_id = self.create_questionnaire('questionnaire_the_basics.json')
with open(data_path('questionnaire_the_basics_resp_multiple_race.json')) as f:
resource = json.load(f)
resource['subject']['reference'] = \
resource['subject']['reference'].format(participant_id=participant_id)
resource['questionnaire']['reference'] = \
resource['questionnaire']['reference'].format(questionnaire_id=questionnaire_id)
with FakeClock(TIME_2):
resource['authored'] = TIME_2.isoformat()
self.send_post(_questionnaire_response_url(participant_id), resource)
code_dao = CodeDao()
code1 = code_dao.get_code('http://terminology.pmi-ops.org/CodeSystem/ppi',
'WhatRaceEthnicity_White')
code2 = code_dao.get_code('http://terminology.pmi-ops.org/CodeSystem/ppi',
'WhatRaceEthnicity_Hispanic')
participant_race_answers_dao = ParticipantRaceAnswersDao()
answers = participant_race_answers_dao.get_all()
self.assertEqual(len(answers), 2)
for answer in answers:
self.assertIn(answer.codeId, [code1.codeId, code2.codeId])
# resubmit the answers, old value should be removed
with open(data_path('questionnaire_the_basics_resp_multiple_race_2.json')) as f:
resource = json.load(f)
resource['subject']['reference'] = \
resource['subject']['reference'].format(participant_id=participant_id)
resource['questionnaire']['reference'] = \
resource['questionnaire']['reference'].format(questionnaire_id=questionnaire_id)
with FakeClock(TIME_2):
resource['authored'] = TIME_2.isoformat()
self.send_post(_questionnaire_response_url(participant_id), resource)
code_dao = CodeDao()
code1 = code_dao.get_code('http://terminology.pmi-ops.org/CodeSystem/ppi',
'WhatRaceEthnicity_NHPI')
code2 = code_dao.get_code('http://terminology.pmi-ops.org/CodeSystem/ppi',
'PMI_PreferNotToAnswer')
answers = participant_race_answers_dao.get_all()
self.assertEqual(len(answers), 2)
for answer in answers:
self.assertIn(answer.codeId, [code1.codeId, code2.codeId])
def from_client_json(self, resource_json, participant_id=None, client_id=None):
#pylint: disable=unused-argument
# Parse the questionnaire response, but preserve the original response when persisting
fhir_qr = fhirclient.models.questionnaireresponse.QuestionnaireResponse(resource_json)
patient_id = fhir_qr.subject.reference
if patient_id != 'Patient/P{}'.format(participant_id):
msg = "Questionnaire response subject reference does not match participant_id %r"
raise BadRequest(msg % participant_id)
questionnaire = self._get_questionnaire(fhir_qr.questionnaire, resource_json)
qr = QuestionnaireResponse(questionnaireId=questionnaire.questionnaireId,
questionnaireVersion=questionnaire.version,
participantId=participant_id,
resource=json.dumps(resource_json))
# Extract a code map and answers from the questionnaire response.
code_map, answers = self._extract_codes_and_answers(fhir_qr.group, questionnaire)
if not answers:
logging.error(
'No answers from QuestionnaireResponse JSON. This is harmless but probably an error.')
# Get or insert codes, and retrieve their database IDs.
code_id_map = CodeDao().get_or_add_codes(code_map,
add_codes_if_missing=_add_codes_if_missing(client_id))
# Now add the child answers, using the IDs in code_id_map
self._add_answers(qr, code_id_map, answers)
return qr
def from_client_json(cls,
resource_json,
id_=None,
expected_version=None,
client_id=None):
#pylint: disable=unused-argument
# Parse the questionnaire to make sure it's valid, but preserve the original JSON
# when saving.
fhir_q = fhirclient.models.questionnaire.Questionnaire(resource_json)
if not fhir_q.group:
raise BadRequest('No top-level group found in questionnaire')
q = Questionnaire(resource=json.dumps(resource_json),
questionnaireId=id_,
version=expected_version)
# Assemble a map of (system, value) -> (display, code_type, parent_id) for passing into CodeDao.
# Also assemble a list of (system, code) for concepts and (system, code, linkId) for questions,
# which we'll use later when assembling the child objects.
code_map, concepts, questions = cls._extract_codes(fhir_q.group)
from dao.code_dao import CodeDao
# Get or insert codes, and retrieve their database IDs.
add_codes_if_missing = _add_codes_if_missing()
code_id_map = CodeDao().get_or_add_codes(
code_map, add_codes_if_missing=add_codes_if_missing)
# Now add the child objects, using the IDs in code_id_map
cls._add_concepts(q, code_id_map, concepts)
cls._add_questions(q, code_id_map, questions)
return q
def __init__(self):
super(ParticipantSummaryDao,
self).__init__(ParticipantSummary,
order_by_ending=_ORDER_BY_ENDING)
self.hpo_dao = HPODao()
self.code_dao = CodeDao()
self.site_dao = SiteDao()
self.organization_dao = OrganizationDao()
def setup_codes(values, code_type):
code_dao = CodeDao()
for value in values:
code_dao.insert(
Code(system=PPI_SYSTEM,
value=value,
codeType=code_type,
mapped=True))
def _query_and_write_reports(exporter, now, report_type, path_received, path_missing, path_modified,
path_withdrawals):
"""Runs the reconciliation MySQL queries and writes result rows to the given CSV writers.
Note that due to syntax differences, the query runs on MySQL only (not SQLite in unit tests).
"""
report_cover_range = 10
if report_type == 'monthly':
report_cover_range = 60
# Gets all sample/order pairs where everything arrived, within the past n days.
received_predicate = lambda result: (result[_RECEIVED_TEST_INDEX] and
result[_SENT_COUNT_INDEX] <= result[_RECEIVED_COUNT_INDEX]
and
in_past_n_days(result, now, report_cover_range))
# Gets samples or orders where something has gone missing within the past n days, and if an order
# was placed, it was placed at least 36 hours ago.
missing_predicate = lambda result: ((result[_SENT_COUNT_INDEX] != result[_RECEIVED_COUNT_INDEX] or
(result[_SENT_FINALIZED_INDEX] and
not result[_RECEIVED_TEST_INDEX])) and
in_past_n_days(result, now, report_cover_range,
ordered_before=now - _THIRTY_SIX_HOURS_AGO))
# Gets samples or orders where something has modified within the past n days.
modified_predicate = lambda result: (result[_EDITED_CANCELLED_RESTORED_STATUS_FLAG_INDEX] and
in_past_n_days(result, now, report_cover_range))
code_dao = CodeDao()
race_question_code = code_dao.get_code(PPI_SYSTEM, RACE_QUESTION_CODE)
native_american_race_code = code_dao.get_code(PPI_SYSTEM, RACE_AIAN_CODE)
# break into three steps to avoid OOM issue
report_paths = [path_received, path_missing, path_modified]
report_predicates = [received_predicate, missing_predicate, modified_predicate]
for report_path, report_predicate in zip(report_paths, report_predicates):
with exporter.open_writer(report_path, report_predicate) as report_writer:
exporter.run_export_with_writer(report_writer, replace_isodate(_RECONCILIATION_REPORT_SQL),
{'race_question_code_id': race_question_code.codeId,
'native_american_race_code_id':
native_american_race_code.codeId,
'biobank_id_prefix': get_biobank_id_prefix(),
'pmi_ops_system': _PMI_OPS_SYSTEM,
'kit_id_system': _KIT_ID_SYSTEM,
'tracking_number_system': _TRACKING_NUMBER_SYSTEM,
'n_days_ago': now - datetime.timedelta(
days=(report_cover_range + 1))})
# Now generate the withdrawal report, within the past n days.
exporter.run_export(path_withdrawals, replace_isodate(_WITHDRAWAL_REPORT_SQL),
{'race_question_code_id': race_question_code.codeId,
'native_american_race_code_id': native_american_race_code.codeId,
'n_days_ago': now - datetime.timedelta(days=report_cover_range),
'biobank_id_prefix': get_biobank_id_prefix()})
def map_answers(reader):
"""Emit (participantId, date|<metric>.<answer>) for each answer.
Metric names are taken from the field name in code_constants.
Code and string answers are accepted.
Incoming rows are expected to be sorted by participant ID, start time, and question code,
such that repeated answers for the same question are next to each other.
"""
last_participant_id = None
last_start_time = None
race_code_values = []
code_dao = CodeDao()
for participant_id, start_time, question_code, answer_code, answer_string in reader:
# Multiple race answer values for the participant at a single time
# are combined into a single race enum.
if race_code_values and (last_participant_id != participant_id or
last_start_time != start_time or
question_code != RACE_QUESTION_CODE):
race_codes = [code_dao.get_code(PPI_SYSTEM, value) for value in race_code_values]
race = get_race(race_codes)
yield(last_participant_id, make_tuple(last_start_time,
make_metric(RACE_METRIC, str(race))))
race_code_values = []
last_participant_id = participant_id
last_start_time = start_time
if question_code == RACE_QUESTION_CODE:
race_code_values.append(answer_code)
continue
if question_code == EHR_CONSENT_QUESTION_CODE:
metric = EHR_CONSENT_ANSWER_METRIC
answer_value = answer_code
else:
question_field = QUESTION_CODE_TO_FIELD[question_code]
metric = transform_participant_summary_field(question_field[0])
if question_field[1] == FieldType.CODE:
answer_value = answer_code
if metric == 'state':
state_val = answer_code[len(answer_code) - 2:]
census_region = census_regions.get(state_val) or UNSET
yield(participant_id, make_tuple(start_time, make_metric(CENSUS_REGION_METRIC,
census_region)))
elif question_field[1] == FieldType.STRING:
answer_value = answer_string
else:
raise AssertionError("Invalid field type: %s" % question_field[1])
yield(participant_id, make_tuple(start_time, make_metric(metric, answer_value)))
# Emit race for the last participant if we saved some values for it.
if race_code_values:
race_codes = [code_dao.get_code(PPI_SYSTEM, value) for value in race_code_values]
race = get_race(race_codes)
yield(last_participant_id, make_tuple(last_start_time,
make_metric(RACE_METRIC, str(race))))
def _get_validation_result(email, codes_to_answers):
result = _ValidationResult()
summaries = ParticipantSummaryDao().get_by_email(email)
if not summaries:
result.add_error('No ParticipantSummary found for %r.' % email)
return result
if len(summaries) > 1:
result.add_error('%d ParticipantSummary values found for %r.' %
(len(summaries), email))
return result
participant_id = summaries[0].participantId
code_dao = CodeDao()
qra_dao = QuestionnaireResponseAnswerDao()
with qra_dao.session() as session:
for code_string, answer_string in codes_to_answers.iteritems():
result.tests_count += 1
question_code = code_dao.get_code(PPI_SYSTEM, code_string)
if not question_code:
result.add_error(
'Could not find question code %r, skipping answer %r.' %
(code_string, answer_string))
continue
if question_code.codeType != CodeType.QUESTION:
result.add_error(
'Code %r type is %s, not QUESTION; skipping.' %
(code_string, question_code.codeType))
continue
qras = qra_dao.get_current_answers_for_concepts(
session, participant_id, [question_code.codeId])
qra_values = set()
for qra in qras:
try:
qra_values.add(
_get_value_for_qra(qra, question_code, code_dao,
session))
except ValueError as e:
result.add_error(e.message)
continue
if answer_string:
expected_values = set(
_boolean_to_lower(v.strip())
for v in answer_string.split('|'))
else:
expected_values = set()
if expected_values != qra_values:
result.add_error(
'%s: Expected %s, found %s.' %
(question_code.value, _format_values(expected_values),
_format_values(qra_values)))
return result
def _setup_questionnaires(self):
"""Locates questionnaires and verifies that they have the appropriate questions in them."""
questionnaire_dao = QuestionnaireDao()
code_dao = CodeDao()
question_code_to_questionnaire_id = {}
self._questionnaire_to_questions = collections.defaultdict(list)
self._question_code_to_answer_codes = {}
# Populate maps of questionnaire ID/version to [(question_code, link ID)] and
# question code to answer codes.
for concept in _QUESTIONNAIRE_CONCEPTS:
code = code_dao.get_code(PPI_SYSTEM, concept)
if code is None:
raise BadRequest(
'Code missing: %s; import data and clear cache.' % concept)
questionnaire = questionnaire_dao.get_latest_questionnaire_with_concept(
code.codeId)
if questionnaire is None:
raise BadRequest(
'Questionnaire for code %s missing; import data.' %
concept)
questionnaire_id_and_version = (questionnaire.questionnaireId,
questionnaire.version)
if concept == CONSENT_FOR_STUDY_ENROLLMENT_MODULE:
self._consent_questionnaire_id_and_version = questionnaire_id_and_version
elif concept == THE_BASICS_PPI_MODULE:
self._the_basics_questionnaire_id_and_version = questionnaire_id_and_version
questions = self._questionnaire_to_questions[
questionnaire_id_and_version]
if questions:
# We already handled this questionnaire.
continue
for question in questionnaire.questions:
question_code = code_dao.get(question.codeId)
if (question_code.value
in _QUESTION_CODES) or (question_code.value
in self._answer_specs):
question_code_to_questionnaire_id[
question_code.value] = questionnaire.questionnaireId
questions.append((question_code.value, question.linkId))
if question_code.value in _QUESTION_CODES:
answer_codes = self._get_answer_codes(question_code)
all_codes = (answer_codes + _CONSTANT_CODES
) if answer_codes else _CONSTANT_CODES
self._question_code_to_answer_codes[
question_code.value] = all_codes
# Log warnings for any question codes not found in the questionnaires.
for code_value in _QUESTION_CODES + self._answer_specs.keys():
questionnaire_id = question_code_to_questionnaire_id.get(
code_value)
if not questionnaire_id:
logging.warning(
'Question for code %s missing; import questionnaires',
code_value)
def __init__(self):
self.code_dao = CodeDao()
super(DvOrderDao, self).__init__(BiobankDVOrder)
# used for testing
self.biobank_address = {
'city': "Rochester",
'state': "MN",
'postalCode': "55901",
'line': ["3050 Superior Drive NW"],
'type': 'postal',
'use': 'work'
}
def _query_and_write_reports(exporter, now, path_received, path_late,
path_missing, path_withdrawals):
"""Runs the reconciliation MySQL queries and writes result rows to the given CSV writers.
Note that due to syntax differences, the query runs on MySQL only (not SQLite in unit tests).
"""
# Gets all sample/order pairs where everything arrived, regardless of timing.
received_predicate = lambda result: (result[_RECEIVED_TEST_INDEX] and
result[_SENT_COUNT_INDEX] == result[
_RECEIVED_COUNT_INDEX])
# Gets orders for which the samples arrived, but they arrived late, within the past 7 days.
late_predicate = lambda result: (result[_ELAPSED_HOURS_INDEX] and int(
result[_ELAPSED_HOURS_INDEX]) >= 24 and in_past_week(result, now))
# Gets samples or orders where something has gone missing within the past 7 days, and if an order
# was placed, it was placed at least 36 hours ago.
missing_predicate = lambda result: (
(result[_SENT_COUNT_INDEX] != result[_RECEIVED_COUNT_INDEX] or (result[
_SENT_FINALIZED_INDEX] and not result[_RECEIVED_TEST_INDEX])) and
in_past_week(result, now, ordered_before=now - _THIRTY_SIX_HOURS_AGO))
code_dao = CodeDao()
race_question_code = code_dao.get_code(PPI_SYSTEM, RACE_QUESTION_CODE)
native_american_race_code = code_dao.get_code(PPI_SYSTEM, RACE_AIAN_CODE)
# Open three files and a database session; run the reconciliation query and pipe the output
# to the files, using per-file predicates to filter out results.
with exporter.open_writer(path_received, received_predicate) as received_writer, \
exporter.open_writer(path_late, late_predicate) as late_writer, \
exporter.open_writer(path_missing, missing_predicate) as missing_writer, \
database_factory.get_database().session() as session:
writer = CompositeSqlExportWriter(
[received_writer, late_writer, missing_writer])
exporter.run_export_with_session(
writer, session, replace_isodate(_RECONCILIATION_REPORT_SQL), {
'race_question_code_id': race_question_code.codeId,
'native_american_race_code_id':
native_american_race_code.codeId,
'biobank_id_prefix': get_biobank_id_prefix(),
'pmi_ops_system': _PMI_OPS_SYSTEM,
'kit_id_system': _KIT_ID_SYSTEM,
'tracking_number_system': _TRACKING_NUMBER_SYSTEM
})
# Now generate the withdrawal report.
exporter.run_export(
path_withdrawals, replace_isodate(_WITHDRAWAL_REPORT_SQL), {
'race_question_code_id': race_question_code.codeId,
'native_american_race_code_id': native_american_race_code.codeId,
'seven_days_ago': now - datetime.timedelta(days=7),
'biobank_id_prefix': get_biobank_id_prefix()
})
def setUp(self):
super(QuestionnaireDaoTest, self).setUp(with_data=False)
self.dao = QuestionnaireDao()
self.questionnaire_history_dao = QuestionnaireHistoryDao()
self.questionnaire_concept_dao = QuestionnaireConceptDao()
self.questionnaire_question_dao = QuestionnaireQuestionDao()
self.code_dao = CodeDao()
self.CODE_1 = Code(codeId=1,
system='a',
value='b',
display=u'c',
topic=u'd',
codeType=CodeType.MODULE,
mapped=True)
self.CODE_2 = Code(codeId=2,
system='a',
value='x',
display=u'y',
codeType=CodeType.MODULE,
mapped=False)
self.CODE_3 = Code(codeId=3,
system='a',
value='z',
display=u'y',
codeType=CodeType.MODULE,
mapped=False)
self.CODE_4 = Code(codeId=4,
system='a',
value='c',
codeType=CodeType.QUESTION,
mapped=True,
parentId=1)
self.CODE_5 = Code(codeId=5,
system='a',
value='d',
codeType=CodeType.QUESTION,
mapped=True,
parentId=2)
self.CODE_6 = Code(codeId=6,
system='a',
value='e',
codeType=CodeType.QUESTION,
mapped=True,
parentId=2)
self.CONCEPT_1 = QuestionnaireConcept(codeId=1)
self.CONCEPT_2 = QuestionnaireConcept(codeId=2)
self.QUESTION_1 = QuestionnaireQuestion(linkId='a',
codeId=4,
repeats=False)
self.QUESTION_2 = QuestionnaireQuestion(linkId='d',
codeId=5,
repeats=True)
self.insert_codes()
def _get_answer_sql(num_shards, shard_number):
code_dao = CodeDao()
code_ids = []
question_codes = list(ANSWER_FIELD_TO_QUESTION_CODE.values())
question_codes.append(RACE_QUESTION_CODE)
question_codes.append(EHR_CONSENT_QUESTION_CODE)
for code_value in question_codes:
code = code_dao.get_code(PPI_SYSTEM, code_value)
code_ids.append(str(code.codeId))
params = _get_params(num_shards, shard_number)
params['unmapped'] = UNMAPPED
return replace_isodate(_ANSWER_QUERY.format((','.join(code_ids)))), params
def format_json_code(obj, field_name):
field_without_id = field_name[0:len(field_name) - 2]
value = obj.get(field_name)
if value:
from dao.code_dao import CodeDao
code = CodeDao().get(value)
if code.mapped:
obj[field_without_id] = code.value
else:
obj[field_without_id] = UNMAPPED
del obj[field_name]
else:
obj[field_without_id] = UNSET
def from_client_json(self,
resource_json,
participant_id=None,
client_id=None):
#pylint: disable=unused-argument
# Parse the questionnaire response, but preserve the original response when persisting
fhir_qr = fhirclient.models.questionnaireresponse.QuestionnaireResponse(
resource_json)
patient_id = fhir_qr.subject.reference
if patient_id != 'Patient/P{}'.format(participant_id):
msg = "Questionnaire response subject reference does not match participant_id %r"
raise BadRequest(msg % participant_id)
questionnaire = self._get_questionnaire(fhir_qr.questionnaire,
resource_json)
if questionnaire.status == QuestionnaireDefinitionStatus.INVALID:
raise BadRequest(
"Submitted questionnaire that is marked as invalid: questionnaire ID %s"
% questionnaire.questionnaireId)
authored = None
if fhir_qr.authored and fhir_qr.authored.date:
authored = fhir_qr.authored.date
language = None
if fhir_qr.extension:
for ext in fhir_qr.extension:
if 'iso21090-ST-language' in ext.url:
language = ext.valueCode[:2]
qr = QuestionnaireResponse(
questionnaireId=questionnaire.questionnaireId,
questionnaireVersion=questionnaire.version,
participantId=participant_id,
authored=authored,
language=language,
resource=json.dumps(resource_json))
# Extract a code map and answers from the questionnaire response.
code_map, answers = self._extract_codes_and_answers(
fhir_qr.group, questionnaire)
if not answers:
logging.error(
'No answers from QuestionnaireResponse JSON. This is harmless but probably an error.'
)
# Get or insert codes, and retrieve their database IDs.
code_id_map = CodeDao().get_or_add_codes(
code_map, add_codes_if_missing=_add_codes_if_missing(client_id))
# Now add the child answers, using the IDs in code_id_map
self._add_answers(qr, code_id_map, answers)
return qr
def get(self, p_id=None, module=None):
"""
Return questionnaire answers for a participant
:param p_id: participant id
:param module: questionnaire module name
"""
with CodeDao().session() as session:
# verify participant exists.
participant = session.query(Participant.participantId).filter(
Participant.participantId == p_id).first()
if not participant:
raise BadRequest('invalid participant')
# verify module exists and is a module.
code = session.query(
Code.codeId, Code.value,
Code.codeType).filter(Code.value == module).first()
if not code or code.codeType != CodeType.MODULE:
raise BadRequest('invalid questionnaire module')
# see if the participant has submitted a questionnaire response for the module.
resp = session.query(QuestionnaireResponse.questionnaireId, QuestionnaireResponse.participantId) \
.join(QuestionnaireConcept,
QuestionnaireConcept.questionnaireId == QuestionnaireResponse.questionnaireId) \
.filter(QuestionnaireConcept.codeId == code.codeId).first()
if not resp:
raise NotFound('participant response for module not found.')
skip_null = True if 'skip_null' in request.args and \
request.args['skip_null'].strip('\"').lower() == 'true' else False
filters = request.args['fields'].strip(
'\"') if 'fields' in request.args else None
# If filtering, always include these fields in the response.
if filters:
filters += ',questionnaire_id,questionnaire_response_id,created,code_id' + \
',version,authored,language,participant_id,module'
try:
results = QuestionnaireResponseDao().call_proc(
'sp_get_questionnaire_answers',
args=[module, p_id],
filters=filters,
skip_null=skip_null)
except ValueError as e:
raise BadRequest(e.message)
except Exception:
raise
# raise BadRequest('zzz invalid request')
return results
def get(self, id_=None):
if id_:
return super(QuestionnaireApi, self).get(id_)
else:
concept = request.args.get('concept')
if not concept:
raise BadRequest('Either questionnaire ID or concept must be specified in request.')
concept_code = CodeDao().get_code(PPI_SYSTEM, concept)
if not concept_code:
raise BadRequest('Code not found: %s' % concept)
questionnaire = self.dao.get_latest_questionnaire_with_concept(concept_code.codeId)
if not questionnaire:
raise NotFound('Could not find questionnaire with concept: %s' % concept)
return self._make_response(questionnaire)
def setUp(self):
super(DvOrderDaoTestBase, self).setUp(use_mysql=True)
self.dao = DvOrderDao()
self.code_dao = CodeDao()
self.participant_dao = ParticipantDao()
self.summary_dao = ParticipantSummaryDao()
self.participant = Participant(participantId=123456789, biobankId=7)
self.participant_dao.insert(self.participant)
self.summary = self.participant_summary(self.participant)
self.summary_dao.insert(self.summary)
def test_insert(self):
questionnaire_files = (
'questionnaire1.json',
'questionnaire2.json',
'questionnaire_demographics.json',
)
for json_file in questionnaire_files:
with open(data_path(json_file)) as f:
questionnaire = json.load(f)
response = self.send_post('Questionnaire', questionnaire)
questionnaire_id = response['id']
del response['id']
questionnaire['version'] = '1'
self.assertJsonResponseMatches(questionnaire, response)
response = self.send_get('Questionnaire/%s' % questionnaire_id)
del response['id']
self.assertJsonResponseMatches(questionnaire, response)
# Ensure we didn't create codes in the extra system
self.assertIsNone(CodeDao().get_code(PPI_EXTRA_SYSTEM, 'IgnoreThis'))
def setUp(self, use_mysql=True, with_data=True):
super(DvOrderApiTestBase, self).setUp(use_mysql=use_mysql,
with_data=with_data)
self.dv_order_dao = DvOrderDao()
self.hpo_dao = HPODao()
self.participant_dao = ParticipantDao()
self.summary_dao = ParticipantSummaryDao()
self.code_dao = CodeDao()
self.hpo = self.hpo_dao.get_by_name('PITT')
self.participant = Participant(hpoId=self.hpo.hpoId,
participantId=123456789,
biobankId=7)
self.participant_dao.insert(self.participant)
self.summary = self.participant_summary(self.participant)
self.summary_dao.insert(self.summary)
mayolinkapi_patcher = mock.patch(
'dao.dv_order_dao.MayoLinkApi',
**{'return_value.post.return_value': self.mayolink_response})
mayolinkapi_patcher.start()
self.addCleanup(mayolinkapi_patcher.stop)
def _sanity_check_codebook():
if not CodeDao().get_code(PPI_SYSTEM, EMAIL_QUESTION_CODE):
raise RuntimeError('No question code found for %s; import codebook.' %
EMAIL_QUESTION_CODE)
def setUp(self):
super(CodeDaoTest, self).setUp()
self.code_book_dao = CodeBookDao()
self.code_dao = CodeDao()
self.code_history_dao = CodeHistoryDao()
def setUp(self):
super(QuestionnaireResponseDaoTest, self).setUp()
self.code_dao = CodeDao()
self.participant_dao = ParticipantDao()
self.questionnaire_dao = QuestionnaireDao()
self.questionnaire_response_dao = QuestionnaireResponseDao()
self.questionnaire_response_answer_dao = QuestionnaireResponseAnswerDao(
)
self.participant_summary_dao = ParticipantSummaryDao()
self.CODE_1 = Code(codeId=1,
system=PPI_SYSTEM,
value=GENDER_IDENTITY_QUESTION_CODE,
display=u'c',
topic=u'd',
codeType=CodeType.QUESTION,
mapped=True)
self.CODE_2 = Code(codeId=2,
system='a',
value='x',
display=u'y',
codeType=CodeType.QUESTION,
mapped=False)
self.CODE_3 = Code(codeId=3,
system='a',
value='c',
codeType=CodeType.ANSWER,
mapped=True,
parentId=1)
self.CODE_4 = Code(codeId=4,
system='a',
value='d',
codeType=CodeType.ANSWER,
mapped=True,
parentId=2)
self.CODE_5 = Code(codeId=5,
system='a',
value='e',
codeType=CodeType.ANSWER,
mapped=False,
parentId=1)
self.CODE_6 = Code(codeId=6,
system='a',
value='f',
codeType=CodeType.ANSWER,
mapped=True,
parentId=1)
self.MODULE_CODE_7 = Code(codeId=7,
system=PPI_SYSTEM,
value=THE_BASICS_PPI_MODULE,
codeType=CodeType.MODULE,
mapped=True)
self.CONCEPT_1 = QuestionnaireConcept(codeId=7)
self.CODE_1_QUESTION_1 = QuestionnaireQuestion(linkId='a',
codeId=1,
repeats=False)
self.CODE_2_QUESTION = QuestionnaireQuestion(linkId='d',
codeId=2,
repeats=True)
# Same code as question 1
self.CODE_1_QUESTION_2 = QuestionnaireQuestion(linkId='x',
codeId=1,
repeats=False)
self.skip_code = Code(codeId=8,
system=PPI_SYSTEM,
value=PMI_SKIP_CODE,
mapped=True,
codeType=CodeType.ANSWER)
config.override_setting(config.CONSENT_PDF_BUCKET, [_FAKE_BUCKET])
def _get_answer_values(question_code_value):
q_code = CodeDao().get_code(PPI_SYSTEM, question_code_value)
if not q_code:
return []
return [answer_code.value for answer_code in q_code.children
if answer_code.codeType == CodeType.ANSWER]
def test_insert(self):
participant_id = self.create_participant()
questionnaire_id = self.create_questionnaire('questionnaire1.json')
with open(data_path('questionnaire_response3.json')) as fd:
resource = json.load(fd)
# Sending response with the dummy participant id in the file is an error
self.send_post(_questionnaire_response_url('{participant_id}'),
resource,
expected_status=httplib.NOT_FOUND)
# Fixing participant id but not the questionnaire id is also an error
resource['subject']['reference'] = \
resource['subject']['reference'].format(participant_id=participant_id)
self.send_post(_questionnaire_response_url(participant_id),
resource,
expected_status=httplib.BAD_REQUEST)
# Fix the reference
resource['questionnaire']['reference'] = \
resource['questionnaire']['reference'].format(questionnaire_id=questionnaire_id)
# Sending the response before the consent is an error.
self.send_post(_questionnaire_response_url(participant_id),
resource,
expected_status=httplib.BAD_REQUEST)
# After consent, the post succeeds
self.send_consent(participant_id)
response = self.send_post(_questionnaire_response_url(participant_id),
resource)
resource['id'] = response['id']
# The resource gets rewritten to include the version
resource['questionnaire'][
'reference'] = 'Questionnaire/%s/_history/1' % questionnaire_id
self.assertJsonResponseMatches(resource, response)
# Do a get to fetch the questionnaire
get_response = self.send_get(
_questionnaire_response_url(participant_id) + "/" + response['id'])
self.assertJsonResponseMatches(resource, get_response)
code_dao = CodeDao()
# Ensure we didn't create codes in the extra system
self.assertIsNone(code_dao.get_code(PPI_EXTRA_SYSTEM, 'IgnoreThis'))
name_of_child = code_dao.get_code("sys", "nameOfChild")
birth_weight = code_dao.get_code("sys", "birthWeight")
birth_length = code_dao.get_code("sys", "birthLength")
vitamin_k_dose_1 = code_dao.get_code("sys", "vitaminKDose1")
vitamin_k_dose_2 = code_dao.get_code("sys", "vitaminKDose2")
hep_b_given = code_dao.get_code("sys", "hepBgiven")
abnormalities_at_birth = code_dao.get_code("sys",
"abnormalitiesAtBirth")
answer_dao = QuestionnaireResponseAnswerDao()
with answer_dao.session() as session:
code_ids = [
code.codeId for code in [
name_of_child, birth_weight, birth_length,
vitamin_k_dose_1, vitamin_k_dose_2, hep_b_given,
abnormalities_at_birth
]
]
current_answers = answer_dao.get_current_answers_for_concepts(session,\
from_client_participant_id(participant_id), code_ids)
self.assertEquals(7, len(current_answers))
questionnaire = QuestionnaireDao().get_with_children(questionnaire_id)
question_id_to_answer = {
answer.questionId: answer
for answer in current_answers
}
code_id_to_answer = {
question.codeId:
question_id_to_answer.get(question.questionnaireQuestionId)
for question in questionnaire.questions
}
self.assertEquals("Cathy Jones",
code_id_to_answer[name_of_child.codeId].valueString)
self.assertEquals(3.25,
code_id_to_answer[birth_weight.codeId].valueDecimal)
self.assertEquals(44.3,
code_id_to_answer[birth_length.codeId].valueDecimal)
self.assertEquals(44,
code_id_to_answer[birth_length.codeId].valueInteger)
self.assertEquals(True,
code_id_to_answer[hep_b_given.codeId].valueBoolean)
self.assertEquals(
0, code_id_to_answer[abnormalities_at_birth.codeId].valueInteger)
self.assertEquals(datetime.date(1972, 11, 30),
code_id_to_answer[vitamin_k_dose_1.codeId].valueDate)
self.assertEquals(
datetime.datetime(1972, 11, 30, 12, 34, 42),
code_id_to_answer[vitamin_k_dose_2.codeId].valueDateTime)
def __init__(self):
self.code_dao = CodeDao()
super(PatientStatusDao, self).__init__(PatientStatus)
def setUp(self):
super(SchemaTest, self).setUp(use_mysql=True)
self.code_dao = CodeDao()
def _update_participant_summary(
self, session, questionnaire_response, code_ids, questions, questionnaire_history):
"""Updates the participant summary based on questions answered and modules completed
in the questionnaire response.
If no participant summary exists already, only a response to the study enrollment consent
questionnaire can be submitted, and it must include first and last name and e-mail address.
"""
# Block on other threads modifying the participant or participant summary.
participant = ParticipantDao().get_for_update(session, questionnaire_response.participantId)
if participant is None:
raise BadRequest('Participant with ID %d is not found.' %
questionnaire_response.participantId)
participant_summary = participant.participantSummary
code_ids.extend([concept.codeId for concept in questionnaire_history.concepts])
code_dao = CodeDao()
something_changed = False
# If no participant summary exists, make sure this is the study enrollment consent.
if not participant_summary:
consent_code = code_dao.get_code(PPI_SYSTEM, CONSENT_FOR_STUDY_ENROLLMENT_MODULE)
if not consent_code:
raise BadRequest('No study enrollment consent code found; import codebook.')
if not consent_code.codeId in code_ids:
raise BadRequest("Can't submit order for participant %s without consent" %
questionnaire_response.participantId)
raise_if_withdrawn(participant)
participant_summary = ParticipantDao.create_summary_for_participant(participant)
something_changed = True
else:
raise_if_withdrawn(participant_summary)
# Fetch the codes for all questions and concepts
codes = code_dao.get_with_ids(code_ids)
code_map = {code.codeId: code for code in codes if code.system == PPI_SYSTEM}
question_map = {question.questionnaireQuestionId: question for question in questions}
race_code_ids = []
ehr_consent = False
# Set summary fields for answers that have questions with codes found in QUESTION_CODE_TO_FIELD
for answer in questionnaire_response.answers:
question = question_map.get(answer.questionId)
if question:
code = code_map.get(question.codeId)
if code:
summary_field = QUESTION_CODE_TO_FIELD.get(code.value)
if summary_field:
something_changed = self._update_field(participant_summary, summary_field[0],
summary_field[1], answer)
elif code.value == RACE_QUESTION_CODE:
race_code_ids.append(answer.valueCodeId)
elif code.value == EHR_CONSENT_QUESTION_CODE:
code = code_dao.get(answer.valueCodeId)
if code and code.value == CONSENT_PERMISSION_YES_CODE:
ehr_consent = True
elif code.value == CABOR_SIGNATURE_QUESTION_CODE:
if answer.valueUri or answer.valueString:
# TODO: validate the URI? [DA-326]
if not participant_summary.consentForCABoR:
participant_summary.consentForCABoR = True
participant_summary.consentForCABoRTime = questionnaire_response.created
something_changed = True
# If race was provided in the response in one or more answers, set the new value.
if race_code_ids:
race_codes = [code_dao.get(code_id) for code_id in race_code_ids]
race = get_race(race_codes)
if race != participant_summary.race:
participant_summary.race = race
something_changed = True
# Set summary fields to SUBMITTED for questionnaire concepts that are found in
# QUESTIONNAIRE_MODULE_CODE_TO_FIELD
module_changed = False
for concept in questionnaire_history.concepts:
code = code_map.get(concept.codeId)
if code:
summary_field = QUESTIONNAIRE_MODULE_CODE_TO_FIELD.get(code.value)
if summary_field:
new_status = QuestionnaireStatus.SUBMITTED
if code.value == CONSENT_FOR_ELECTRONIC_HEALTH_RECORDS_MODULE and not ehr_consent:
new_status = QuestionnaireStatus.SUBMITTED_NO_CONSENT
if getattr(participant_summary, summary_field) != new_status:
setattr(participant_summary, summary_field, new_status)
setattr(participant_summary, summary_field + 'Time', questionnaire_response.created)
something_changed = True
module_changed = True
if module_changed:
participant_summary.numCompletedBaselinePPIModules = \
count_completed_baseline_ppi_modules(participant_summary)
participant_summary.numCompletedPPIModules = \
count_completed_ppi_modules(participant_summary)
if something_changed:
first_last_email = (
participant_summary.firstName, participant_summary.lastName, participant_summary.email)
if not all(first_last_email):
raise BadRequest(
'First name (%s), last name (%s), and email address (%s) required for consenting.'
% tuple(['present' if part else 'missing' for part in first_last_email]))
ParticipantSummaryDao().update_enrollment_status(participant_summary)
participant_summary.lastModified = clock.CLOCK.now()
session.merge(participant_summary)
def _update_participant_summary(self, session, questionnaire_response,
code_ids, questions, questionnaire_history,
resource_json):
"""Updates the participant summary based on questions answered and modules completed
in the questionnaire response.
If no participant summary exists already, only a response to the study enrollment consent
questionnaire can be submitted, and it must include first and last name and e-mail address.
"""
# Block on other threads modifying the participant or participant summary.
participant = ParticipantDao().get_for_update(
session, questionnaire_response.participantId)
if participant is None:
raise BadRequest('Participant with ID %d is not found.' %
questionnaire_response.participantId)
participant_summary = participant.participantSummary
code_ids.extend(
[concept.codeId for concept in questionnaire_history.concepts])
code_dao = CodeDao()
something_changed = False
# If no participant summary exists, make sure this is the study enrollment consent.
if not participant_summary:
consent_code = code_dao.get_code(
PPI_SYSTEM, CONSENT_FOR_STUDY_ENROLLMENT_MODULE)
if not consent_code:
raise BadRequest(
'No study enrollment consent code found; import codebook.')
if not consent_code.codeId in code_ids:
raise BadRequest(
"Can't submit order for participant %s without consent" %
questionnaire_response.participantId)
raise_if_withdrawn(participant)
participant_summary = ParticipantDao.create_summary_for_participant(
participant)
something_changed = True
else:
raise_if_withdrawn(participant_summary)
# Fetch the codes for all questions and concepts
codes = code_dao.get_with_ids(code_ids)
code_map = {
code.codeId: code
for code in codes if code.system == PPI_SYSTEM
}
question_map = {
question.questionnaireQuestionId: question
for question in questions
}
race_code_ids = []
ehr_consent = False
dvehr_consent = QuestionnaireStatus.SUBMITTED_NO_CONSENT
# Set summary fields for answers that have questions with codes found in QUESTION_CODE_TO_FIELD
for answer in questionnaire_response.answers:
question = question_map.get(answer.questionId)
if question:
code = code_map.get(question.codeId)
if code:
summary_field = QUESTION_CODE_TO_FIELD.get(code.value)
if summary_field:
if something_changed:
self._update_field(participant_summary,
summary_field[0],
summary_field[1], answer)
else:
something_changed = self._update_field(
participant_summary, summary_field[0],
summary_field[1], answer)
elif code.value == RACE_QUESTION_CODE:
race_code_ids.append(answer.valueCodeId)
elif code.value == DVEHR_SHARING_QUESTION_CODE:
code = code_dao.get(answer.valueCodeId)
if code and code.value == DVEHRSHARING_CONSENT_CODE_YES:
dvehr_consent = QuestionnaireStatus.SUBMITTED
elif code and code.value == DVEHRSHARING_CONSENT_CODE_NOT_SURE:
dvehr_consent = QuestionnaireStatus.SUBMITTED_NOT_SURE
elif code.value == EHR_CONSENT_QUESTION_CODE:
code = code_dao.get(answer.valueCodeId)
if code and code.value == CONSENT_PERMISSION_YES_CODE:
ehr_consent = True
elif code.value == CABOR_SIGNATURE_QUESTION_CODE:
if answer.valueUri or answer.valueString:
# TODO: validate the URI? [DA-326]
if not participant_summary.consentForCABoR:
participant_summary.consentForCABoR = True
participant_summary.consentForCABoRTime = questionnaire_response.created
something_changed = True
# If race was provided in the response in one or more answers, set the new value.
if race_code_ids:
race_codes = [code_dao.get(code_id) for code_id in race_code_ids]
race = get_race(race_codes)
if race != participant_summary.race:
participant_summary.race = race
something_changed = True
# Set summary fields to SUBMITTED for questionnaire concepts that are found in
# QUESTIONNAIRE_MODULE_CODE_TO_FIELD
module_changed = False
for concept in questionnaire_history.concepts:
code = code_map.get(concept.codeId)
if code:
summary_field = QUESTIONNAIRE_MODULE_CODE_TO_FIELD.get(
code.value)
if summary_field:
new_status = QuestionnaireStatus.SUBMITTED
if code.value == CONSENT_FOR_ELECTRONIC_HEALTH_RECORDS_MODULE and not ehr_consent:
new_status = QuestionnaireStatus.SUBMITTED_NO_CONSENT
elif code.value == CONSENT_FOR_DVEHR_MODULE:
new_status = dvehr_consent
elif code.value == CONSENT_FOR_STUDY_ENROLLMENT_MODULE:
# set language of consent to participant summary
for extension in resource_json.get('extension', []):
if extension.get('url') == _LANGUAGE_EXTENSION and \
extension.get('valueCode') in LANGUAGE_OF_CONSENT:
if participant_summary.primaryLanguage != extension.get(
'valueCode'):
participant_summary.primaryLanguage = extension.get(
'valueCode')
something_changed = True
break
elif extension.get('url') == _LANGUAGE_EXTENSION and \
extension.get('valueCode') not in LANGUAGE_OF_CONSENT:
logging.warn(
'consent language %s not recognized.' %
extension.get('valueCode'))
if getattr(participant_summary,
summary_field) != new_status:
setattr(participant_summary, summary_field, new_status)
setattr(participant_summary, summary_field + 'Time',
questionnaire_response.created)
something_changed = True
module_changed = True
if module_changed:
participant_summary.numCompletedBaselinePPIModules = \
count_completed_baseline_ppi_modules(participant_summary)
participant_summary.numCompletedPPIModules = \
count_completed_ppi_modules(participant_summary)
if something_changed:
first_last = (participant_summary.firstName,
participant_summary.lastName)
email_phone = (participant_summary.email,
participant_summary.loginPhoneNumber)
if not all(first_last):
raise BadRequest(
'First name (%s), and last name (%s) required for consenting.'
% tuple([
'present' if part else 'missing' for part in first_last
]))
if not any(email_phone):
raise BadRequest(
'Email address (%s), or phone number (%s) required for consenting.'
% tuple([
'present' if part else 'missing'
for part in email_phone
]))
ParticipantSummaryDao().update_enrollment_status(
participant_summary)
participant_summary.lastModified = clock.CLOCK.now()
session.merge(participant_summary)
# switch account to test account if the phone number is start with 444
# this is a requirement from PTSC
if participant_summary.loginPhoneNumber is not None and \
participant_summary.loginPhoneNumber.startswith(TEST_LOGIN_PHONE_NUMBER_PREFIX):
ParticipantDao().switch_to_test_account(
session, participant_summary.participantId)
def _setup_consent_codes(self):
"""
Proactively setup Codebook entries in the Code table so parent/child relationship is
created. Matches 'test_data/study_consent.json`.
"""
def create_code(topic, name, code_type, parent):
code = Code(system=PPI_SYSTEM,
topic=topic,
value=name,
display=name,
codeType=code_type,
mapped=True,
shortValue=name,
created=datetime.datetime.utcnow())
if parent:
parent.children.append(code)
return code
with CodeDao().session() as session:
module = create_code(u'Module Name', u'ConsentPII',
CodeType.MODULE, None)
session.add(module)
topic = create_code(u'Language', u'ConsentPII_Language',
CodeType.TOPIC, module)
session.add(topic)
qn = create_code(u'Language', u'Language_SpokenWrittenLanguage',
CodeType.QUESTION, topic)
session.add(qn)
session.add(
create_code(u'Language', u'SpokenWrittenLanguage_English',
CodeType.ANSWER, qn))
session.add(
create_code(u'Language', u'SpokenWrittenLanguage_ChineseChina',
CodeType.ANSWER, qn))
session.add(
create_code(u'Language', u'SpokenWrittenLanguage_French',
CodeType.ANSWER, qn))
topic = create_code(u'Address', u'ConsentPII_PIIAddress',
CodeType.TOPIC, module)
session.add(topic)
session.add(
create_code(u'Address', u'PIIAddress_StreetAddress',
CodeType.QUESTION, topic))
session.add(
create_code(u'Address', u'PIIAddress_StreetAddress2',
CodeType.QUESTION, topic))
topic = create_code(u'Name', u'ConsentPII_PIIName', CodeType.TOPIC,
module)
session.add(topic)
session.add(
create_code(u'Name', u'PIIName_First', CodeType.QUESTION,
topic))
session.add(
create_code(u'Name', u'PIIName_Middle', CodeType.QUESTION,
topic))
session.add(
create_code(u'Name', u'PIIName_Last', CodeType.QUESTION,
topic))
session.add(
create_code(u'Email Address', u'ConsentPII_EmailAddress',
CodeType.QUESTION, module))
topic = create_code(u'Extra Consent Items',
u'ConsentPII_ExtraConsent', CodeType.TOPIC,
module)
session.add(
create_code(u'Extra Consent Items',
u'ExtraConsent_CABoRSignature', CodeType.QUESTION,
topic))
module = create_code(u'Module Name', u'OverallHealth',
CodeType.MODULE, None)
session.add(module)
session.commit()