def test_valid_module_answers(self):
p_id = self.create_participant()
email = '*****@*****.**'
code_answers = [('language',
Concept(PPI_SYSTEM,
u'SpokenWrittenLanguage_English')),
('language',
Concept(PPI_SYSTEM,
u'SpokenWrittenLanguage_ChineseChina'))]
self.send_consent(p_id,
email=email,
language='en',
code_values=code_answers)
response = self.send_get(self._answers_url(p_id, u'ConsentPII'))
self.assertIsNotNone(response)
self.assertEqual(len(response), 1)
answers = response[0]
self.assertEqual(answers['ConsentPII_EmailAddress'],
'*****@*****.**')
# verify multiple selection question, both languages should be in the response.
self.assertIn('SpokenWrittenLanguage_English',
answers['Language_SpokenWrittenLanguage'])
self.assertIn('SpokenWrittenLanguage_ChineseChina',
answers['Language_SpokenWrittenLanguage'])
def handle_measurement(measurement_map, m):
"""Populating measurement_map with information extracted from measurement and its
descendants."""
code_concept = Concept(m.codeSystem, m.codeValue)
measurement_data = measurement_map.get(code_concept)
if not measurement_data:
measurement_data = {'bodySites': set(), 'types': set(), 'units': set(),
'codes': set(), 'submeasurements': set(), 'qualifiers': set()}
measurement_map[code_concept] = measurement_data
if m.bodySiteCodeSystem:
measurement_data['bodySites'].add(Concept(m.bodySiteCodeSystem,
m.bodySiteCodeValue))
if m.valueString:
measurement_data['types'].add('string')
if m.valueDecimal:
measurement_data['types'].add('decimal')
min_decimal = measurement_data.get('min')
max_decimal = measurement_data.get('max')
if min_decimal is None or min_decimal > m.valueDecimal:
measurement_data['min'] = m.valueDecimal
if max_decimal is None or max_decimal < m.valueDecimal:
measurement_data['max'] = m.valueDecimal
if m.valueUnit:
measurement_data['units'].add(m.valueUnit)
if m.valueCodeSystem:
measurement_data['codes'].add(Concept(m.valueCodeSystem,
m.valueCodeValue))
if m.valueDateTime:
measurement_data['types'].add('date')
for sm in m.measurements:
measurement_data['submeasurements'].add(Concept(sm.codeSystem, sm.codeValue))
PhysicalMeasurementsDao.handle_measurement(measurement_map, sm)
for q in m.qualifiers:
measurement_data['qualifiers'].add(Concept(q.codeSystem,
q.codeValue))
def assertTaxonomyAssignment(self, taxonomyAssignmentJSON):
parents = list()
children = list()
def extractConcept(JSON):
return JSON['concept']
if 'component' in taxonomyAssignmentJSON['taxonomy_assignment']['parent']:
raise Exception('assertTaxonomyAssignment: Cannot use /= to assert taxonomy of a component')
elif isinstance(taxonomyAssignmentJSON['taxonomy_assignment']['parent'], list):
parents.extend(map(extractConcept, taxonomyAssignmentJSON['taxonomy_assignment']['parent']))
else:
parents.append(taxonomyAssignmentJSON['taxonomy_assignment']['parent']['concept'])
if isinstance(taxonomyAssignmentJSON['taxonomy_assignment']['child'], list):
children.extend(map(extractConcept, taxonomyAssignmentJSON['taxonomy_assignment']['child']))
else:
children.append(taxonomyAssignmentJSON['taxonomy_assignment']['child']['concept'])
self.context.recentlyMentionedPhrases |= set(parents)
self.context.recentlyMentionedPhrases |= set(children)
c = Concept()
for parent in parents:
for child in children:
c.classify(child, parent)
for child in children:
examples = self.context.queryNounPhrases(child)
if not examples:
self.context.newNounPhrase(child)
def _setup_data(self):
self._zip_code_to_state = {}
with open('app_data/zipcodes.txt') as zipcodes:
reader = csv.reader(zipcodes)
for zipcode, state in reader:
self._zip_code_to_state[zipcode] = state
self._first_names = self._read_all_lines('first_names.txt')
self._middle_names = self._read_all_lines('middle_names.txt')
self._last_names = self._read_all_lines('last_names.txt')
self._city_names = self._read_all_lines('city_names.txt')
self._street_names = self._read_all_lines('street_names.txt')
measurement_specs = self._read_json('measurement_specs.json')
if self._use_local_files or app_identity.get_application_id() == 'None':
# Read CSV from a local file when running dev_appserver.
answer_specs = self._read_csv_from_file(_ANSWER_SPECS_FILE)
else:
# Read from GCS when running in Cloud environments.
answer_specs = self._read_csv_from_gcs(_ANSWER_SPECS_BUCKET, _ANSWER_SPECS_FILE)
# Save all the answer specs for questions that don't have special handling already.
self._answer_specs = {answer_spec['question_code']: answer_spec for answer_spec in answer_specs
if answer_spec['question_code'] not in _QUESTION_CODES}
# Serves as the denominator when deciding whether to answer a question.
self._answer_specs_max_participants = max([int(answer_spec['num_participants'])
for answer_spec in answer_specs])
qualifier_concepts = set()
for measurement in measurement_specs:
for qualifier in measurement['qualifiers']:
qualifier_concepts.add(Concept(qualifier['system'], qualifier['code']))
measurement_map = {Concept(measurement['code']['system'], measurement['code']['code']):
measurement for measurement in measurement_specs}
self._measurement_specs = [measurement for measurement in measurement_specs if
Concept(measurement['code']['system'], measurement['code']['code'])
not in qualifier_concepts]
self._qualifier_map = {qualifier_concept: measurement_map[qualifier_concept] for
qualifier_concept in qualifier_concepts}
def __init__(self):
"""
Initialize physical measurements generator.
"""
super(PhysicalMeasurementsGen, self).__init__()
qualifier_concepts = set()
measurement_specs = self._app_data['measurement_specs']
for measurement in measurement_specs:
for qualifier in measurement['qualifiers']:
qualifier_concepts.add(
Concept(qualifier['system'], qualifier['code']))
measurement_map = {
Concept(measurement['code']['system'],
measurement['code']['code']): measurement
for measurement in measurement_specs
}
self._measurement_specs = [
measurement for measurement in measurement_specs
if Concept(measurement['code']['system'], measurement['code']
['code']) not in qualifier_concepts
]
self._qualifier_map = {
qualifier_concept: measurement_map[qualifier_concept]
for qualifier_concept in qualifier_concepts
}
def __init__(self, name):
self.isVerb = False
if re.match('.*\*$', name.strip()):
name = name[:-1]
if not utilities.sanitize(name).istitle():
type = utilities.sanitize(name).split()[-1]
else:
type = None
Concept.__init__(self, name, type)
def __init__(self, name):
self.isVerb = False
quantitative = re.search('^[0-9.]+', name)
if not quantitative:
quantitative = re.search('^-[0-9.]+', name)
if quantitative:
self.quantity = float(quantitative.group(0))
self.isQuantity = True
type = 'quantity'
else:
self.quantity = None
self.isQuantity = False
type = name
Concept.__init__(self, name, type)
def post_demographics_questionnaire(self,
participant_id,
questionnaire_id,
cabor_signature_string=False,
time=TIME_1,
**kwargs):
"""POSTs answers to the demographics questionnaire for the participant"""
answers = {
'code_answers': [],
'string_answers': [],
'date_answers': [('dateOfBirth', kwargs.get('dateOfBirth'))]
}
if cabor_signature_string:
answers['string_answers'].append(
('CABoRSignature', kwargs.get('CABoRSignature')))
else:
answers['uri_answers'] = [('CABoRSignature',
kwargs.get('CABoRSignature'))]
for link_id in self.code_link_ids:
if link_id in kwargs:
concept = Concept(PPI_SYSTEM, kwargs[link_id])
answers['code_answers'].append((link_id, concept))
for link_id in self.string_link_ids:
code = kwargs.get(link_id)
answers['string_answers'].append((link_id, code))
response_data = make_questionnaire_response_json(
participant_id, questionnaire_id, **answers)
with FakeClock(time):
url = 'Participant/%s/QuestionnaireResponse' % participant_id
return self.send_post(url, request_data=response_data)
def _make_measurement_resource(self, measurement, qualifier_set):
resource = {
"code": {
"coding": [{
"code": measurement['code']['code'],
"display": "measurement",
"system": measurement['code']['system']
}],
"text":
"text"
}
}
if 'decimal' in measurement['types']:
# Arguably min and max should vary with units, but in our data there's only one unit
# so we won't bother for now.
min_value = measurement['min']
max_value = measurement['max']
unit = random.choice(measurement['units'])
if min_value == int(min_value) and max_value == int(max_value):
# Assume int min and max means an integer
value = random.randint(min_value, max_value)
else:
# Otherwise assume a floating point number with one digit after the decimal place
value = round(random.uniform(min_value, max_value), 1)
resource['valueQuantity'] = {
"code": unit,
"system": "http://unitsofmeasure.org",
"unit": unit,
"value": value
}
if measurement['valueCodes']:
value_code = random.choice(measurement['valueCodes'])
resource['valueCodeableConcept'] = {
"coding": [{
"system": value_code['system'],
"code": value_code['code'],
"display": "value"
}],
"text":
"value text"
}
if measurement['qualifiers']:
if random.random() <= _PHYSICAL_MEASURMENT_QUALIFIED:
qualifiers = random.sample(
measurement['qualifiers'],
random.randint(1, len(measurement['qualifiers'])))
qualifier_set.update(
Concept(qualifier['system'], qualifier['code'])
for qualifier in qualifiers)
resource['related'] = [{
"type": "qualified-by",
"target": {
"reference": self._make_full_url(qualifier)
}
} for qualifier in qualifiers]
return resource
def submit_questionnaire_response(self, participant_id, questionnaire_id,
race_code, gender_code, state,
date_of_birth):
code_answers = []
date_answers = []
if race_code:
code_answers.append(('race', Concept(PPI_SYSTEM, race_code)))
if gender_code:
code_answers.append(('genderIdentity', Concept(PPI_SYSTEM, gender_code)))
if date_of_birth:
date_answers.append(('dateOfBirth', date_of_birth))
if state:
code_answers.append(('state', Concept(PPI_SYSTEM, state)))
qr = make_questionnaire_response_json(
participant_id,
questionnaire_id,
code_answers=code_answers,
date_answers=date_answers)
self.send_post('Participant/%s/QuestionnaireResponse' % participant_id, qr)
def submit_consent_questionnaire_response(self, participant_id,
questionnaire_id,
ehr_consent_answer):
code_answers = [('ehrConsent', Concept(PPI_SYSTEM,
ehr_consent_answer))]
qr = make_questionnaire_response_json(participant_id,
questionnaire_id,
code_answers=code_answers)
self.send_post('Participant/%s/QuestionnaireResponse' % participant_id,
qr)
def test_module_not_answered(self):
"""
Test an existing module, but no questionnaire response has been submitted.
"""
p_id = self.create_participant()
email = '*****@*****.**'
code_answers = [('language',
Concept(PPI_SYSTEM,
u'SpokenWrittenLanguage_English')),
('language',
Concept(PPI_SYSTEM,
u'SpokenWrittenLanguage_ChineseChina'))]
self.send_consent(p_id,
email=email,
language='en',
code_values=code_answers)
response = self.send_get(self._answers_url(p_id, u'OverallHealth'),
expected_status=httplib.NOT_FOUND)
self.assertEqual(response, None)
def test_invalid_module(self):
"""
Test an invalid module name.
"""
p_id = self.create_participant()
email = '*****@*****.**'
code_answers = [('language',
Concept(PPI_SYSTEM,
u'SpokenWrittenLanguage_English')),
('language',
Concept(PPI_SYSTEM,
u'SpokenWrittenLanguage_ChineseChina'))]
self.send_consent(p_id,
email=email,
language='en',
code_values=code_answers)
response = self.send_get(self._answers_url(p_id, u'InvalidModule'),
expected_status=httplib.BAD_REQUEST)
self.assertEqual(response, None)
"""Tests for validation."""
import unittest
from fhirclient.models.quantity import Quantity
from test.unit_test.unit_test_util import TestBase
from concepts import Concept, UNIT_KG, UNIT_MM_HG
from extraction import Value
from werkzeug.exceptions import BadRequest
from field_validation import FieldValidation, has_units, validate_fields, lessthan, within_range
CONCEPT_A = Concept("http://foo.com/system", "concept_a_code")
CONCEPT_B = Concept("http://foo.com/system", "concept_b_code")
FIELD_A = FieldValidation(CONCEPT_A,
"Concept A", [within_range(0, 300)],
required=True)
FIELD_REQUIRED = FieldValidation(CONCEPT_B,
"Concept B Required", [],
required=True)
FIELD_NOT_REQUIRED = FieldValidation(CONCEPT_B,
"Concept B Not Required", [],
required=False)
FIELD_LESSTHAN = FieldValidation(CONCEPT_B,
"B less than A", [lessthan(FIELD_A)],
required=True)
FIELD_MM_HG = FieldValidation(CONCEPT_A,
"Unit MM_HG", [has_units(UNIT_MM_HG)],
required=True)
FIELD_KG = FieldValidation(CONCEPT_A,
"Unit KG", [has_units(UNIT_KG)],
def _make_measurement_resource(self, measurement, qualifier_set, previous_resource,
measurement_count):
resource = self._make_base_measurement_resource(measurement, False, measurement_count)
if 'decimal' in measurement['types']:
previous_value = None
previous_unit = None
if previous_resource:
previous_value_quantity = previous_resource['valueQuantity']
previous_unit = previous_value_quantity['unit']
previous_value = previous_value_quantity['value']
# Arguably min and max should vary with units, but in our data there's only one unit
# so we won't bother for now.
min_value = measurement['min']
max_value = measurement['max']
if previous_unit:
unit = previous_unit
else:
unit = random.choice(measurement['units'])
if min_value == int(min_value) and max_value == int(max_value):
# Assume int min and max means an integer
if previous_value:
# Find a value that is up to some percent of the possible range higher or lower.
delta = int(_PERCENT_DEVIATION_FOR_REPEATED_MEASUREMENTS * (max_value - min_value))
value = random.randint(previous_value - delta, previous_value + delta)
else:
value = random.randint(min_value, max_value)
else:
# Otherwise assume a floating point number with one digit after the decimal place
if previous_value:
delta = _PERCENT_DEVIATION_FOR_REPEATED_MEASUREMENTS * (max_value - min_value)
value = round(random.uniform(previous_value - delta, previous_value + delta), 1)
else:
value = round(random.uniform(min_value, max_value), 1)
resource['valueQuantity'] = {
"code": unit,
"system": "http://unitsofmeasure.org",
"unit": unit,
"value": value
}
if 'string' in measurement['types']:
resource['valueString'] = ''.join([random.choice(string.lowercase) for _ in xrange(20)])
if measurement['valueCodes']:
value_code = random.choice(measurement['valueCodes'])
resource['valueCodeableConcept'] = {
"coding": [{
"system": value_code['system'],
"code": value_code['code'],
"display": "value"
}],
"text": "value text"
}
if measurement['qualifiers']:
if random.random() <= _PHYSICAL_MEASURMENT_QUALIFIED:
qualifiers = random.sample(measurement['qualifiers'],
random.randint(1, len(measurement['qualifiers'])))
qualifier_set.update(Concept(qualifier['system'], qualifier['code']) for qualifier in
qualifiers)
resource['related'] = [{
"type": "qualified-by",
"target": {
"reference": self._make_full_url(qualifier)
}
} for qualifier in qualifiers]
return resource
def _add_code_answer(code_answers, link_id, code):
if code:
code_answers.append((link_id, Concept(PPI_SYSTEM, code)))
def extract_units(self):
"""Returns the units (as a Concept) that this value is represented in."""
return Concept(system=self.value.system, code=self.value.code)
def extract_concept(node):
"""Extracts a valueCodeableConcept."""
return Concept(system=str(node.system), code=str(node.code))
def __init__(self, name): self.isVerb = True type = str(utilities.sanitize(name).split()[0]) Concept.__init__(self, name, type)
from concepts import migrate, Concept, Relation
migrate()
# from concepts.train.vocab import train
from concepts.train.metadata import train
# c = Concept("foo", "noun")
# c.save()
# c.add_metadata('BLABLABLA', 'foobarbaz')
#
# print(c.metadata())
# print()
#
# b = Concept("bar", "noun")
# b.save()
#
# c.add_relation("SYNONYM", b)
# print(c.relations())
b = Concept("balloon", "noun")
b.save()
train(b)
