def test_insert_without_codebook_or_parent(self): code = Code(system="a", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True) with FakeClock(TIME): self.code_dao.insert(code) expected_code = Code(codeId=1, system="a", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True, created=TIME) self.assertEquals(expected_code.asdict(), self.code_dao.get(1).asdict()) expected_code_history = CodeHistory(codeHistoryId=1, codeId=1, system="a", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True, created=TIME) self.assertEquals(expected_code_history.asdict(), self.code_history_dao.get(1).asdict())
def setup_codes(values, code_type): code_dao = CodeDao() for value in values: code_dao.insert( Code(system=PPI_SYSTEM, value=value, codeType=code_type, mapped=True))
def _import_concept(self, session, existing_codes, concept, system, code_book_id, parent_id): """Recursively imports a concept and its descendants as codes. Existing codes will be updated; codes that weren't there before will be inserted. Codes that are in the database but not in the codebook will be left untouched. """ property_dict = { p['code']: p['valueCode'] for p in concept['property'] } topic = property_dict['concept-topic'] value = concept['code'] short_value = property_dict.get('short-code') or value[:50] display = concept['display'] code_type = _CODE_TYPE_MAP.get(property_dict['concept-type']) if code_type is None: logging.warning( "Unrecognized concept type: %s, value: %s; ignoring." % (property_dict['concept-type'], value)) return 0 code = Code(system=system, codeBookId=code_book_id, value=value, shortValue=short_value, display=display, topic=topic, codeType=code_type, mapped=True, parentId=parent_id) existing_code = existing_codes.get((system, value)) if existing_code: code.codeId = existing_code.codeId self.code_dao._do_update(session, code, existing_code) else: self.code_dao.insert_with_session(session, code) child_concepts = concept.get('concept') code_count = 1 if child_concepts: session.flush() for child_concept in child_concepts: code_count += self._import_concept(session, existing_codes, child_concept, system, code_book_id, code.codeId) return code_count
def create_code(topic, name, code_type, parent): code = Code(system=PPI_SYSTEM, topic=topic, value=name, display=name, codeType=code_type, mapped=True, shortValue=name, created=datetime.datetime.utcnow()) if parent: parent.children.append(code) return code
def test_update_codes_no_codebook_id(self): code_book_1 = CodeBook(name="pmi", version="v1", system="c") with FakeClock(TIME): self.code_book_dao.insert(code_book_1) code_1 = Code(codeBookId=1, system="a", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True) with FakeClock(TIME_2): self.code_dao.insert(code_1) new_code_1 = Code(codeId=1, system="x", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True) with self.assertRaises(BadRequest): self.code_dao.update(new_code_1)
def get_or_add_codes(self, code_map, add_codes_if_missing=True): """Accepts a map of (system, value) -> (display, code_type, parent_id) for codes found in a questionnaire or questionnaire response. Returns a map of (system, value) -> codeId for new and existing codes. Adds new unmapped codes for anything that is missing. """ # First get whatever is already in the cache. result_map = {} for system, value in code_map.keys(): code = self.get_code(system, value) if code: result_map[(system, value)] = code.codeId if len(result_map) == len(code_map): return result_map with self.session() as session: for system, value in code_map.keys(): existing_code = result_map.get((system, value)) if not existing_code: # Check to see if it's in the database. (Normally it won't be.) existing_code = self._get_code_with_session( session, system, value) if existing_code: result_map[(system, value)] = code.codeId continue if not add_codes_if_missing: raise BadRequest( "Couldn't find code: system = %s, value = %s" % (system, value)) # If it's not in the database, add it. display, code_type, parent_id = code_map[(system, value)] code = Code(system=system, value=value, display=display, codeType=code_type, mapped=False, parentId=parent_id) # Log the traceback so that stackdriver error reporting reports on it. logging.error( "Adding unmapped code: system = %s, value = %s: %s", code.system, code.value, traceback.format_exc()) self.insert_with_session(session, code) session.flush() result_map[(system, value)] = code.codeId return result_map
def test_biobank_address_received(self, patched_code_id): patched_code_id.return_value = 1 code = Code(system="a", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True) self.code_dao.insert(code) self.send_post('SupplyRequest', request_data=self.get_payload( 'dv_order_api_post_supply_request.json'), expected_status=httplib.CREATED) response = self.send_post( 'SupplyDelivery', request_data=self.get_payload( 'dv_order_api_post_supply_delivery.json'), expected_status=httplib.CREATED) request = self.get_payload('dv_order_api_put_supply_delivery.json') biobank_address = self.dv_order_dao.biobank_address request['contained'][0]['address'] = biobank_address location_id = response.location.rsplit('/', 1)[-1] self.send_put('SupplyDelivery/{}'.format(location_id), request_data=request) order = self.get_orders() self.assertEquals(order[0].biobankCity, 'Rochester') self.assertEquals(order[0].city, 'Fairfax') self.assertEquals(order[0].biobankStreetAddress1, '3050 Superior Drive NW') self.assertEquals(order[0].streetAddress1, '4114 Legato Rd') self.assertEquals(order[0].streetAddress2, 'test line 2') self.assertEquals(order[0].biobankStateId, 1) self.assertEquals(order[0].stateId, 1) self.assertEquals(order[0].biobankZipCode, '55901') self.assertEquals(order[0].zipCode, '22033') self.assertTrue(response.location.endswith('/SupplyDelivery/999999')) self.assertEqual(1, len(order)) for i in order: self.assertEqual(i.id, long(1)) self.assertEqual(i.order_id, long(999999))
def setUp(self): super(QuestionnaireDaoTest, self).setUp(with_data=False) self.dao = QuestionnaireDao() self.questionnaire_history_dao = QuestionnaireHistoryDao() self.questionnaire_concept_dao = QuestionnaireConceptDao() self.questionnaire_question_dao = QuestionnaireQuestionDao() self.code_dao = CodeDao() self.CODE_1 = Code(codeId=1, system='a', value='b', display=u'c', topic=u'd', codeType=CodeType.MODULE, mapped=True) self.CODE_2 = Code(codeId=2, system='a', value='x', display=u'y', codeType=CodeType.MODULE, mapped=False) self.CODE_3 = Code(codeId=3, system='a', value='z', display=u'y', codeType=CodeType.MODULE, mapped=False) self.CODE_4 = Code(codeId=4, system='a', value='c', codeType=CodeType.QUESTION, mapped=True, parentId=1) self.CODE_5 = Code(codeId=5, system='a', value='d', codeType=CodeType.QUESTION, mapped=True, parentId=2) self.CODE_6 = Code(codeId=6, system='a', value='e', codeType=CodeType.QUESTION, mapped=True, parentId=2) self.CONCEPT_1 = QuestionnaireConcept(codeId=1) self.CONCEPT_2 = QuestionnaireConcept(codeId=2) self.QUESTION_1 = QuestionnaireQuestion(linkId='a', codeId=4, repeats=False) self.QUESTION_2 = QuestionnaireQuestion(linkId='d', codeId=5, repeats=True) self.insert_codes()
def vs_to_codes(): from util import natus_config import pandas as pd from model.code import Code from model.valueset import ValueSet from db.mongo import connector import json config = natus_config.NATUSConfig('ncqa') excel_filename = config.read_value('valueset', 'excel_filename') sheet_name = config.read_value('valueset', 'excel_sheet_name') print('Reading ' + excel_filename) df = pd.read_excel( excel_filename, sheet_name=sheet_name ) print('Processing ' + sheet_name) value_set_info = None codes_processed = set() db_con = connector.Connector() for index, row in df.iterrows(): vs_name = row['Value Set Name'] code = row['Code'] code_system = row['Code System'] code_system_oid = row['Code System OID'] code_info = Code(code, code_system, code_system_oid) if vs_name not in codes_processed: if len(codes_processed) > 0: db_con.insert(vars(value_set_info), 'vs_to_codes') codes_processed.add(vs_name) vs_oid = row['Value Set OID'] value_set_info = ValueSet(vs_name, vs_oid, [vars(code_info)]) else: value_set_info.codes.append(vars(code_info))
def consent_code(): return Code(system=PPI_SYSTEM, value=CONSENT_FOR_STUDY_ENROLLMENT_MODULE, mapped=True, codeType=CodeType.MODULE)
def test_schema(self): session = self.database.make_session() hpo = HPO(hpoId=1, name='UNSET', displayName='No organization set', organizationType=OrganizationType.UNSET) code_book = CodeBook(codeBookId=1, created=datetime.datetime.now(), latest=True, name="pmi", system="http://foo/bar", version="v0.1.1") session.add(hpo) session.add(code_book) session.commit() organization = Organization(organizationId=1, externalId='org', displayName='Organization', hpoId=1) session.add(organization) session.commit() site = Site(siteId=1, siteName='site', googleGroup='*****@*****.**', mayolinkClientNumber=12345, organizationId=1) code1 = Code(codeId=1, codeBookId=1, system="a", value="b", shortValue="q", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True, created=datetime.datetime.now()) codeHistory1 = CodeHistory(codeId=1, codeBookId=1, system="a", value="b", shortValue="q", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True, created=datetime.datetime.now()) session.add(site) session.add(code1) session.add(codeHistory1) session.commit() code2 = Code(codeId=2, codeBookId=1, parentId=1, system="a", value="c", display=u"X", topic=u"d", codeType=CodeType.QUESTION, mapped=True, created=datetime.datetime.now()) codeHistory2 = CodeHistory(codeId=2, codeBookId=1, parentId=1, system="a", value="c", display=u"X", topic=u"d", codeType=CodeType.QUESTION, mapped=True, created=datetime.datetime.now()) session.add(code2) session.add(codeHistory2) session.commit() code3 = Code(codeId=3, codeBookId=1, parentId=2, system="a", value="d", display=u"Y", topic=u"d", codeType=CodeType.ANSWER, mapped=False, created=datetime.datetime.now()) codeHistory3 = CodeHistory(codeId=3, codeBookId=1, parentId=2, system="a", value="d", display=u"Y", topic=u"d", codeType=CodeType.ANSWER, mapped=False, created=datetime.datetime.now()) session.add(code3) session.add(codeHistory3) session.commit() session.commit() p = self._participant_with_defaults( participantId=1, version=1, biobankId=2, clientId='*****@*****.**', hpoId=hpo.hpoId, signUpTime=datetime.datetime.now(), lastModified=datetime.datetime.now()) ps = self._participant_summary_with_defaults( participantId=1, biobankId=2, lastModified=datetime.datetime.now(), hpoId=hpo.hpoId, firstName=self.fake.first_name(), middleName=self.fake.first_name(), lastName=self.fake.last_name(), email=self.fake.email(), zipCode='78751', dateOfBirth=datetime.date.today(), genderIdentityId=1, consentForStudyEnrollment=QuestionnaireStatus.SUBMITTED, consentForStudyEnrollmentTime=datetime.datetime.now(), numBaselineSamplesArrived=2) p.participantSummary = ps session.add(p) ph = self._participant_history_with_defaults( participantId=1, biobankId=2, clientId='*****@*****.**', hpoId=hpo.hpoId, signUpTime=datetime.datetime.now(), lastModified=datetime.datetime.now()) session.add(ph) session.commit() session.add( BiobankStoredSample(biobankStoredSampleId='WEB1234542', biobankId=p.biobankId, biobankOrderIdentifier='KIT', test='1UR10', confirmed=datetime.datetime.utcnow())) session.add( BiobankStoredSample( biobankStoredSampleId='WEB99999', # Sample ID must be unique. biobankId=p. biobankId, # Participant ID and test may be duplicated. biobankOrderIdentifier='KIT', test='1UR10', confirmed=datetime.datetime.utcnow())) pm = PhysicalMeasurements(physicalMeasurementsId=1, participantId=1, created=datetime.datetime.now(), resource='blah', final=False, logPosition=LogPosition()) pm2 = PhysicalMeasurements(physicalMeasurementsId=2, participantId=1, created=datetime.datetime.now(), resource='blah', final=True, amendedMeasurementsId=1, logPosition=LogPosition()) session.add(pm) session.add(pm2) session.commit() q1 = Measurement(measurementId=3, physicalMeasurementsId=pm.physicalMeasurementsId, codeSystem='codeSystem', codeValue='codeValue', measurementTime=datetime.datetime.now(), valueCodeSystem='valueCodeSystem', valueCodeValue='value3') session.add(q1) session.commit() m1 = Measurement(measurementId=1, physicalMeasurementsId=pm.physicalMeasurementsId, codeSystem='codeSystem', codeValue='codeValue', measurementTime=datetime.datetime.now(), bodySiteCodeSystem='bodySiteCodeSystem', bodySiteCodeValue='bodySiteCodeValue', valueString='a', valueDecimal=1.2, valueUnit='cm', valueCodeSystem='valueCodeSystem', valueCodeValue='value', valueDateTime=datetime.datetime.now(), qualifierId=q1.measurementId) session.add(m1) session.commit() m2 = Measurement(measurementId=2, physicalMeasurementsId=pm.physicalMeasurementsId, codeSystem='codeSystem', codeValue='codeValue', measurementTime=datetime.datetime.now(), valueCodeSystem='valueCodeSystem', valueCodeValue='value2', parentId=m1.measurementId, qualifierId=q1.measurementId) session.add(m2) session.commit() q = Questionnaire(questionnaireId=1, version=1, created=datetime.datetime.now(), lastModified=datetime.datetime.now(), resource='what?') qh = QuestionnaireHistory(questionnaireId=1, version=1, created=datetime.datetime.now(), lastModified=datetime.datetime.now(), resource='what?') qh.questions.append( QuestionnaireQuestion(questionnaireQuestionId=1, questionnaireId=1, questionnaireVersion=1, linkId="1.2.3", codeId=2, repeats=True)) qh.concepts.append( QuestionnaireConcept(questionnaireConceptId=1, questionnaireId=1, questionnaireVersion=1, codeId=1)) session.add(q) session.add(qh) session.commit() qr = QuestionnaireResponse(questionnaireResponseId=1, questionnaireId=1, questionnaireVersion=1, participantId=1, created=datetime.datetime.now(), resource='blah') qr.answers.append( QuestionnaireResponseAnswer(questionnaireResponseAnswerId=1, questionnaireResponseId=1, questionId=1, endTime=datetime.datetime.now(), valueSystem='a', valueCodeId=3, valueDecimal=123, valueString=self.fake.first_name(), valueDate=datetime.date.today())) session.add(qr) session.commit() mv = MetricsVersion(metricsVersionId=1, inProgress=False, complete=True, date=datetime.datetime.utcnow(), dataVersion=1) session.add(mv) session.commit() mb = MetricsBucket(metricsVersionId=1, date=datetime.date.today(), hpoId='PITT', metrics='blah') session.add(mb) session.commit()
def login_phone_number_code(): return Code(system=PPI_SYSTEM, value=LOGIN_PHONE_NUMBER_QUESTION_CODE, mapped=True, codeType=CodeType.QUESTION)
def test_insert_with_codebook_and_parent(self): code_book_1 = CodeBook(name="pmi", version="v1", system="a") with FakeClock(TIME): self.code_book_dao.insert(code_book_1) expected_code_book = CodeBook(codeBookId=1, latest=True, created=TIME, name="pmi", version="v1", system="a") self.assertEquals(expected_code_book.asdict(), self.code_book_dao.get(1).asdict()) code_1 = Code(codeBookId=1, system="a", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True) with FakeClock(TIME_2): self.code_dao.insert(code_1) expected_code = Code(codeBookId=1, codeId=1, system="a", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True, created=TIME_2) self.assertEquals(expected_code.asdict(), self.code_dao.get(1).asdict()) expected_code_history = CodeHistory(codeBookId=1, codeHistoryId=1, codeId=1, system="a", value=u"b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True, created=TIME_2) self.assertEquals(expected_code_history.asdict(), self.code_history_dao.get(1).asdict()) code_2 = Code(codeBookId=1, system="x", value="y", display=u"z", topic=u"q", codeType=CodeType.QUESTION, mapped=False, parentId=1) with FakeClock(TIME_3): self.code_dao.insert(code_2) expected_code_2 = Code(codeBookId=1, codeId=2, system="x", value="y", display=u"z", topic=u"q", codeType=CodeType.QUESTION, mapped=False, created=TIME_3, parentId=1) self.assertEquals(expected_code_2.asdict(), self.code_dao.get(2).asdict())
def test_import_codebook(self): answer_1 = _make_concept(u"t1", "Answer", "c1", u"d1") answer_2 = _make_concept(u"t2", "Answer", "c2", u"d2") answer_3 = _make_concept(u"t2", "Answer", "c3", u"d3") question_1 = _make_concept(u"t1", "Question", "q1", u"d4", [answer_1]) question_2 = _make_concept(u"t2", "Question", "q2", u"d5", [answer_2, answer_3]) topic_1 = _make_concept(u"t1", "Topic", "t1", u"d6", [question_1]) module_1 = _make_concept(u"mt1", "Module Name", "m1", u"d7", [topic_1]) module_2 = _make_concept(u"mt2", "Module Name", "m2", u"d8", [question_2]) system = 'http://blah/foo' codebook = { 'name': 'pmi', 'version': 'v1', 'url': system, 'concept': [module_1, module_2] } with FakeClock(TIME): self.code_book_dao.import_codebook(codebook) expectedCodeBook = CodeBook(codeBookId=1, latest=True, created=TIME, name="pmi", version="v1", system=system) self.assertEquals(expectedCodeBook.asdict(), self.code_book_dao.get(1).asdict()) expectedModule1 = Code(codeBookId=1, codeId=1, system=system, value="m1", shortValue="m1", display=u"d7", topic=u"mt1", codeType=CodeType.MODULE, mapped=True, created=TIME) self.assertEquals(expectedModule1.asdict(), self.code_dao.get(1).asdict()) expectedModuleHistory1 = CodeHistory(codeHistoryId=1, codeBookId=1, codeId=1, system=system, value="m1", shortValue="m1", display=u"d7", topic=u"mt1", codeType=CodeType.MODULE, mapped=True, created=TIME) self.assertEquals(expectedModuleHistory1.asdict(), self.code_history_dao.get(1).asdict()) expectedTopic1 = Code(codeBookId=1, codeId=2, system=system, value="t1", shortValue="t1", display=u"d6", topic=u"t1", codeType=CodeType.TOPIC, mapped=True, created=TIME, parentId=1) self.assertEquals(expectedTopic1.asdict(), self.code_dao.get(2).asdict()) expectedQuestion1 = Code(codeBookId=1, codeId=3, system=system, value="q1", shortValue="q1", display=u"d4", topic=u"t1", codeType=CodeType.QUESTION, mapped=True, created=TIME, parentId=2) self.assertEquals(expectedQuestion1.asdict(), self.code_dao.get(3).asdict()) expectedAnswer1 = Code(codeBookId=1, codeId=4, system=system, value="c1", shortValue="c1", display=u"d1", topic=u"t1", codeType=CodeType.ANSWER, mapped=True, created=TIME, parentId=3) self.assertEquals(expectedAnswer1.asdict(), self.code_dao.get(4).asdict())
def test_update_codes_new_codebook_id(self): code_book_1 = CodeBook(name="pmi", version="v1", system="a") with FakeClock(TIME): self.code_book_dao.insert(code_book_1) code_1 = Code(codeBookId=1, system="a", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True) with FakeClock(TIME_2): self.code_dao.insert(code_1) code_book_2 = CodeBook(name="pmi", version="v2", system="a") with FakeClock(TIME_3): self.code_book_dao.insert(code_book_2) new_code_1 = Code(codeBookId=2, codeId=1, system="x", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True) with FakeClock(TIME_4): self.code_dao.update(new_code_1) expected_code = Code(codeBookId=2, codeId=1, system="x", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True, created=TIME_2) self.assertEquals(expected_code.asdict(), self.code_dao.get(1).asdict()) expected_code_history = CodeHistory(codeBookId=1, codeHistoryId=1, codeId=1, system="a", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True, created=TIME_2) self.assertEquals(expected_code_history.asdict(), self.code_history_dao.get(1).asdict()) expected_code_history_2 = CodeHistory(codeHistoryId=2, codeBookId=2, codeId=1, system="x", value="b", display=u"c", topic=u"d", codeType=CodeType.MODULE, mapped=True, created=TIME_2) self.assertEquals(expected_code_history_2.asdict(), self.code_history_dao.get(2).asdict())
def last_name_code(): return Code(system=PPI_SYSTEM, value=LAST_NAME_QUESTION_CODE, mapped=True, codeType=CodeType.QUESTION)
def email_code(): return Code(system=PPI_SYSTEM, value=EMAIL_QUESTION_CODE, mapped=True, codeType=CodeType.QUESTION)
def setUp(self): super(QuestionnaireResponseDaoTest, self).setUp() self.code_dao = CodeDao() self.participant_dao = ParticipantDao() self.questionnaire_dao = QuestionnaireDao() self.questionnaire_response_dao = QuestionnaireResponseDao() self.questionnaire_response_answer_dao = QuestionnaireResponseAnswerDao( ) self.participant_summary_dao = ParticipantSummaryDao() self.CODE_1 = Code(codeId=1, system=PPI_SYSTEM, value=GENDER_IDENTITY_QUESTION_CODE, display=u'c', topic=u'd', codeType=CodeType.QUESTION, mapped=True) self.CODE_2 = Code(codeId=2, system='a', value='x', display=u'y', codeType=CodeType.QUESTION, mapped=False) self.CODE_3 = Code(codeId=3, system='a', value='c', codeType=CodeType.ANSWER, mapped=True, parentId=1) self.CODE_4 = Code(codeId=4, system='a', value='d', codeType=CodeType.ANSWER, mapped=True, parentId=2) self.CODE_5 = Code(codeId=5, system='a', value='e', codeType=CodeType.ANSWER, mapped=False, parentId=1) self.CODE_6 = Code(codeId=6, system='a', value='f', codeType=CodeType.ANSWER, mapped=True, parentId=1) self.MODULE_CODE_7 = Code(codeId=7, system=PPI_SYSTEM, value=THE_BASICS_PPI_MODULE, codeType=CodeType.MODULE, mapped=True) self.CONCEPT_1 = QuestionnaireConcept(codeId=7) self.CODE_1_QUESTION_1 = QuestionnaireQuestion(linkId='a', codeId=1, repeats=False) self.CODE_2_QUESTION = QuestionnaireQuestion(linkId='d', codeId=2, repeats=True) # Same code as question 1 self.CODE_1_QUESTION_2 = QuestionnaireQuestion(linkId='x', codeId=1, repeats=False) self.skip_code = Code(codeId=8, system=PPI_SYSTEM, value=PMI_SKIP_CODE, mapped=True, codeType=CodeType.ANSWER) config.override_setting(config.CONSENT_PDF_BUCKET, [_FAKE_BUCKET])