def gen_hist(patient_id_ref, patient_name):
h = familymemberhistory.FamilyMemberHistory()
h.patient = {"reference": patient_id_ref, "display": patient_name}
h.status = 'completed'
h.relationship = {
"coding": [{
"code": "MTH",
"system": "http://hl7.org/fhir/familial-relationship"
}]
}
measurement = quantity.Quantity()
measurement.unit = "cm"
measurement.value = 162
h.extension = [{
"url":
"http://fhir-registry.smarthealthit.org/StructureDefinition/family-history#height",
"valueQuantity": {
"unit": "cm",
"value": 162
}
}]
with open(os.path.join('json_data', 'ob-clark', 'ob-clark-mth.json'),
'w') as f:
print(json.dumps(OrderedDict(h.as_json()),
indent=4,
separators=(',', ': ')),
file=f)
h = familymemberhistory.FamilyMemberHistory()
h.patient = {"reference": patient_id_ref, "display": patient_name}
h.status = 'completed'
h.relationship = {
"coding": [{
"code": "FTH",
"system": "http://hl7.org/fhir/familial-relationship"
}]
}
measurement = quantity.Quantity()
measurement.unit = "cm"
measurement.value = 177
h.extension = [{
"url":
"http://fhir-registry.smarthealthit.org/StructureDefinition/family-history#height",
"valueQuantity": {
"unit": "cm",
"value": 177
}
}]
with open(os.path.join('json_data', 'ob-clark', 'ob-clark-fth.json'),
'w') as f:
print(json.dumps(OrderedDict(h.as_json()),
indent=4,
separators=(',', ': ')),
file=f)
def value(self):
return q.Quantity({
"value": float(self.val),
"unit": "mmol/l",
"system": "http://unitsofmeasure.org",
"code": "mmol/L"
})
def fhir_age(obj, mapping, field):
""" Generic function to convert Age or AgeRange to FHIR Age. """
age_extension = extension.Extension()
age_extension.url = mapping
if isinstance(obj[field]['age'], dict):
age_extension.valueRange = range.Range()
age_extension.valueRange.low = quantity.Quantity()
age_extension.valueRange.low.unit = obj[field]['age']['start']['age']
age_extension.valueRange.high = quantity.Quantity()
age_extension.valueRange.high.unit = obj[field]['age']['end']['age']
else:
age_extension.valueAge = age.Age()
age_extension.valueAge.unit = obj[field]['age']
return age_extension
def value(self):
return q.Quantity({
"value": float(self.val),
"unit": "breaths/minute",
"system": "http://unitsofmeasure.org",
"code": "/min"
})
def fhir_age(obj, mapping, field):
""" Generic function to convert Age or AgeRange to FHIR Age. """
age_extension = extension.Extension()
age_extension.url = mapping
if "start" in obj[field]: # Is an age range
age_extension.valueRange = range_.Range()
age_extension.valueRange.low = quantity.Quantity()
age_extension.valueRange.low.unit = obj[field]['start']['age']
age_extension.valueRange.high = quantity.Quantity()
age_extension.valueRange.high.unit = obj[field]['end']['age']
else: # Is a precise age
age_extension.valueAge = age.Age()
age_extension.valueAge.unit = obj[field]['age']
return age_extension
def to_fhir(self):
med_quantity = quantity.Quantity()
med_quantity.value = self.value
med_quantity.unit = self.unit
med_quantity.system = self.system
med_quantity.code = self.code
return med_quantity
def age_to_fhir(obj, mapping, field):
"""Generic function to convert Phenopackets Age or AgeRange to FHIR Age.
:param obj: object to which field Age or AgeRange belongs to
:param mapping: mapping from PHENOPACKETS_ON_FHIR
:param field: name of the field that stores age
:return: age extension object
"""
age_extension = extension.Extension()
age_extension.url = mapping
if isinstance(obj[field]['age'], dict):
age_extension.valueRange = range.Range()
age_extension.valueRange.low = quantity.Quantity()
age_extension.valueRange.low.unit = obj[field]['age']['start']['age']
age_extension.valueRange.high = quantity.Quantity()
age_extension.valueRange.high.unit = obj[field]['age']['end']['age']
else:
age_extension.valueAge = age.Age()
age_extension.valueAge.unit = obj[field]['age']
return age_extension
def _add_quantity_value(self, Observation, measurement):
"""
Adds a quantity value object to Observation.
:param Observation: fhirclient.models.observation.Observation object
:param measurement: measurement dictionary
:returns: Observation object
"""
Quantity = q.Quantity()
Quantity.value = self.observation_dict[measurement]['value']
Quantity.unit = self.observation_dict[measurement]['unit']
Observation.valueQuantity = Quantity
return Observation
def _add_value(self, Observation, measurement):
"""
Adds values to an Observation FHIR object. Uses 'type' within dictionary to determine logic.
:param self:
:param Observation: Observation FHIR object.
:param measurement: Specific observation measurement. References a dictionary.
:returns: Observation FHIR object.
"""
if measurement['type'] == 'quantity':
Observation.valueCodeableConcept = self._create_FHIRCodeableConcept(
code=measurement['code'],
system=measurement['system'],
display=measurement['display'])
elif measurement['type'] == 'codeable':
Quantity = q.Quantity()
Quantity.value = self.observation_dict[measurement]['value']
Quantity.unit = self.observation_dict[measurement]['unit']
Observation.valueQuantity = Quantity
elif measurement['type'] == 'valuestring':
Observation.valueString = measurement['value']
return Observation
def addObservationToServer(self, patient_id, value_code, value_unit,
value_quantity, coding_code, coding_display,
coding_system, timestamp):
observation = o.Observation()
# Create Value Quantity
quantity = q.Quantity()
quantity.code = value_code
quantity.unit = value_unit
quantity.value = value_quantity
observation.valueQuantity = quantity
# Create Coding
code = cc.CodeableConcept()
coding_item = c.Coding()
coding_item.code = coding_code
coding_item.display = coding_display
coding_item.system = coding_system
coding_list = [coding_item]
code.coding = coding_list
observation.code = code
# Create Subject
reference = r.FHIRReference()
reference.reference = self.getPatientById(patient_id).relativePath()
observation.subject = reference
# Create Status
observation.status = 'final'
# Create Issued/EffectiveDateTime
observation.effectiveDateTime = d.FHIRDate(timestamp)
# Write the observation
result_json = observation.create(self.smart.server)
observation.id = self.getCreatedId(result_json)
return observation.id
def writeLabProfile(labs_counts, labs_frequencyPerYear, labs_fractionOfSubjects,labs_units, labs_names,
labs_stats, labs_aboveBelowNorm, labs_correlatedLabsCoefficients, labs_abscorrelation,
labs_correlatedMedsCoefficients, labs_correlatedProceduresCoefficients,
labs_correlatedDiagnosisCoefficients, labs_correlatedPhenotypesCoefficients,
cohort='All', sex='All', race='All', age_low='All', age_high=None,
topN=10, correlationCutoff=0.3):
"""Write out Lab Clinical Profile to JSON File and save locally
Keywords:
Structures from output of calculateAnyProfile(profileType='labs')
cohort -- short name for cohort, special characters besides hyphens are prohibited (default 'All')
sex -- specification of whether this is a 'All', 'Male', or 'Female' sex profile (default 'All')
race -- specification of whether this is 'All', 'White or Caucasian', 'Black or African American', 'Other' race profile (default 'All')
age_low -- low age range for this profile (default 'All')
age_high -- high age range for this profile (default None)
topN -- integer representing the maximum number of correlations to report in the profile, ranked descending (default 10)
correlationCutoff -- minimum correlation coefficient value to report for whole profile (default 0.3)
"""
import os
import sys
import sqlalchemy
import urllib.parse
import pandas as pd
import numpy as np
import getpass
from dataclasses import dataclass
from SciServer import Authentication
from datetime import datetime
import json
from fhir_loader import fhir_loader
from fhirclient.models import clinicalprofile, fhirreference, identifier, codeableconcept, fhirdate, quantity
import pymssql
# Initialize profile
clinicalProfile = clinicalprofile.ClinicalProfile()
clinicalProfile.resourceType = 'ClinicalProfile'
if sex == 'M':
sex = 'Male'
elif sex =='F':
sex = 'Female'
# Header info
if (age_low != 'All'):
clinicalProfile.id = 'jh-labs-'+cohort+'-'+sex+'-'+race+'-'+str(int(age_low))+'-'+str(int(age_high))
clinicalProfile.identifier = [identifier.Identifier({'value':
'jh-labs-'+cohort+'-'+sex+'-'+race+'-'+
str(int(age_low))+'-'+str(int(age_high))})]
clinicalProfile.cohort = fhirreference.FHIRReference({'reference':
'Group/jh-labs-'+cohort+'-'+sex+'-'+race+'-'+str(int(age_low))
+'-'+str(int(age_high))})
else:
clinicalProfile.id = 'jh-labs-'+cohort+'-'+sex+'-'+race+'-'+str(age_low)
clinicalProfile.identifier = [identifier.Identifier({'value':
'jh-labs-'+cohort+'-'+sex+'-'+race+'-'+str(age_low)})]
clinicalProfile.cohort = fhirreference.FHIRReference({'reference':
'Group/jh-labs-'+cohort+'-'+sex+'-'+race+'-'+str(age_low)})
clinicalProfile.status = 'draft'
clinicalProfile.population = fhirreference.FHIRReference({'reference': 'Group/jh-labs-'+cohort})
clinicalProfile.date = fhirdate.FHIRDate(str(datetime.now()).replace(' ', 'T'))
clinicalProfile.reporter = fhirreference.FHIRReference({'reference': 'Organization/JHM',
'type': 'Organization',
'display': 'Johns Hopkins School of Medicine'})
## LABS
labs = list()
corrmat = (pd.DataFrame(labs_correlatedLabsCoefficients).unstack(level=[0,1]).corr(min_periods=50)
.droplevel(level=0).droplevel(level=0,axis=1))
lab_names = pd.DataFrame({'lab_name':labs_names}).reset_index()
lab_counts = pd.DataFrame({'lab_counts':labs_counts}).reset_index().rename({'index':'LAB_LOINC'},axis=1)
lab_info = lab_names.merge(lab_counts, how='inner', on='LAB_LOINC').set_index('LAB_LOINC')
for thisLab in lab_info.index:
# Check if STDEV is NaN and skip that lab if so
if np.isnan(float(labs_stats.loc[thisLab]['std'].median())):
continue
# Build the profile
thisCPLab = clinicalprofile.ClinicalProfileLab()
# try:
thisCPLab.code = [codeableconcept.CodeableConcept(dict(coding=[dict(system='http://loinc.org',
code=thisLab)],
text=lab_info.loc[thisLab]['lab_name'][0]))]
thisCPLab.count = int(lab_info.loc[thisLab]['lab_counts'])
thisCPLab.frequencyPerYear = round(float(labs_frequencyPerYear.loc[thisLab].mean()),3)
thisCPLab.fractionOfSubjects = round(float(labs_fractionOfSubjects.loc[thisLab].mean()),3)
thisCPLab.scalarDistribution = clinicalprofile.ClinicalProfileLabScalarDistribution()
thisCPLab.scalarDistribution.units = quantity.Quantity(dict(unit=str(labs_units.loc[thisLab][0])))
thisCPLab.scalarDistribution.min = round(float(labs_stats.loc[thisLab]['min'].min()),3)
thisCPLab.scalarDistribution.max = round(float(labs_stats.loc[thisLab]['max'].max()),3)
thisCPLab.scalarDistribution.mean = round(float(labs_stats.loc[thisLab]['mean'].mean()),3)
thisCPLab.scalarDistribution.median = round(float(labs_stats.loc[thisLab]['median'].median()),3)
thisCPLab.scalarDistribution.stdDev = round(float(labs_stats.loc[thisLab]['std'].median()),3)
deciles = list()
for dec in labs_stats.columns[5:]:
deciles.append(clinicalprofile.ClinicalProfileLabScalarDistributionDecile(
dict(nth=int(dec),
value=round(labs_stats.loc[thisLab][dec].mean(),3))))
thisCPLab.scalarDistribution.decile = deciles
thisCPLab.scalarDistribution.fractionAboveNormal = round(float(labs_aboveBelowNorm.loc[thisLab].aboveNorm.mean()),3)
thisCPLab.scalarDistribution.fractionBelowNormal = round(float(labs_aboveBelowNorm.loc[thisLab].belowNorm.mean()),3)
try:
yearly_vals = dict()
for year in corrmat.loc[thisLab].index:
crosstab = corrmat.loc[(thisLab, year)]
yearly_vals[year] = (crosstab[crosstab.index.get_level_values(level=1).astype('float') == year]
.droplevel(level=1))
topNcorrs = pd.DataFrame(yearly_vals).apply(np.mean, axis=1).drop(thisLab).nlargest(topN).round(3)
entries = list()
for code, corr in topNcorrs.iteritems():
if corr <= correlationCutoff:
continue
otherLoinc = [(dict(coding=[dict(system='http://loinc.org', code=code)],
text=str(lab_info.loc[code]['lab_name'][0])))]
entries.append(dict(labcode=otherLoinc, coefficient=corr))
if not entries:
print('No correlated Labs for Lab ', thisLab)
else:
thisCPLab.scalarDistribution.correlatedLabs = clinicalprofile.ClinicalProfileLabScalarDistributionCorrelatedLabs(
dict(topn=topN,
entry=entries))
except:
print('No correlated Labs for Lab ', thisLab)
try:
topNcorrs = (pd.DataFrame(labs_correlatedMedsCoefficients.loc[thisLab].groupby(['JH_INGREDIENT_RXNORM_CODE'])
.Relative_Counts.mean())
.Relative_Counts.nlargest(topN).round(3))
entries = list()
for code, corr in topNcorrs.iteritems():
if corr <= correlationCutoff:
continue
otherRX = [dict(medicationCodeableConcept=dict(coding=
[dict(system='http://www.nlm.nih.gov/research/umls/rxnorm/', code=code)]))]
entries.append(dict(meds=otherRX, coefficient=corr))
if not entries:
print('No correlated Meds for Lab ', thisLab)
else:
thisCPLab.scalarDistribution.correlatedMedications = clinicalprofile.\
ClinicalProfileLabScalarDistributionCorrelatedMedications(
dict(topn=topN,
entry=entries))
except:
print('No correlated Meds for Lab ', thisLab)
try:
topNcorrs = (pd.DataFrame(labs_correlatedDiagnosisCoefficients.loc[thisLab].groupby(['DX'])
.Relative_Counts.mean())
.Relative_Counts.nlargest(topN).round(3))
entries = list()
for code, corr in topNcorrs.iteritems():
if corr <= correlationCutoff:
continue
otherDX = (dict(coding=[dict(system='http://www.icd10data.com/', code=code)]))
entries.append(dict(code=otherDX, coefficient=corr))
if not entries:
print('No correlated Diagnoses for Lab ', thisLab)
else:
thisCPLab.scalarDistribution.correlatedDiagnoses = clinicalprofile.\
ClinicalProfileLabScalarDistributionCorrelatedDiagnoses(
dict(topn=topN,
entry=entries))
except:
print('No correlated Diagnoses for Lab ', thisLab)
try:
topNcorrs = (pd.DataFrame(labs_correlatedProceduresCoefficients.loc[thisLab].groupby(['RAW_PX'])
.Relative_Counts.mean())
.Relative_Counts.nlargest(topN).round(3))
entries = list()
for code, corr in topNcorrs.iteritems():
if corr <= correlationCutoff:
continue
otherProc = [(dict(coding=[dict(system='http://www.ama-assn.org/practice-management/cpt', code=code)]))]
entries.append(dict(code=otherProc, coefficient=corr))
if not entries:
print('No correlated Procedures for Lab ', thisLab)
else:
thisCPLab.scalarDistribution.correlatedProcedures = clinicalprofile.\
ClinicalProfileLabScalarDistributionCorrelatedProcedures(
dict(topn=topN,
entry=entries))
except:
print('No correlated Procedures for Lab ', thisLab)
try:
topNcorrs = (pd.DataFrame(labs_correlatedPhenotypesCoefficients.loc[thisLab].groupby(['HPO'])
.Relative_Counts.mean())
.Relative_Counts.nlargest(topN).round(3))
entries = list()
for code, corr in topNcorrs.iteritems():
if corr <= correlationCutoff:
continue
otherHPO = (dict(coding=[dict(system='http://hpo.jax.org/app/', code=code)]))
entries.append(dict(code=otherHPO, coefficient=corr))
if not entries:
print('No correlated Phenotypes for Lab ', thisLab)
else:
thisCPLab.scalarDistribution.correlatedPhenotypes = clinicalprofile.\
ClinicalProfileLabScalarDistributionCorrelatedPhenotypes(
dict(topn=topN,
entry=entries))
except:
print('No correlated Phenotypes for Lab ', thisLab)
labs.append(thisCPLab)
# except:
# print('This lab did not work ', thisLab)
clinicalProfile.lab = labs
if age_high != None:
filename = cohort+'_resources/jh-labs-'+cohort+'-'+sex+'-'+race+'-'+str(int(age_low))+'-'+str(int(age_high))+'.json'
else:
filename = cohort+'_resources/jh-labs-'+cohort+'-'+sex+'-'+race+'-'+str(age_low)+'.json'
with open(filename, 'w') as outfile:
json.dump(clinicalProfile.as_json(), outfile, indent=4)
del(clinicalProfile)
return print('Write to '+ filename + ' successful')
import logging
from logging.handlers import RotatingFileHandler
from fhirclient.models.fhirabstractbase import FHIRValidationError
# from googletrans import Translator # unavailable for 3.4 as a pip install
application = Flask(__name__)
application.secret_key = 'you-will-never-guess'
# -=================== globals constants used across sessions=============================
cache = SimpleCache()
# ref_server = 'http://test.fhir.org/r3/' # fhir reference serv≠er
ref_server = 'http://sqlonfhir-stu3.azurewebsites.net/fhir/'
ref_server_name = 'Telstra'
quantity = QT.Quantity() # for Quantity type answers
no_answer = [[''],[],None,['0'],['0', '']]
# item.type to valueType conversion
answer_type = {
'boolean': 'valueBoolean',
'decimal': 'valueDecimal',
'integer': 'valueInteger',
'date': 'valueDate',
'dateTime': 'valueDateTime',
'time': 'valueTime',
'string': 'valueString',
'text': 'valueString',
'url': 'valueUri',
'attachment': 'valueAttachment',
'reference': 'valueReference',
'quantity': 'valueString', # TODO need to fix this