def check_sources(self):
""" Validates that all reference sources in OpenMRS have been defined in OCL. """
url_base = self.OCL_API_URL[self.ocl_api_env]
headers = {'Authorization': 'Token %s' % self.ocl_api_token}
reference_sources = ConceptReferenceSource.objects.all()
reference_sources = reference_sources.filter(retired=0)
enum_reference_sources = enumerate(reference_sources)
for num, source in enum_reference_sources:
source_id = OclOpenmrsHelper.get_ocl_source_id_from_omrs_id(source.name)
if self.verbosity >= 1:
print 'Checking source "%s"' % source_id
# Check that source exists in the source directory (which maps sources to orgs)
org_id = OclOpenmrsHelper.get_source_owner_id(ocl_source_id=source_id)
if self.verbosity >= 1:
print '...found owner "%s" in source directory' % org_id
# Check that org:source exists in OCL
if self.ocl_api_token:
url = url_base + 'orgs/%s/sources/%s/' % (org_id, source_id)
r = requests.head(url, headers=headers)
if r.status_code != requests.codes.OK:
raise UnrecognizedSourceException('%s not found in OCL.' % url)
if self.verbosity >= 1:
print '...found %s in OCL' % url
elif self.verbosity >= 1:
print '...no api token provided, skipping check on OCL.'
return True
def export_concept_mappings(self, concept):
"""
Generate OCL-formatted mappings for the concept, excluding set members and Q/A.
Creates both internal and external mappings, based on the mapping definition.
:param concept: Concept with the mappings to export from OpenMRS database.
:returns: List of OCL-formatted mapping dictionaries for the concept.
"""
export_data = []
for ref_map in concept.conceptreferencemap_set.all():
map_dict = None
# Internal Mapping
if ref_map.concept_reference_term.concept_source.name == self.org_id:
if str(concept.concept_id) == ref_map.concept_reference_term.code:
# mapping to self, so ignore
self.cnt_ignored_self_mappings += 1
continue
map_dict = self.generate_internal_mapping(
map_type=ref_map.map_type.name,
from_concept=concept,
to_concept_code=ref_map.concept_reference_term.code,
external_id=ref_map.concept_reference_term.uuid)
self.cnt_internal_mappings_exported += 1
# External Mapping
else:
# Prepare to_source_id
omrs_to_source_id = ref_map.concept_reference_term.concept_source.name
to_source_id = OclOpenmrsHelper.get_ocl_source_id_from_omrs_id(omrs_to_source_id)
to_org_id = OclOpenmrsHelper.get_source_owner_id(ocl_source_id=to_source_id)
# Generate the external mapping dictionary
map_dict = self.generate_external_mapping(
map_type=ref_map.map_type.name,
from_concept=concept,
to_org_id=to_org_id,
to_source_id=to_source_id,
to_concept_code=ref_map.concept_reference_term.code,
to_concept_name=ref_map.concept_reference_term.name,
external_id=ref_map.uuid)
self.cnt_external_mappings_exported += 1
if map_dict:
export_data.append(map_dict)
return export_data
def export_concept_mappings(self, concept):
"""
Generate OCL-formatted mappings for the concept, excluding set members and Q/A.
Creates both internal and external mappings, based on the mapping definition.
:param concept: Concept with the mappings to export from OpenMRS database.
:returns: List of OCL-formatted mapping dictionaries for the concept.
"""
export_data = []
for ref_map in concept.conceptreferencemap_set.all():
map_dict = None
# Internal Mapping
if ref_map.concept_reference_term.concept_source.name == self.org_id:
map_dict = self.generate_internal_mapping(
map_type=ref_map.map_type.name,
from_concept=concept,
to_concept_code=ref_map.concept_reference_term.code,
external_id=ref_map.concept_reference_term.uuid)
self.cnt_internal_mappings_exported += 1
# External Mapping
else:
# Prepare to_source_id
omrs_to_source_id = ref_map.concept_reference_term.concept_source.name
to_source_id = OclOpenmrsHelper.get_ocl_source_id_from_omrs_id(
omrs_to_source_id)
to_org_id = OclOpenmrsHelper.get_source_owner_id(
ocl_source_id=to_source_id)
# Generate the external mapping dictionary
map_dict = self.generate_external_mapping(
map_type=ref_map.map_type.name,
from_concept=concept,
to_org_id=to_org_id,
to_source_id=to_source_id,
to_concept_code=ref_map.concept_reference_term.code,
to_concept_name=ref_map.concept_reference_term.name,
external_id=ref_map.uuid)
self.cnt_external_mappings_exported += 1
if map_dict:
export_data.append(map_dict)
return export_data
def validate_reference_map(self, m_ocl):
map_type = m_ocl['map_type']
from_concept_id = m_ocl['from_concept_code']
to_concept_code = m_ocl['to_concept_code']
to_source_name = OclOpenmrsHelper.get_omrs_source_id_from_ocl_id(m_ocl['to_source_name'])
try:
m_omrs = ConceptReferenceMap.objects.get(map_type__name=map_type,
concept_reference_term__code=to_concept_code,
concept_id=from_concept_id,
concept_reference_term__concept_source__name=to_source_name)
return m_omrs.concept_map_id
except ConceptReferenceMap.DoesNotExist:
return False
except ConceptReferenceMap.MultipleObjectsReturned:
print 'Multiple objects returned from MySQL for reference mapping: %s\n' % m_ocl
return False
def create_ciel_mapping(self, to_source, ciel_id, map_type, iad_id,
external_id):
source_id = OclOpenmrsHelper.get_omrs_source_id_from_ocl_id(to_source)
if source_id is None:
print 'Missing source in ciel mapping "%s"' % to_source
else:
creference_sources = ConceptReferenceSource.objects.filter(
name=source_id)
if len(creference_sources) != 0:
creference_source = creference_sources[0]
else:
creference_source = ConceptReferenceSource(
name=source_id,
hl7_code=None,
creator=1,
retired=False,
uuid=uuid.uuid1(),
date_created=timezone.now())
creference_source.save()
creference_terms = ConceptReferenceTerm.objects.filter(
code=ciel_id, concept_source=creference_source)
if len(creference_terms) != 0:
creference_term = creference_terms[0]
else:
creference_term = ConceptReferenceTerm(
code=ciel_id,
concept_source=creference_source,
creator=1,
retired=False,
uuid=uuid.uuid1(),
date_created=timezone.now())
creference_term.save()
creference_map_types = ConceptMapType.objects.filter(name=map_type)
if len(creference_map_types) != 0:
creference_map_type = creference_map_types[0]
else:
creference_map_type = ConceptMapType(
name=map_type,
creator=1,
uuid=uuid.uuid1(),
date_created=timezone.now())
creference_map_type.save()
creference_maps = ConceptReferenceMap.objects.filter(
concept_id=iad_id,
concept_reference_term=creference_term,
map_type=creference_map_type)
if len(creference_maps) != 0:
creference_map = creference_maps[0]
else:
creference_map = ConceptReferenceMap(
concept_reference_term=creference_term,
concept_id=iad_id,
uuid=external_id,
map_type_id=creference_map_type.concept_map_type_id,
creator=1,
date_created=timezone.now())
creference_map.save()
return
def create_external_mapping(self,
concepts,
map_type,
from_concept_url,
to_source_url,
to_concept_code,
external_id,
retired=False):
""" Generate OCL-formatted dictionary for an external mapping based on passed params. """
ss = to_source_url.split("/")
source_name = ss[4]
source_id = OclOpenmrsHelper.get_omrs_source_id_from_ocl_id(
source_name)
if source_id is None:
print 'Missing source in external mapping "%s"' % source_name
else:
cc = from_concept_url.split("/")
concept_id = cc[6]
# if self.verbosity >= 1:
# print 'Checking source "%s" at uuid "%s"' % (source_id, external_id)
creference_sources = ConceptReferenceSource.objects.filter(
name=source_id)
if len(creference_sources) != 0:
creference_source = creference_sources[0]
else:
creference_source = ConceptReferenceSource(
name=source_id,
hl7_code=None,
creator=1,
retired=False,
uuid=uuid.uuid1(),
date_created=timezone.now())
creference_source.save()
creference_map_types = ConceptMapType.objects.filter(name=map_type)
if len(creference_map_types) != 0:
creference_map_type = creference_map_types[0]
else:
creference_map_type = ConceptMapType(
name=map_type,
creator=1,
uuid=uuid.uuid1(),
date_created=timezone.now())
creference_map_type.save()
creference_terms = ConceptReferenceTerm.objects.filter(
code=to_concept_code, concept_source=creference_source)
if len(creference_terms) != 0:
# creference_term = creference_terms[0]
# Nothing to be done, the mapping must also exist, just return
return
else:
creference_term = ConceptReferenceTerm(
code=to_concept_code,
concept_source=creference_source,
creator=1,
retired=False,
uuid=uuid.uuid1(),
date_created=timezone.now())
creference_term.save()
new_concept_id = ConceptHelper.get_new_id(concepts,
int(concept_id))
if new_concept_id != None:
creference_maps = ConceptReferenceMap.objects.filter(
concept_id=new_concept_id,
concept_reference_term=creference_term,
map_type_id=creference_map_type.concept_map_type_id)
if len(creference_maps) != 0:
creference_map = creference_maps[0]
else:
creference_map = ConceptReferenceMap(
concept_reference_term_id=creference_term.
concept_reference_term_id,
concept_id=new_concept_id,
uuid=external_id,
map_type_id=creference_map_type.concept_map_type_id,
creator=1,
date_created=timezone.now())
creference_map.save()
self.create_ciel_mapping(cc[4], int(concept_id), map_type,
new_concept_id, uuid.uuid1())
return
def check_sources(self):
""" Validates that all reference sources in OpenMRS have been defined in OCL. """
# Set things up
ocl_env_url = '%s/' % OclOpenmrsHelper.OCL_API_URL[self.ocl_api_env]
headers = {}
if self.ocl_api_token:
headers['Authorization'] = 'Token %s' % self.ocl_api_token
# Grab the list of references sources in the OpenMRS dictionary from MySql
reference_sources = ConceptReferenceSource.objects.all()
reference_sources = reference_sources.filter(retired=0)
enum_reference_sources = enumerate(reference_sources)
# Check if each reference source exists in the specified OCL environment
matching_sources = []
missing_source_in_ocl = []
missing_source_definition = []
org_fields_to_remove = [
'collections_url', 'members_url', 'uuid', 'public_sources', 'url',
'public_collections', 'created_by', 'created_on', 'updated_by',
'members', 'updated_on', 'sources_url'
]
source_fields_to_remove = [
'versions_url', 'created_on', 'updated_by', 'uuid', 'created_by',
'mappings_url', 'owner_url', 'concepts_url', 'versions',
'active_mappings', 'url', 'active_concepts', 'updated_on'
]
for num, source in enum_reference_sources:
if self.verbosity >= 2:
print 'Checking OpenMRS Reference source: "%s"' % source.name
# Check if the source definition exists in local source directory
try:
ocl_source_id = OclOpenmrsHelper.get_ocl_source_id_from_omrs_id(
source.name)
ocl_org_id = OclOpenmrsHelper.get_source_owner_id(
ocl_source_id=ocl_source_id)
except UnrecognizedSourceException:
missing_source_definition.append({
'omrs_source':
source,
'omrs_ref_source_name':
source.name
})
if self.verbosity >= 2:
print ' ...Missing reference source definition'
continue
if self.verbosity >= 2:
print ' ...Corresponding source "%s" and owner "%s" found in source directory' % (
ocl_source_id, ocl_org_id)
# Check that org:source exists in OCL
ocl_org_url = ocl_env_url + 'orgs/%s/' % (q(ocl_org_id))
ocl_source_url = ocl_env_url + \
'orgs/%s/sources/%s/' % (q(ocl_org_id), q(ocl_source_id))
ocl_org_response = requests.get(ocl_org_url, headers=headers)
ocl_source_response = requests.get(ocl_source_url, headers=headers)
if self.verbosity >= 2:
try:
ocl_org_json = ocl_org_response.json()
ocl_source_json = ocl_source_response.json()
[ocl_org_json.pop(key) for key in org_fields_to_remove]
[
ocl_source_json.pop(key)
for key in source_fields_to_remove
]
print json.dumps(ocl_org_json)
print json.dumps(ocl_source_json)
except Exception:
pass
current_source = {
'omrs_source': source,
'omrs_ref_source_name': source.name,
'ocl_org_id': ocl_org_id,
'ocl_source_id': ocl_source_id,
'ocl_source_url': ocl_source_url,
'response_code': ocl_source_response.status_code
}
if ocl_source_response.status_code != requests.codes.OK:
missing_source_in_ocl.append(current_source)
if self.verbosity >= 2:
print ' ...Org or source not found in OCL: %s' % ocl_source_url
else:
matching_sources.append(current_source)
if self.verbosity >= 2:
print ' ...Org and source found in OCL: %s' % ocl_source_url
# Display the results
if self.verbosity:
print '------------------------------------------------------'
print 'CHECK SOURCES RESULTS:'
print '** Matched Sources:', len(matching_sources)
for source in matching_sources:
print '%s: %s' % (source['omrs_ref_source_name'],
source['ocl_source_url'])
print '\n** Missing Source Definitions:', len(
missing_source_definition)
for source in missing_source_definition:
source_definition = {
'owner_type': 'org',
'omrs_id': source['omrs_ref_source_name'],
'owner_id': '',
'ocl_id': ''
}
print '\n%s:' % source['omrs_ref_source_name']
print '- OpenMRS Reference Source Definition:', source[
'omrs_source'].__dict__
print '- Template to add to source definitions:', json.dumps(
source_definition)
print '\n** Missing Sources in OCL:', len(missing_source_in_ocl)
for source in missing_source_in_ocl:
print '\n%s:' % source['omrs_ref_source_name']
print '- OpenMRS Reference Source Definition:', source[
'omrs_source'].__dict__
print '- Source Details:', source
if missing_source_definition or missing_source_in_ocl:
return False
return True
def export_concept_mappings(self, id1, m, conv_ids):
"""
Generate OCL-formatted mappings for the concept, excluding set members and Q/A.
Creates both internal and external mappings, based on the mapping definition.
:param concept: Concept with the mappings to export from OpenMRS database.
:returns: List of OCL-formatted mapping dictionaries for the concept.
"""
fromconc = Concept.objects.get(concept_id=id1)
#print len(conv_ids)
if 'to_source_url' in m: #external mapping
#All external mappings are OpenMRS mappings
map_type_id = ConceptMapType.objects.get(name=m['map_type'])
src = m['to_source_url'].split('/')
src_name = src[4]
#print src_name
src_name = OclOpenmrsHelper.get_omrs_source_id_from_ocl_id(
src_name)
source = ConceptReferenceSource.objects.get(name=src_name)
#print source.name
code = m['to_concept_code']
uuid_ref_term = m['ref_term']
uuid_ref_map = m['external_id'] # uuid of ref_map
# update concept_reference_term if not present
conc_ref_term = ConceptReferenceTerm.objects.filter(
code=code, concept_source=source)
if len(conc_ref_term) == 0:
conc_ref_term = ConceptReferenceTerm(concept_source=source,
code=code,
retired=m['retired'],
uuid=uuid_ref_term)
conc_ref_term.save()
#print conc_ref_term
conc_ref_term = ConceptReferenceTerm.objects.get(
code=code, concept_source=source)
#update concept_reference_map
conc_ref_map = ConceptReferenceMap.objects.filter(
concept_reference_term=conc_ref_term,
map_type=map_type_id,
concept=fromconc)
if len(conc_ref_map) == 0:
conc_ref_map = ConceptReferenceMap(
concept=fromconc,
uuid=uuid_ref_map,
concept_reference_term=conc_ref_term,
map_type=map_type_id)
conc_ref_map.save()
#print conc_ref_map
else: #internal mapping
s = m['to_concept_url'].split('/')
code = (int)(s[6])
if code in conv_ids:
toconc = Concept.objects.get(concept_id=conv_ids[code])
#print toconc
uuid = m['external_id'] # uuid of concept_answer
#Q-AND-A and set members are always internal mappings
if (m['map_type'] == 'Q-AND-A'):
srt_wt = (float)(m['sort_weight'])
ans = ConceptAnswer.objects.filter(question_concept=fromconc,
answer_concept=toconc)
if len(ans) == 0:
ans = ConceptAnswer(question_concept=fromconc,
answer_concept=toconc,
uuid=uuid,
sort_weight=srt_wt)
ans.save()
#print ans
elif m['map_type'] == 'CONCEPT-SET':
srt_wt = (float)(m['sort_weight'])
conc_set = ConceptSet.objects.filter(
concept_set_owner=fromconc, concept=toconc)
if len(conc_set) == 0:
conc_set = ConceptSet(concept_set_owner=fromconc,
concept=toconc,
uuid=uuid,
sort_weight=srt_wt)
conc_set.save()
#print conc_set
else:
map_type_id = ConceptMapType.objects.get(name=m['map_type'])
src = m['to_concept_url'].split('/')
src_name = src[4]
#print src_name
src_name = OclOpenmrsHelper.get_omrs_source_id_from_ocl_id(
src_name)
source = ConceptReferenceSource.objects.get(name=src_name)
#print source.name
code = (str)(code)
uuid_ref_map = m['ref_m']
uuid_ref_term = m['external_id'] # uuid of ref_map
# update concept_reference_term if not present
conc_ref_term = ConceptReferenceTerm.objects.filter(
uuid=uuid_ref_term)
if len(conc_ref_term) == 0:
conc_ref_term = ConceptReferenceTerm(concept_source=source,
code=code,
retired=m['retired'],
uuid=uuid_ref_term)
conc_ref_term.save()
#print conc_ref_term
conc_ref_term = ConceptReferenceTerm.objects.get(
uuid=uuid_ref_term)
# update concept_reference_map
conc_ref_map = ConceptReferenceMap(
concept=fromconc,
uuid=uuid_ref_map,
concept_reference_term=conc_ref_term,
map_type=map_type_id)
conc_ref_map.save()