def reference_type(self):
reference = None
if is_concept(self.expression):
reference = CONCEPTS_EXPRESSIONS
if is_mapping(self.expression):
reference = MAPPINGS_EXPRESSIONS
return reference
def process(self, collection_uri, expressions):
self.processed += 1
self.log("Processing: {} ({}/{})".format(collection_uri,
self.processed, self.total))
collection = Collection.objects.filter(uri=collection_uri).first()
saved_references = []
concepts = []
mappings = []
if collection:
for expression in expressions:
self.log("Processing Expression: {} ".format(expression))
__is_concept = is_concept(expression)
if __is_concept:
model = Concept
_instances = concepts
else:
model = Mapping
_instances = mappings
instance = model.objects.filter(uri=expression).first()
if self.drop_version_if_version_missing and not instance:
instance = model.objects.filter(
uri=drop_version(expression)).first()
if not instance:
self.not_found_references.append(expression)
continue
latest_version = instance.get_latest_version()
if not latest_version:
latest_version = model.create_initial_version(instance)
if __is_concept:
latest_version.cloned_names = [
name.clone() for name in instance.names.all()
]
latest_version.cloned_descriptions = [
desc.clone()
for desc in instance.descriptions.all()
]
latest_version.set_locales()
parent = instance.parent
latest_version.sources.set([parent, parent.head])
reference = CollectionReference(expression=latest_version.uri)
reference.save()
saved_references.append(reference)
_instances.append(latest_version)
self.created.append(expression)
collection.references.add(*saved_references)
if concepts:
collection.concepts.add(*concepts)
collection.batch_index(collection.concepts, ConceptDocument)
if mappings:
collection.mappings.add(*mappings)
collection.batch_index(collection.mappings, MappingDocument)
else:
self.not_found.append(collection_uri)
def create_entities_from_expressions(self):
if is_concept(self.expression):
self.concepts = self.get_concepts()
elif is_mapping(self.expression):
self.mappings = self.get_mappings()
if (not self.concepts or not self.concepts.exists()) and (
not self.mappings or not self.mappings.exists()):
raise ValidationError({'detail': [EXPRESSION_INVALID]})
def __get_children_from_expressions(expressions):
concepts = Concept.objects.none()
mappings = Mapping.objects.none()
for expression in expressions:
if is_concept(expression):
concepts |= Concept.from_uri_queryset(expression)
if is_mapping(expression):
mappings |= Mapping.from_uri_queryset(expression)
return concepts, mappings
def get_related_mappings_with_version_information(expressions):
related_mappings = []
for expression in expressions:
if is_concept(expression):
concepts = CollectionReference.get_concept_heads_from_expression(expression)
for concept in concepts:
related_mappings += concept.get_latest_unidirectional_mappings()
return [mapping.uri for mapping in related_mappings]
def get_cascaded_mapping_uris_from_concept_expressions(self, expressions):
mapping_uris = []
for expression in expressions:
if is_concept(expression):
mapping_uris += list(
self.mappings.filter(
from_concept__uri__icontains=drop_version(
expression)).values_list('uri', flat=True))
return mapping_uris
def test_is_concept(self):
self.assertFalse(is_concept(None))
self.assertFalse(is_concept(''))
self.assertFalse(is_concept('orgs/org-1/sources/source-1/concept/'))
self.assertTrue(is_concept('orgs/org-1/sources/source-1/concepts/'))
self.assertTrue(is_concept('users/user-1/sources/source-1/concepts/'))
self.assertTrue(
is_concept('users/user-1/collections/coll-1/concepts/'))
self.assertTrue(is_concept('/concepts/'))
def get_related_mappings_with_version_information(self, instance,
expressions):
related_mappings = []
for expression in expressions:
if is_concept(expression):
concepts = CollectionReference.get_concept_heads_from_expression(
expression)
for concept in concepts:
related_mappings += concept.get_unidirectional_mappings()
return self.get_version_information_of_related_mappings(
instance, related_mappings)
def get_all_related_mappings(self, expressions):
all_related_mappings = []
unversioned_mappings = concept_expressions = []
for expression in expressions:
if is_mapping(expression):
unversioned_mappings.append(drop_version(expression))
elif is_concept(expression):
concept_expressions.append(expression)
for concept_expression in concept_expressions:
ref = CollectionReference(expression=concept_expression)
try:
self.validate(ref)
all_related_mappings += ref.get_related_mappings(
unversioned_mappings)
except: # pylint: disable=bare-except
continue
return all_related_mappings
def process_line(self, line): # pylint: disable=too-many-locals,too-many-statements,too-many-branches
data = json.loads(line)
original_data = data.copy()
self.processed += 1
_id = data.pop('_id')
data['internal_reference_id'] = get(_id, '$oid')
for attr in [
'active_concepts', 'active_mappings', 'last_child_update',
'last_concept_update', 'last_mapping_update',
'parent_version_id', 'previous_version_id',
'versioned_object_type_id', 'concepts', 'mappings'
]:
data.pop(attr, None)
data['snapshot'] = data.pop('collection_snapshot', None)
data['external_id'] = data.pop('version_external_id', None)
versioned_object_id = data.pop('versioned_object_id')
versioned_object = self.get_collection(versioned_object_id)
version = data.pop('mnemonic')
created_at = data.pop('created_at')
updated_at = data.pop('updated_at')
created_by = data.get('created_by')
updated_by = data.get('updated_by')
creator = self.get_user(created_by)
updater = self.get_user(updated_by)
if creator:
data['created_by'] = creator
if updater:
data['updated_by'] = updater
data['created_at'] = get(created_at, '$date')
data['updated_at'] = get(updated_at, '$date')
data['organization_id'] = versioned_object.organization_id
data['user_id'] = versioned_object.user_id
data['collection_type'] = versioned_object.collection_type
references = data.pop('references') or []
self.log("Processing: {} ({}/{})".format(version, self.processed,
self.total))
uri = data['uri']
if Collection.objects.filter(uri=uri).exists():
self.existed.append(original_data)
else:
collection = Collection.objects.create(
**data, version=version, mnemonic=versioned_object.mnemonic)
if collection.id:
self.created.append(original_data)
else:
self.failed.append(original_data)
return
saved_references = []
concepts = []
mappings = []
for ref in references:
expression = ref.get('expression')
__is_concept = is_concept(expression)
concept = None
mapping = None
if __is_concept:
concept = Concept.objects.filter(uri=expression).first()
if concept:
concepts.append(concept)
else:
mapping = Mapping.objects.filter(uri=expression).first()
if mapping:
mappings.append(mapping)
if not concept and not mapping:
self.add_in_not_found_expression(uri, expression)
continue
reference = CollectionReference(expression=expression)
reference.save()
saved_references.append(reference)
collection.references.set(saved_references)
collection.concepts.set(concepts)
collection.mappings.set(mappings)
collection.batch_index(collection.concepts, ConceptDocument)
collection.batch_index(collection.mappings, MappingDocument)
