def reference_type(self): reference = None if is_concept(self.expression): reference = CONCEPTS_EXPRESSIONS if is_mapping(self.expression): reference = MAPPINGS_EXPRESSIONS return reference
def process(self, collection_uri, expressions): self.processed += 1 self.log("Processing: {} ({}/{})".format(collection_uri, self.processed, self.total)) collection = Collection.objects.filter(uri=collection_uri).first() saved_references = [] concepts = [] mappings = [] if collection: for expression in expressions: self.log("Processing Expression: {} ".format(expression)) __is_concept = is_concept(expression) if __is_concept: model = Concept _instances = concepts else: model = Mapping _instances = mappings instance = model.objects.filter(uri=expression).first() if self.drop_version_if_version_missing and not instance: instance = model.objects.filter( uri=drop_version(expression)).first() if not instance: self.not_found_references.append(expression) continue latest_version = instance.get_latest_version() if not latest_version: latest_version = model.create_initial_version(instance) if __is_concept: latest_version.cloned_names = [ name.clone() for name in instance.names.all() ] latest_version.cloned_descriptions = [ desc.clone() for desc in instance.descriptions.all() ] latest_version.set_locales() parent = instance.parent latest_version.sources.set([parent, parent.head]) reference = CollectionReference(expression=latest_version.uri) reference.save() saved_references.append(reference) _instances.append(latest_version) self.created.append(expression) collection.references.add(*saved_references) if concepts: collection.concepts.add(*concepts) collection.batch_index(collection.concepts, ConceptDocument) if mappings: collection.mappings.add(*mappings) collection.batch_index(collection.mappings, MappingDocument) else: self.not_found.append(collection_uri)
def create_entities_from_expressions(self): if is_concept(self.expression): self.concepts = self.get_concepts() elif is_mapping(self.expression): self.mappings = self.get_mappings() if (not self.concepts or not self.concepts.exists()) and ( not self.mappings or not self.mappings.exists()): raise ValidationError({'detail': [EXPRESSION_INVALID]})
def __get_children_from_expressions(expressions): concepts = Concept.objects.none() mappings = Mapping.objects.none() for expression in expressions: if is_concept(expression): concepts |= Concept.from_uri_queryset(expression) if is_mapping(expression): mappings |= Mapping.from_uri_queryset(expression) return concepts, mappings
def get_related_mappings_with_version_information(expressions): related_mappings = [] for expression in expressions: if is_concept(expression): concepts = CollectionReference.get_concept_heads_from_expression(expression) for concept in concepts: related_mappings += concept.get_latest_unidirectional_mappings() return [mapping.uri for mapping in related_mappings]
def get_cascaded_mapping_uris_from_concept_expressions(self, expressions): mapping_uris = [] for expression in expressions: if is_concept(expression): mapping_uris += list( self.mappings.filter( from_concept__uri__icontains=drop_version( expression)).values_list('uri', flat=True)) return mapping_uris
def test_is_concept(self): self.assertFalse(is_concept(None)) self.assertFalse(is_concept('')) self.assertFalse(is_concept('orgs/org-1/sources/source-1/concept/')) self.assertTrue(is_concept('orgs/org-1/sources/source-1/concepts/')) self.assertTrue(is_concept('users/user-1/sources/source-1/concepts/')) self.assertTrue( is_concept('users/user-1/collections/coll-1/concepts/')) self.assertTrue(is_concept('/concepts/'))
def get_related_mappings_with_version_information(self, instance, expressions): related_mappings = [] for expression in expressions: if is_concept(expression): concepts = CollectionReference.get_concept_heads_from_expression( expression) for concept in concepts: related_mappings += concept.get_unidirectional_mappings() return self.get_version_information_of_related_mappings( instance, related_mappings)
def get_all_related_mappings(self, expressions): all_related_mappings = [] unversioned_mappings = concept_expressions = [] for expression in expressions: if is_mapping(expression): unversioned_mappings.append(drop_version(expression)) elif is_concept(expression): concept_expressions.append(expression) for concept_expression in concept_expressions: ref = CollectionReference(expression=concept_expression) try: self.validate(ref) all_related_mappings += ref.get_related_mappings( unversioned_mappings) except: # pylint: disable=bare-except continue return all_related_mappings
def process_line(self, line): # pylint: disable=too-many-locals,too-many-statements,too-many-branches data = json.loads(line) original_data = data.copy() self.processed += 1 _id = data.pop('_id') data['internal_reference_id'] = get(_id, '$oid') for attr in [ 'active_concepts', 'active_mappings', 'last_child_update', 'last_concept_update', 'last_mapping_update', 'parent_version_id', 'previous_version_id', 'versioned_object_type_id', 'concepts', 'mappings' ]: data.pop(attr, None) data['snapshot'] = data.pop('collection_snapshot', None) data['external_id'] = data.pop('version_external_id', None) versioned_object_id = data.pop('versioned_object_id') versioned_object = self.get_collection(versioned_object_id) version = data.pop('mnemonic') created_at = data.pop('created_at') updated_at = data.pop('updated_at') created_by = data.get('created_by') updated_by = data.get('updated_by') creator = self.get_user(created_by) updater = self.get_user(updated_by) if creator: data['created_by'] = creator if updater: data['updated_by'] = updater data['created_at'] = get(created_at, '$date') data['updated_at'] = get(updated_at, '$date') data['organization_id'] = versioned_object.organization_id data['user_id'] = versioned_object.user_id data['collection_type'] = versioned_object.collection_type references = data.pop('references') or [] self.log("Processing: {} ({}/{})".format(version, self.processed, self.total)) uri = data['uri'] if Collection.objects.filter(uri=uri).exists(): self.existed.append(original_data) else: collection = Collection.objects.create( **data, version=version, mnemonic=versioned_object.mnemonic) if collection.id: self.created.append(original_data) else: self.failed.append(original_data) return saved_references = [] concepts = [] mappings = [] for ref in references: expression = ref.get('expression') __is_concept = is_concept(expression) concept = None mapping = None if __is_concept: concept = Concept.objects.filter(uri=expression).first() if concept: concepts.append(concept) else: mapping = Mapping.objects.filter(uri=expression).first() if mapping: mappings.append(mapping) if not concept and not mapping: self.add_in_not_found_expression(uri, expression) continue reference = CollectionReference(expression=expression) reference.save() saved_references.append(reference) collection.references.set(saved_references) collection.concepts.set(concepts) collection.mappings.set(mappings) collection.batch_index(collection.concepts, ConceptDocument) collection.batch_index(collection.mappings, MappingDocument)