def test_persist_new_positive__earlier_source_version(self):
version1 = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(version1.mappings))
version2 = SourceVersion.for_base_object(self.source1, label='version2')
version2.save()
self.assertEquals(0, len(version2.mappings))
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
kwargs = {
'parent_resource': self.source1,
'parent_resource_version': version1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
version1 = SourceVersion.objects.get(id=version1.id)
self.assertEquals(1, len(version1.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in version1.mappings)
version2 = SourceVersion.objects.get(id=version2.id)
self.assertEquals(0, len(version2.mappings))
latest_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(latest_version.mappings))
def test_retire_positive(self):
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.concepts))
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='First',
names=[self.name],
)
kwargs = {
'parent_resource': self.source1,
}
errors = Concept.persist_new(concept, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertFalse(concept.retired)
self.assertEquals(1, concept.num_versions)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertTrue(concept_version.is_latest_version)
self.assertFalse(concept_version.retired)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(1, len(source_version.concepts))
self.assertEquals(concept_version.id, source_version.concepts[0])
errors = Concept.retire(concept, self.user1)
self.assertFalse(errors)
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertTrue(concept.retired)
self.assertEquals(2, concept.num_versions)
previous_version = ConceptVersion.objects.get(id=concept_version.id)
self.assertFalse(previous_version.is_latest_version)
self.assertFalse(previous_version.retired)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertTrue(concept_version.is_latest_version)
self.assertTrue(concept_version.retired)
self.assertEquals(self.user1.username, concept_version.version_created_by)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(1, len(source_version.concepts))
self.assertEquals(concept_version.id, source_version.concepts[0])
self.assertEquals(
1, ConceptVersion.objects.filter(versioned_object_id=concept.id, retired=True).count())
self.assertEquals(
1, ConceptVersion.objects.filter(versioned_object_id=concept.id, retired=False).count())
errors = Concept.retire(concept, self.user1)
self.assertEquals(1, len(errors))
def test_persist_changes_positive__seed_from_previous_over_parent(self):
version1 = SourceVersion.for_base_object(self.source1, 'version1')
version1.concepts = [1]
version1.full_clean()
version1.save()
version2 = SourceVersion.for_base_object(self.source1, 'version2')
version2.concepts = [2]
version2.full_clean()
version2.save()
self.assertIsNone(version2.previous_version)
version3 = SourceVersion.for_base_object(self.source1, 'version3')
version3.full_clean()
version3.save()
mnemonic = version3.mnemonic
released = version3.released
description = version3.description
external_id = version3.external_id
id = version3.id
version3._parent_version_mnemonic = 'version2'
version3._previous_version_mnemonic = 'version1'
version3.mnemonic = "%s-prime" % mnemonic
version3.released = not released
version3.description = "%s-prime" % description
version3.external_id = "%s-prime" % external_id
errors = SourceVersion.persist_changes(version3)
self.assertEquals(0, len(errors))
version3 = SourceVersion.objects.get(id=id)
self.assertEquals(self.source1, version3.versioned_object)
self.assertEquals(3, self.source1.num_versions)
self.assertEquals(version3, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(version1, version3.previous_version)
self.assertEquals(version2, version3.parent_version)
self.assertEquals([], version3.concepts)
self.assertNotEquals(mnemonic, version3.mnemonic)
self.assertNotEquals(released, version3.released)
self.assertNotEquals(description, version3.description)
self.assertNotEquals(external_id, version3.external_id)
errors = SourceVersion.persist_changes(version3, seed_concepts=True)
self.assertEquals(0, len(errors))
version3 = SourceVersion.objects.get(id=id)
self.assertEquals(self.source1, version3.versioned_object)
self.assertEquals(3, self.source1.num_versions)
self.assertEquals(version3, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(version2, version3.parent_version)
self.assertEquals(version1, version3.previous_version)
self.assertEquals([1], version3.concepts)
def test_persist_new_positive(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
self.assertEquals(self.source1.public_access, mapping.public_access)
self.assertEquals('user1', mapping.created_by)
self.assertEquals('user1', mapping.updated_by)
self.assertEquals(self.source1, mapping.parent)
self.assertEquals('Same As', mapping.map_type)
self.assertEquals(self.concept1, mapping.from_concept)
self.assertEquals(self.concept2, mapping.to_concept)
self.assertEquals(self.source1, mapping.from_source)
self.assertEquals(self.source1.owner_name, mapping.from_source_owner)
self.assertEquals(self.source1.mnemonic, mapping.from_source_name)
self.assertEquals(self.source1, mapping.get_to_source())
self.assertEquals(self.source1.owner_name, mapping.to_source_owner)
self.assertEquals(self.concept2.mnemonic, mapping.get_to_concept_code())
self.assertEquals(self.concept2.display_name, mapping.get_to_concept_name())
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
def test_edit_mapping_make_new_version_positive(self):
mapping1 = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping1, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertEquals(1,len(MappingVersion.objects.filter(versioned_object_id=mapping1.id)))
mapping1.map_type='BROADER_THAN'
Mapping.persist_changes(mapping1, self.user1)
self.assertEquals(2, len(MappingVersion.objects.filter(versioned_object_id=mapping1.id)))
old_version = MappingVersion.objects.get(versioned_object_id=mapping1.id, is_latest_version=False)
new_version= MappingVersion.objects.get(versioned_object_id=mapping1.id, is_latest_version=True)
self.assertFalse(old_version.is_latest_version)
self.assertTrue(new_version.is_latest_version)
self.assertEquals(new_version.map_type,'BROADER_THAN')
self.assertEquals(old_version.map_type,'Same As')
def test_persist_changes_negative__parent_version_is_self(self):
version1 = SourceVersion.for_base_object(self.source1, 'version1', released=True)
version1.full_clean()
version1.save()
mnemonic = version1.mnemonic
released = version1.released
description = version1.description
external_id = version1.external_id
id = version1.id
version1._parent_version_mnemonic = mnemonic
version1.released = not released
version1.description = "%s-prime" % description
version1.external_id = "%s-prime" % external_id
errors = SourceVersion.persist_changes(version1)
self.assertEquals(1, len(errors))
self.assertTrue('parentVersion' in errors)
version1 = SourceVersion.objects.get(id=id)
self.assertEquals(self.source1, version1.versioned_object)
self.assertEquals(1, self.source1.num_versions)
self.assertEquals(version1, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(mnemonic, version1.mnemonic)
self.assertEquals(released, version1.released)
self.assertEquals(description, version1.description)
self.assertEquals(external_id, version1.external_id)
def test_persist_persist_changes_positive(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
Mapping.persist_new(mapping, self.user1, **kwargs)
mapping = Mapping.objects.get(external_id='mapping1')
to_concept = mapping.to_concept
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
mapping.to_concept = self.concept3
errors = Mapping.persist_changes(mapping, self.user1)
self.assertEquals(0, len(errors))
mapping = Mapping.objects.get(external_id='mapping1')
self.assertEquals(self.concept3, mapping.to_concept)
self.assertNotEquals(to_concept, mapping.to_concept)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
mv = MappingVersion.objects.filter(versioned_object_id=mapping.id)
self.assertTrue(mv[1].id in source_version.mappings)
def test_import_job_for_change_in_data(self):
stdout_stub = TestStream()
stderr_stub = TestStream()
mapping = Mapping(
parent=self.source1,
map_type="SAME-AS",
from_concept=self.concept3,
to_source=self.source1,
to_concept_code="413532003",
external_id="junk",
)
kwargs = {"parent_resource": self.source1}
Mapping.persist_new(mapping, self.user1, **kwargs)
source_version = SourceVersion.get_latest_version_of(self.source1)
source_version.mappings = [mapping.id]
source_version.save()
importer = MappingsImporter(self.source1, self.testfile, stdout_stub, stderr_stub, "test")
importer.import_mappings(total=1)
self.assertTrue("**** Processed 1 out of 1 mappings - 1 updated, ****" in stdout_stub.getvalue())
self.assertTrue(("Updated mapping with ID " + mapping.id) in stdout_stub.getvalue())
updated_mapping = Mapping.objects.get(to_concept_code="413532003")
self.assertTrue(updated_mapping.retired)
self.assertEquals(updated_mapping.external_id, "70279ABBBBBBBBBBBBBBBBBBBBBBBBBBBBBB")
def test_create_concept_reference_source_version__positive(self):
source_version = SourceVersion.get_latest_version_of(self.source1)
concept_reference = ConceptReference(
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
mnemonic='reference1',
concept=self.concept1,
source_version=source_version,
)
concept_reference.full_clean()
concept_reference.save()
self.assertTrue(ConceptReference.objects.filter(mnemonic='reference1').exists())
self.assertEquals(self.concept1.concept_class, concept_reference.concept_class)
self.assertEquals(self.concept1.datatype, concept_reference.data_type)
self.assertEquals(self.concept1.parent, concept_reference.source)
self.assertEquals(self.concept1.owner_name, concept_reference.owner_name)
self.assertEquals(self.concept1.owner_type, concept_reference.owner_type)
self.assertEquals(self.concept1.display_name, concept_reference.display_name)
self.assertEquals(self.concept1.display_locale, concept_reference.display_locale)
self.assertEquals(
'/users/user1/sources/source1/%s/concepts/concept1/' % source_version.mnemonic,
concept_reference.concept_reference_url)
self.assertFalse(concept_reference.is_current_version)
def test_persist_new_positive(self):
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
parent=self.source1,
concept_class='First',
names=[self.name],
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.concepts))
kwargs = {
'parent_resource': self.source1,
}
errors = Concept.persist_new(concept, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertFalse(concept.retired)
self.assertEquals(self.name.name, concept.display_name)
self.assertEquals(self.name.locale, concept.display_locale)
self.assertEquals(self.source1.owner_name, concept.owner_name)
self.assertEquals(self.source1.owner_type, concept.owner_type)
self.assertEquals(self.source1.public_access, concept.public_access)
self.assertEquals(1, concept.num_versions)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertEquals(concept_version, concept_version.root_version)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.concepts))
self.assertTrue(concept_version.id in source_version.concepts)
self.assertEquals(concept_version.mnemonic, concept_version.id)
def test_persist_changes_positive__good_parent_version(self):
version1 = SourceVersion.for_base_object(self.source1, 'version1')
version1.full_clean()
version1.save()
version2 = SourceVersion.for_base_object(self.source1, 'version2')
version2.full_clean()
version2.save()
self.assertIsNone(version2.parent_version)
mnemonic = version2.mnemonic
released = version2.released
description = version2.description
external_id = version2.external_id
id = version2.id
version2._parent_version_mnemonic = 'version1'
version2.mnemonic = "%s-prime" % mnemonic
version2.released = not released
version2.description = "%s-prime" % description
version2.external_id = "%s-prime" % external_id
errors = SourceVersion.persist_changes(version2)
self.assertEquals(0, len(errors))
version2 = SourceVersion.objects.get(id=id)
self.assertEquals(self.source1, version2.versioned_object)
self.assertEquals(2, self.source1.num_versions)
self.assertEquals(version2, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(version1, version2.parent_version)
self.assertNotEquals(mnemonic, version2.mnemonic)
self.assertNotEquals(released, version2.released)
self.assertNotEquals(description, version2.description)
self.assertNotEquals(external_id, version2.external_id)
def test_persist_clone_positive(self):
self.assertEquals(2, self.concept1.num_versions)
self.assertEquals(
self.concept_version, ConceptVersion.get_latest_version_of(self.concept1))
source_version = SourceVersion.get_latest_version_of(self.source1)
source_version.update_concept_version(self.concept_version)
self.assertEquals(1, len(source_version.concepts))
self.assertEquals(self.concept_version.id, source_version.concepts[0])
version2 = self.concept_version.clone()
errors = ConceptVersion.persist_clone(version2, self.user1)
self.assertEquals(0, len(errors))
self.assertEquals(3, self.concept1.num_versions)
self.assertEquals(version2, ConceptVersion.get_latest_version_of(self.concept1))
self.assertEquals(self.concept_version.public_access, version2.public_access)
self.assertEquals(self.concept_version, version2.previous_version)
self.assertEquals(self.concept_version.root_version, version2.root_version)
self.assertEquals(self.concept_version.external_id, version2.external_id)
self.assertEquals(self.user1.username, version2.version_created_by)
source_version.update_concept_version(version2)
self.assertEquals(1, len(source_version.concepts))
self.assertEquals(version2.id, source_version.concepts[0])
def test_source_version_create_positive__child_and_subsequent_versions(self):
version1 = SourceVersion(
name='version1',
mnemonic='version1',
versioned_object=self.source1,
released=True,
created_by=self.user1,
updated_by=self.user1,
)
version1.full_clean()
version1.save()
self.assertTrue(SourceVersion.objects.filter(
mnemonic='version1',
versioned_object_type=ContentType.objects.get_for_model(Source),
versioned_object_id=self.source1.id
).exists())
self.assertEquals(version1, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(1, self.source1.num_versions)
version2 = SourceVersion(
name='version2',
mnemonic='version2',
versioned_object=self.source1,
parent_version=version1,
created_by=self.user1,
updated_by=self.user1,
)
version2.full_clean()
version2.save()
self.assertTrue(SourceVersion.objects.filter(
mnemonic='version2',
versioned_object_type=ContentType.objects.get_for_model(Source),
versioned_object_id=self.source1.id
).exists())
self.assertEquals(version1, version2.parent_version)
self.assertEquals(version1.mnemonic, version2.parent_version_mnemonic)
self.assertIsNone(version2.previous_version)
self.assertIsNone(version2.previous_version_mnemonic)
self.assertEquals(2, self.source1.num_versions)
version3 = SourceVersion(
name='version3',
mnemonic='version3',
versioned_object=self.source1,
previous_version=version2,
created_by=self.user1,
updated_by=self.user1,
)
version3.full_clean()
version3.save()
self.assertTrue(SourceVersion.objects.filter(
mnemonic='version3',
versioned_object_type=ContentType.objects.get_for_model(Source),
versioned_object_id=self.source1.id
).exists())
self.assertEquals(version2, version3.previous_version)
self.assertEquals(version2.mnemonic, version3.previous_version_mnemonic)
self.assertIsNone(version3.parent_version)
self.assertIsNone(version3.parent_version_mnemonic)
self.assertEquals(3, self.source1.num_versions)
def test_persist_persist_changes_negative__no_updated_by(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
Mapping.persist_new(mapping, self.user1, **kwargs)
mapping = Mapping.objects.get(external_id='mapping1')
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
mapping.to_concept = self.concept3
errors = Mapping.persist_changes(mapping, None)
self.assertEquals(1, len(errors))
self.assertTrue('updated_by' in errors)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
def test_import_job_for_change_in_data(self):
stdout_stub = TestStream()
stderr_stub = TestStream()
mapping = Mapping(
parent=self.source1,
map_type='SAME-AS',
from_concept=self.concept3,
to_source=self.source1,
to_concept_code='413532003',
external_id='junk'
)
kwargs = {
'parent_resource': self.source1,
}
Mapping.persist_new(mapping, self.user1, **kwargs)
source_version = SourceVersion.get_latest_version_of(self.source1)
source_version.mappings = [mapping.id]
source_version.save()
importer = MappingsImporter(self.source1, self.testfile, stdout_stub, stderr_stub, 'test')
importer.import_mappings(total=1)
self.assertTrue('mappings of 1 1 - 1 updated' in stdout_stub.getvalue())
self.assertTrue(('Updated mapping with ID ' + mapping.id) in stdout_stub.getvalue())
updated_mapping = Mapping.objects.get(to_concept_code='413532003')
self.assertTrue(updated_mapping.retired)
self.assertEquals(updated_mapping.external_id, '70279ABBBBBBBBBBBBBBBBBBBBBBBBBBBBBB')
def test_source_version_create_positive(self):
source_version = SourceVersion(
name='version1',
mnemonic='version1',
versioned_object=self.source1,
released=True,
created_by=self.user1,
updated_by=self.user1,
)
source_version.full_clean()
source_version.save()
self.assertTrue(SourceVersion.objects.filter(
mnemonic='version1',
versioned_object_type=ContentType.objects.get_for_model(Source),
versioned_object_id=self.source1.id
).exists())
self.assertIsNone(source_version.previous_version)
self.assertIsNone(source_version.previous_version_mnemonic)
self.assertIsNone(source_version.parent_version)
self.assertIsNone(source_version.parent_version_mnemonic)
self.assertEquals(self.org1.mnemonic, source_version.parent_resource)
self.assertEquals(self.org1.resource_type, source_version.parent_resource_type)
self.assertEquals(source_version, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(1, self.source1.num_versions)
def test_import_job_for_change_in_data(self):
stdout_stub = TestStream()
source_version_latest = SourceVersion.get_latest_version_of(self.source1)
concept = Concept(
mnemonic='1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='Diagnosis',
external_id='1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA',
names=[self.name])
kwargs = {
'parent_resource': self.source1,
}
Concept.persist_new(concept, self.user1, **kwargs)
importer = ConceptsImporter(self.source1, self.testfile, 'test', stdout_stub, TestStream())
importer.import_concepts(total=1)
all_concept_versions = ConceptVersion.objects.all()
self.assertEquals(len(all_concept_versions), 2)
concept = Concept.objects.get(mnemonic='1')
latest_concept_version = [version for version in all_concept_versions if version.previous_version][0]
self.assertEquals(len(latest_concept_version.names), 4)
self.assertTrue(('Updated concept, replacing version ID ' + latest_concept_version.previous_version.id) in stdout_stub.getvalue())
self.assertTrue('concepts of 1 1 - 1 updated' in stdout_stub.getvalue())
def retrieve(self, request, *args, **kwargs):
super(SourceRetrieveUpdateDestroyView, self).retrieve(request, *args, **kwargs)
self.object = self.get_object()
serializer = self.get_serializer(self.object)
data = serializer.data
source_version = SourceVersion.get_latest_version_of(self.object)
self.includeConceptsAndMappings(request, data, source_version)
return Response(data)
def initialize(self, request, path_info_segment, **kwargs):
super(MappingBaseView, self).initialize(request, path_info_segment, **kwargs)
if self.parent_resource:
if hasattr(self.parent_resource, 'versioned_object'):
self.parent_resource_version = self.parent_resource
self.parent_resource = self.parent_resource.versioned_object
else:
self.parent_resource_version = SourceVersion.get_latest_version_of(self.parent_resource)
def __init__(self, source, concepts_file, user, output_stream, error_stream): """ Initialize mapping importer """ self.source = source self.concepts_file = concepts_file self.stdout = output_stream self.stderr = error_stream self.user = user # Retrieve latest source version and, if specified, create a new one self.source_version = SourceVersion.get_latest_version_of(self.source)
def test_persist_new_positive__same_mapping_different_source(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
# Repeat with same concepts
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping2',
)
kwargs = {
'parent_resource': self.source2,
}
source_version = SourceVersion.get_latest_version_of(self.source2)
self.assertEquals(0, len(source_version.mappings))
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Mapping.objects.filter(external_id='mapping2').exists())
mapping = Mapping.objects.get(external_id='mapping2')
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
def test_persist_new_positive(self):
kwargs = {
'parent_resource': self.userprofile1
}
errors = Source.persist_new(self.new_source, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Source.objects.filter(name='source1').exists())
source = Source.objects.get(name='source1')
self.assertTrue(SourceVersion.objects.filter(versioned_object_id=source.id))
source_version = SourceVersion.objects.get(versioned_object_id=source.id)
self.assertEquals(1, source.num_versions)
self.assertEquals(source_version, SourceVersion.get_latest_version_of(source))
def test_import_job_for_one_record(self):
stdout_stub = TestStream()
importer = ConceptsImporter(self.source1, self.testfile, 'test', stdout_stub, TestStream())
importer.import_concepts(total=1)
self.assertTrue('Created new concept: 1 = Diagnosis' in stdout_stub.getvalue())
self.assertTrue('Finished importing concepts!' in stdout_stub.getvalue())
inserted_concept = Concept.objects.get(mnemonic='1')
self.assertEquals(inserted_concept.parent, self.source1)
inserted_concept_version = ConceptVersion.objects.get(versioned_object_id=inserted_concept.id)
source_version_latest = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(source_version_latest.concepts, [inserted_concept_version.id])
def test_create_concept_reference_concept_and_source_versions__negative(self):
with self.assertRaises(ValidationError):
concept_reference = ConceptReference(
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
mnemonic='reference1',
concept=self.concept1,
concept_version=self.version1,
source_version=SourceVersion.get_latest_version_of(self.source1)
)
concept_reference.full_clean()
concept_reference.save()
def test_import_job_for_one_record(self):
stdout_stub = TestStream()
stderr_stub = TestStream()
importer = MappingsImporter(self.source1, self.testfile, stdout_stub, stderr_stub, 'test')
importer.import_mappings(total=1)
self.assertTrue('Created new mapping:' in stdout_stub.getvalue())
self.assertTrue('/users/user1/sources/source1/:413532003' in stdout_stub.getvalue())
inserted_mapping = Mapping.objects.get(to_concept_code='413532003')
self.assertEquals(inserted_mapping.to_source, self.source1)
self.assertEquals(inserted_mapping.from_source, self.source2)
mapping_ids = SourceVersion.get_latest_version_of(self.source1).mappings
mapping_version = MappingVersion.objects.get(versioned_object_id=inserted_mapping.id, is_latest_version=True)
self.assertEquals(mapping_ids[0], mapping_version.id)
def __init__(self, source, concepts_file, user, output_stream, error_stream, save_validation_errors=True, validation_logger=None):
""" Initialize mapping importer """
self.source = source
self.concepts_file = concepts_file
self.stdout = output_stream
self.stderr = error_stream
self.user = user
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_latest_version_of(self.source)
self.validation_logger = validation_logger
self.save_validation_errors = save_validation_errors
if self.save_validation_errors and self.validation_logger is None:
self.validation_logger = ValidationLogger()
def test_get_latest_version_of(self):
latest_version_identity = 'version'
expected_latest_source_version = SourceVersion(
name=latest_version_identity,
mnemonic=latest_version_identity,
versioned_object=self.source,
released=True,
created_by=self.user,
updated_by=self.user,
)
expected_latest_source_version.full_clean()
expected_latest_source_version.save()
self.assertEquals(SourceVersion.get_latest_version_of(self.source).name, latest_version_identity)
def retrieve(self, request, *args, **kwargs):
super(SourceRetrieveUpdateDestroyView, self).retrieve(request, *args, **kwargs)
self.object = self.get_object()
serializer = self.get_serializer(self.object)
data = serializer.data
source_version = None
offset = request.QUERY_PARAMS.get(OFFSET_PARAM, DEFAULT_OFFSET)
try:
offset = int(offset)
except ValueError:
offset = DEFAULT_OFFSET
limit = settings.REST_FRAMEWORK.get('MAX_PAGINATE_BY', self.paginate_by)
include_retired = False
include_concepts = request.QUERY_PARAMS.get(INCLUDE_CONCEPTS_PARAM, False)
include_mappings = request.QUERY_PARAMS.get(INCLUDE_MAPPINGS_PARAM, False)
updated_since = None
if include_concepts or include_mappings:
source_version = SourceVersion.get_latest_version_of(self.object)
paginate_by = self.get_paginate_by(EmptyQuerySet())
if paginate_by:
limit = min(limit, paginate_by)
include_retired = request.QUERY_PARAMS.get(INCLUDE_RETIRED_PARAM, False)
updated_since = parse_updated_since_param(request)
if include_concepts:
source_version_concepts = source_version.concepts
queryset = ConceptVersion.objects.filter(is_active=True)
if not include_retired:
queryset = queryset.filter(~Q(retired=True))
if updated_since:
queryset = queryset.filter(updated_at__gte=updated_since)
queryset = queryset.filter(id__in=source_version_concepts)
queryset = queryset[offset:offset+limit]
serializer = ConceptVersionDetailSerializer(queryset, many=True)
data['concepts'] = serializer.data
if include_mappings:
all_children = source_version.mappings
queryset = Mapping.objects.filter(is_active=True)
if not include_retired:
queryset = queryset.filter(~Q(retired=True))
queryset = queryset.filter(id__in=all_children)
if updated_since:
queryset = queryset.filter(updated_at__gte=updated_since)
queryset = queryset[offset:offset+limit]
serializer = MappingDetailSerializer(queryset, many=True)
data['mappings'] = serializer.data
return Response(data)
def initialize(self, request, path_info_segment, **kwargs):
self.include_retired = request.QUERY_PARAMS.get(INCLUDE_RETIRED_PARAM, False)
include_inverse_param = request.GET.get(INCLUDE_INVERSE_MAPPINGS_PARAM, 'false')
self.include_inverse_mappings = 'true' == include_inverse_param
parent_path_info = self.get_parent_in_path(path_info_segment, levels=1)
if parent_path_info and '/' != parent_path_info:
self.concept = self.get_object_for_path(parent_path_info, self.request)
parent_path_info = self.get_parent_in_path(parent_path_info, levels=2)
if parent_path_info and '/' != parent_path_info:
self.parent_resource = self.get_object_for_path(parent_path_info, self.request)
if hasattr(self.parent_resource, 'versioned_object'):
self.parent_resource_version = self.parent_resource
self.parent_resource = self.parent_resource.versioned_object
else:
self.parent_resource_version = SourceVersion.get_latest_version_of(self.parent_resource)
def test_update_mapping_with_invalid_record(self):
mapping = Mapping(parent=self.source1, map_type="SAME-AS", from_concept=self.concept3, to_concept=self.concept1)
kwargs = {"parent_resource": self.source1}
Mapping.persist_new(mapping, self.user1, **kwargs)
source_version = SourceVersion.get_latest_version_of(self.source1)
source_version.mappings = [mapping.id]
source_version.save()
stderr_stub = TestStream()
invalid_json_file = open("./integration_tests/fixtures/one_internal_invalid_mapping.json", "rb")
importer = MappingsImporter(self.source1, invalid_json_file, TestStream(), stderr_stub, "test")
importer.import_mappings(total=1)
self.assertTrue(
"Must specify either 'to_concept' or 'to_source' & 'to_concept_code'. Cannot specify both."
in stderr_stub.getvalue()
)
def test_persist_new_negative__no_parent(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(1, len(errors))
self.assertTrue('non_field_errors' in errors)
non_field_errors = errors['non_field_errors']
self.assertEquals(1, len(non_field_errors))
self.assertTrue('parent' in non_field_errors[0])
self.assertFalse(
Mapping.objects.filter(external_id='mapping1').exists())
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(0, len(source_version.mappings))
def __init__(self,
source,
concepts_file,
user,
output_stream,
error_stream,
save_validation_errors=True,
validation_logger=None):
""" Initialize mapping importer """
self.source = source
self.concepts_file = concepts_file
self.stdout = output_stream
self.stderr = error_stream
self.user = user
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_latest_version_of(self.source)
self.validation_logger = validation_logger
self.save_validation_errors = save_validation_errors
if self.save_validation_errors and self.validation_logger is None:
self.validation_logger = ValidationLogger()
def initialize(self, request, path_info_segment, **kwargs):
self.include_retired = request.QUERY_PARAMS.get(
INCLUDE_RETIRED_PARAM, False)
include_inverse_param = request.GET.get(INCLUDE_INVERSE_MAPPINGS_PARAM,
'false')
self.include_inverse_mappings = 'true' == include_inverse_param
parent_path_info = self.get_parent_in_path(path_info_segment, levels=1)
if parent_path_info and '/' != parent_path_info:
self.concept = self.get_object_for_path(parent_path_info,
self.request)
parent_path_info = self.get_parent_in_path(parent_path_info,
levels=2)
if parent_path_info and '/' != parent_path_info:
self.parent_resource = self.get_object_for_path(
parent_path_info, self.request)
if hasattr(self.parent_resource, 'versioned_object'):
self.parent_resource_version = self.parent_resource
self.parent_resource = self.parent_resource.versioned_object
else:
self.parent_resource_version = SourceVersion.get_latest_version_of(
self.parent_resource)
def test_persist_new_positive(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(
Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
self.assertEquals(self.source1.public_access, mapping.public_access)
self.assertEquals('user1', mapping.created_by)
self.assertEquals('user1', mapping.updated_by)
self.assertEquals(self.source1, mapping.parent)
self.assertEquals('Same As', mapping.map_type)
self.assertEquals(self.concept1, mapping.from_concept)
self.assertEquals(self.concept2, mapping.to_concept)
self.assertEquals(self.source1, mapping.from_source)
self.assertEquals(self.source1.owner_name, mapping.from_source_owner)
self.assertEquals(self.source1.mnemonic, mapping.from_source_name)
self.assertEquals(self.source1, mapping.get_to_source())
self.assertEquals(self.source1.owner_name, mapping.to_source_owner)
self.assertEquals(self.concept2.mnemonic,
mapping.get_to_concept_code())
self.assertEquals(self.concept2.display_name,
mapping.get_to_concept_name())
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(
MappingVersion.objects.get(
versioned_object_id=mapping.id).id in source_version.mappings)
def test_persist_new_negative__same_mapping(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
mv = MappingVersion.objects.get(versioned_object_id=mapping.id)
self.assertTrue(mv.id in source_version.mappings)
# Repeat with same concepts
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping2',
)
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(1, len(errors))
self.assertEquals(1, len(errors))
self.assertTrue('__all__' in errors)
non_field_errors = errors['__all__']
self.assertEquals(1, len(non_field_errors))
self.assertTrue('already exists' in non_field_errors[0])
self.assertEquals(1, len(source_version.mappings))
def test_import_job_for_one_record(self):
stdout_stub = TestStream()
importer = ConceptsImporter(self.source1,
self.testfile,
'test',
stdout_stub,
TestStream(),
save_validation_errors=False)
importer.import_concepts(total=1)
self.assertTrue(
'Created new concept: 1 = Diagnosis' in stdout_stub.getvalue())
self.assertTrue(
'Finished importing concepts!' in stdout_stub.getvalue())
inserted_concept = Concept.objects.get(mnemonic='1')
self.assertEquals(inserted_concept.parent, self.source1)
inserted_concept_version = ConceptVersion.objects.get(
versioned_object_id=inserted_concept.id)
source_version_latest = SourceVersion.get_latest_version_of(
self.source1)
self.assertItemsEqual(source_version_latest.get_concept_ids(),
[inserted_concept_version.id])
def test_edit_mapping_make_new_version_positive(self):
mapping1 = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping1, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertEquals(
1,
len(MappingVersion.objects.filter(
versioned_object_id=mapping1.id)))
mapping1.map_type = 'BROADER_THAN'
Mapping.persist_changes(mapping1, self.user1)
self.assertEquals(
2,
len(MappingVersion.objects.filter(
versioned_object_id=mapping1.id)))
old_version = MappingVersion.objects.get(
versioned_object_id=mapping1.id, is_latest_version=False)
new_version = MappingVersion.objects.get(
versioned_object_id=mapping1.id, is_latest_version=True)
self.assertFalse(old_version.is_latest_version)
self.assertTrue(new_version.is_latest_version)
self.assertEquals(new_version.map_type, 'BROADER_THAN')
self.assertEquals(old_version.map_type, 'Same As')
def test_persist_new_negative__no_owner(self):
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='First',
names=[self.name],
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.concepts))
kwargs = {
'parent_resource': self.source1,
}
errors = Concept.persist_new(concept, None, **kwargs)
self.assertEquals(1, len(errors))
self.assertTrue('created_by' in errors)
self.assertFalse(Concept.objects.filter(mnemonic='concept1').exists())
self.assertEquals(0, concept.num_versions)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(0, len(source_version.concepts))
def test_update_mapping_with_invalid_record(self):
mapping = Mapping(parent=self.source1,
map_type='SAME-AS',
from_concept=self.concept3,
to_concept=self.concept1)
kwargs = {
'parent_resource': self.source1,
}
Mapping.persist_new(mapping, self.user1, **kwargs)
source_version = SourceVersion.get_latest_version_of(self.source1)
source_version.mappings = [mapping.id]
source_version.save()
stderr_stub = TestStream()
invalid_json_file = open(
'./integration_tests/fixtures/one_internal_invalid_mapping.json',
'rb')
importer = MappingsImporter(self.source1, invalid_json_file,
TestStream(), stderr_stub, 'test')
importer.import_mappings(total=1)
self.assertTrue(
"Must specify either 'to_concept' or 'to_source' & 'to_concept_code'. Cannot specify both."
in stderr_stub.getvalue())
def test_persist_new_positive__repeated_mnemonic(self):
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='First',
names=[self.name],
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.concepts))
kwargs = {
'parent_resource': self.source1,
}
errors = Concept.persist_new(concept, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertFalse(concept.retired)
self.assertEquals(self.name.name, concept.display_name)
self.assertEquals(self.name.locale, concept.display_locale)
self.assertEquals(self.source1.owner_name, concept.owner_name)
self.assertEquals(self.source1.owner_type, concept.owner_type)
self.assertEquals(self.source1.public_access, concept.public_access)
self.assertEquals(1, concept.num_versions)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertEquals(concept_version, concept_version.root_version)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.concepts))
self.assertTrue(concept_version.id in source_version.concepts)
# Repeat with same mnemonic, different parent
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source2,
concept_class='First',
names=[self.name],
)
source_version = SourceVersion.get_latest_version_of(self.source2)
self.assertEquals(0, len(source_version.concepts))
kwargs = {
'parent_resource': self.source2,
}
errors = Concept.persist_new(concept, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertFalse(concept.retired)
self.assertEquals(self.name.name, concept.display_name)
self.assertEquals(self.name.locale, concept.display_locale)
self.assertEquals(self.source2.parent_resource, concept.owner_name)
self.assertEquals(self.source2.owner_type, concept.owner_type)
self.assertEquals(self.source2.public_access, concept.public_access)
self.assertEquals(1, concept.num_versions)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertEquals(concept_version, concept_version.root_version)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.concepts))
self.assertTrue(concept_version.id in source_version.concepts)
def import_mappings(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs):
""" Main mapping importer loop """
logger.info('Import mappings to source...')
self.test_mode = test_mode
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_latest_version_of(self.source)
if new_version:
try:
new_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_version.seed_concepts()
new_version.seed_mappings()
new_version.full_clean()
new_version.save()
self.source_version = new_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
self.mapping_ids = set(self.source_version.mappings)
self.count = 0
concepts_for_uris = Concept.objects.filter(parent_id=self.source.id)
self.concepts_cache = dict((x.uri, x) for x in concepts_for_uris)
for line in self.mappings_file:
# Load the next JSON line
self.count += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_mapping(data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bars
if (self.count % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (self.count % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.mappings_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining unhandled mapping IDs:\n'
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
str_log = ','.join(str(el) for el in self.mapping_ids)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old mappings...\n'
self.stdout.write(str_log)
logger.info(str_log)
for mapping_id in self.mapping_ids:
try:
if self.remove_mapping(mapping_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated mapping: %s\n' % mapping_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing mappings!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
