def test_persist_new_positive__earlier_source_version(self): version1 = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(version1.mappings)) version2 = SourceVersion.for_base_object(self.source1, label='version2') version2.save() self.assertEquals(0, len(version2.mappings)) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.mappings)) mapping = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping1', ) kwargs = { 'parent_resource': self.source1, 'parent_resource_version': version1, } errors = Mapping.persist_new(mapping, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists()) mapping = Mapping.objects.get(external_id='mapping1') version1 = SourceVersion.objects.get(id=version1.id) self.assertEquals(1, len(version1.mappings)) self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in version1.mappings) version2 = SourceVersion.objects.get(id=version2.id) self.assertEquals(0, len(version2.mappings)) latest_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(latest_version.mappings))
def test_retire_positive(self): source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.concepts)) concept = Concept( mnemonic='concept1', created_by=self.user1, updated_by=self.user1, parent=self.source1, concept_class='First', names=[self.name], ) kwargs = { 'parent_resource': self.source1, } errors = Concept.persist_new(concept, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists()) self.assertFalse(concept.retired) self.assertEquals(1, concept.num_versions) concept_version = ConceptVersion.get_latest_version_of(concept) self.assertTrue(concept_version.is_latest_version) self.assertFalse(concept_version.retired) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(1, len(source_version.concepts)) self.assertEquals(concept_version.id, source_version.concepts[0]) errors = Concept.retire(concept, self.user1) self.assertFalse(errors) self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists()) self.assertTrue(concept.retired) self.assertEquals(2, concept.num_versions) previous_version = ConceptVersion.objects.get(id=concept_version.id) self.assertFalse(previous_version.is_latest_version) self.assertFalse(previous_version.retired) concept_version = ConceptVersion.get_latest_version_of(concept) self.assertTrue(concept_version.is_latest_version) self.assertTrue(concept_version.retired) self.assertEquals(self.user1.username, concept_version.version_created_by) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(1, len(source_version.concepts)) self.assertEquals(concept_version.id, source_version.concepts[0]) self.assertEquals( 1, ConceptVersion.objects.filter(versioned_object_id=concept.id, retired=True).count()) self.assertEquals( 1, ConceptVersion.objects.filter(versioned_object_id=concept.id, retired=False).count()) errors = Concept.retire(concept, self.user1) self.assertEquals(1, len(errors))
def test_persist_changes_positive__seed_from_previous_over_parent(self): version1 = SourceVersion.for_base_object(self.source1, 'version1') version1.concepts = [1] version1.full_clean() version1.save() version2 = SourceVersion.for_base_object(self.source1, 'version2') version2.concepts = [2] version2.full_clean() version2.save() self.assertIsNone(version2.previous_version) version3 = SourceVersion.for_base_object(self.source1, 'version3') version3.full_clean() version3.save() mnemonic = version3.mnemonic released = version3.released description = version3.description external_id = version3.external_id id = version3.id version3._parent_version_mnemonic = 'version2' version3._previous_version_mnemonic = 'version1' version3.mnemonic = "%s-prime" % mnemonic version3.released = not released version3.description = "%s-prime" % description version3.external_id = "%s-prime" % external_id errors = SourceVersion.persist_changes(version3) self.assertEquals(0, len(errors)) version3 = SourceVersion.objects.get(id=id) self.assertEquals(self.source1, version3.versioned_object) self.assertEquals(3, self.source1.num_versions) self.assertEquals(version3, SourceVersion.get_latest_version_of(self.source1)) self.assertEquals(version1, version3.previous_version) self.assertEquals(version2, version3.parent_version) self.assertEquals([], version3.concepts) self.assertNotEquals(mnemonic, version3.mnemonic) self.assertNotEquals(released, version3.released) self.assertNotEquals(description, version3.description) self.assertNotEquals(external_id, version3.external_id) errors = SourceVersion.persist_changes(version3, seed_concepts=True) self.assertEquals(0, len(errors)) version3 = SourceVersion.objects.get(id=id) self.assertEquals(self.source1, version3.versioned_object) self.assertEquals(3, self.source1.num_versions) self.assertEquals(version3, SourceVersion.get_latest_version_of(self.source1)) self.assertEquals(version2, version3.parent_version) self.assertEquals(version1, version3.previous_version) self.assertEquals([1], version3.concepts)
def test_persist_new_positive(self): mapping = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping1', ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.mappings)) kwargs = { 'parent_resource': self.source1, } errors = Mapping.persist_new(mapping, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists()) mapping = Mapping.objects.get(external_id='mapping1') self.assertEquals(self.source1.public_access, mapping.public_access) self.assertEquals('user1', mapping.created_by) self.assertEquals('user1', mapping.updated_by) self.assertEquals(self.source1, mapping.parent) self.assertEquals('Same As', mapping.map_type) self.assertEquals(self.concept1, mapping.from_concept) self.assertEquals(self.concept2, mapping.to_concept) self.assertEquals(self.source1, mapping.from_source) self.assertEquals(self.source1.owner_name, mapping.from_source_owner) self.assertEquals(self.source1.mnemonic, mapping.from_source_name) self.assertEquals(self.source1, mapping.get_to_source()) self.assertEquals(self.source1.owner_name, mapping.to_source_owner) self.assertEquals(self.concept2.mnemonic, mapping.get_to_concept_code()) self.assertEquals(self.concept2.display_name, mapping.get_to_concept_name()) source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.mappings)) self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
def test_edit_mapping_make_new_version_positive(self): mapping1 = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping1', ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.mappings)) kwargs = { 'parent_resource': self.source1, } errors = Mapping.persist_new(mapping1, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertEquals(1,len(MappingVersion.objects.filter(versioned_object_id=mapping1.id))) mapping1.map_type='BROADER_THAN' Mapping.persist_changes(mapping1, self.user1) self.assertEquals(2, len(MappingVersion.objects.filter(versioned_object_id=mapping1.id))) old_version = MappingVersion.objects.get(versioned_object_id=mapping1.id, is_latest_version=False) new_version= MappingVersion.objects.get(versioned_object_id=mapping1.id, is_latest_version=True) self.assertFalse(old_version.is_latest_version) self.assertTrue(new_version.is_latest_version) self.assertEquals(new_version.map_type,'BROADER_THAN') self.assertEquals(old_version.map_type,'Same As')
def test_persist_changes_negative__parent_version_is_self(self): version1 = SourceVersion.for_base_object(self.source1, 'version1', released=True) version1.full_clean() version1.save() mnemonic = version1.mnemonic released = version1.released description = version1.description external_id = version1.external_id id = version1.id version1._parent_version_mnemonic = mnemonic version1.released = not released version1.description = "%s-prime" % description version1.external_id = "%s-prime" % external_id errors = SourceVersion.persist_changes(version1) self.assertEquals(1, len(errors)) self.assertTrue('parentVersion' in errors) version1 = SourceVersion.objects.get(id=id) self.assertEquals(self.source1, version1.versioned_object) self.assertEquals(1, self.source1.num_versions) self.assertEquals(version1, SourceVersion.get_latest_version_of(self.source1)) self.assertEquals(mnemonic, version1.mnemonic) self.assertEquals(released, version1.released) self.assertEquals(description, version1.description) self.assertEquals(external_id, version1.external_id)
def test_persist_persist_changes_positive(self): mapping = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping1', ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.mappings)) kwargs = { 'parent_resource': self.source1, } Mapping.persist_new(mapping, self.user1, **kwargs) mapping = Mapping.objects.get(external_id='mapping1') to_concept = mapping.to_concept source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.mappings)) self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings) mapping.to_concept = self.concept3 errors = Mapping.persist_changes(mapping, self.user1) self.assertEquals(0, len(errors)) mapping = Mapping.objects.get(external_id='mapping1') self.assertEquals(self.concept3, mapping.to_concept) self.assertNotEquals(to_concept, mapping.to_concept) source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.mappings)) mv = MappingVersion.objects.filter(versioned_object_id=mapping.id) self.assertTrue(mv[1].id in source_version.mappings)
def test_import_job_for_change_in_data(self): stdout_stub = TestStream() stderr_stub = TestStream() mapping = Mapping( parent=self.source1, map_type="SAME-AS", from_concept=self.concept3, to_source=self.source1, to_concept_code="413532003", external_id="junk", ) kwargs = {"parent_resource": self.source1} Mapping.persist_new(mapping, self.user1, **kwargs) source_version = SourceVersion.get_latest_version_of(self.source1) source_version.mappings = [mapping.id] source_version.save() importer = MappingsImporter(self.source1, self.testfile, stdout_stub, stderr_stub, "test") importer.import_mappings(total=1) self.assertTrue("**** Processed 1 out of 1 mappings - 1 updated, ****" in stdout_stub.getvalue()) self.assertTrue(("Updated mapping with ID " + mapping.id) in stdout_stub.getvalue()) updated_mapping = Mapping.objects.get(to_concept_code="413532003") self.assertTrue(updated_mapping.retired) self.assertEquals(updated_mapping.external_id, "70279ABBBBBBBBBBBBBBBBBBBBBBBBBBBBBB")
def test_create_concept_reference_source_version__positive(self): source_version = SourceVersion.get_latest_version_of(self.source1) concept_reference = ConceptReference( created_by=self.user1, updated_by=self.user1, parent=self.source1, mnemonic='reference1', concept=self.concept1, source_version=source_version, ) concept_reference.full_clean() concept_reference.save() self.assertTrue(ConceptReference.objects.filter(mnemonic='reference1').exists()) self.assertEquals(self.concept1.concept_class, concept_reference.concept_class) self.assertEquals(self.concept1.datatype, concept_reference.data_type) self.assertEquals(self.concept1.parent, concept_reference.source) self.assertEquals(self.concept1.owner_name, concept_reference.owner_name) self.assertEquals(self.concept1.owner_type, concept_reference.owner_type) self.assertEquals(self.concept1.display_name, concept_reference.display_name) self.assertEquals(self.concept1.display_locale, concept_reference.display_locale) self.assertEquals( '/users/user1/sources/source1/%s/concepts/concept1/' % source_version.mnemonic, concept_reference.concept_reference_url) self.assertFalse(concept_reference.is_current_version)
def test_persist_new_positive(self): concept = Concept( mnemonic='concept1', created_by=self.user1, parent=self.source1, concept_class='First', names=[self.name], ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.concepts)) kwargs = { 'parent_resource': self.source1, } errors = Concept.persist_new(concept, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists()) self.assertFalse(concept.retired) self.assertEquals(self.name.name, concept.display_name) self.assertEquals(self.name.locale, concept.display_locale) self.assertEquals(self.source1.owner_name, concept.owner_name) self.assertEquals(self.source1.owner_type, concept.owner_type) self.assertEquals(self.source1.public_access, concept.public_access) self.assertEquals(1, concept.num_versions) concept_version = ConceptVersion.get_latest_version_of(concept) self.assertEquals(concept_version, concept_version.root_version) source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.concepts)) self.assertTrue(concept_version.id in source_version.concepts) self.assertEquals(concept_version.mnemonic, concept_version.id)
def test_persist_changes_positive__good_parent_version(self): version1 = SourceVersion.for_base_object(self.source1, 'version1') version1.full_clean() version1.save() version2 = SourceVersion.for_base_object(self.source1, 'version2') version2.full_clean() version2.save() self.assertIsNone(version2.parent_version) mnemonic = version2.mnemonic released = version2.released description = version2.description external_id = version2.external_id id = version2.id version2._parent_version_mnemonic = 'version1' version2.mnemonic = "%s-prime" % mnemonic version2.released = not released version2.description = "%s-prime" % description version2.external_id = "%s-prime" % external_id errors = SourceVersion.persist_changes(version2) self.assertEquals(0, len(errors)) version2 = SourceVersion.objects.get(id=id) self.assertEquals(self.source1, version2.versioned_object) self.assertEquals(2, self.source1.num_versions) self.assertEquals(version2, SourceVersion.get_latest_version_of(self.source1)) self.assertEquals(version1, version2.parent_version) self.assertNotEquals(mnemonic, version2.mnemonic) self.assertNotEquals(released, version2.released) self.assertNotEquals(description, version2.description) self.assertNotEquals(external_id, version2.external_id)
def test_persist_clone_positive(self): self.assertEquals(2, self.concept1.num_versions) self.assertEquals( self.concept_version, ConceptVersion.get_latest_version_of(self.concept1)) source_version = SourceVersion.get_latest_version_of(self.source1) source_version.update_concept_version(self.concept_version) self.assertEquals(1, len(source_version.concepts)) self.assertEquals(self.concept_version.id, source_version.concepts[0]) version2 = self.concept_version.clone() errors = ConceptVersion.persist_clone(version2, self.user1) self.assertEquals(0, len(errors)) self.assertEquals(3, self.concept1.num_versions) self.assertEquals(version2, ConceptVersion.get_latest_version_of(self.concept1)) self.assertEquals(self.concept_version.public_access, version2.public_access) self.assertEquals(self.concept_version, version2.previous_version) self.assertEquals(self.concept_version.root_version, version2.root_version) self.assertEquals(self.concept_version.external_id, version2.external_id) self.assertEquals(self.user1.username, version2.version_created_by) source_version.update_concept_version(version2) self.assertEquals(1, len(source_version.concepts)) self.assertEquals(version2.id, source_version.concepts[0])
def test_source_version_create_positive__child_and_subsequent_versions(self): version1 = SourceVersion( name='version1', mnemonic='version1', versioned_object=self.source1, released=True, created_by=self.user1, updated_by=self.user1, ) version1.full_clean() version1.save() self.assertTrue(SourceVersion.objects.filter( mnemonic='version1', versioned_object_type=ContentType.objects.get_for_model(Source), versioned_object_id=self.source1.id ).exists()) self.assertEquals(version1, SourceVersion.get_latest_version_of(self.source1)) self.assertEquals(1, self.source1.num_versions) version2 = SourceVersion( name='version2', mnemonic='version2', versioned_object=self.source1, parent_version=version1, created_by=self.user1, updated_by=self.user1, ) version2.full_clean() version2.save() self.assertTrue(SourceVersion.objects.filter( mnemonic='version2', versioned_object_type=ContentType.objects.get_for_model(Source), versioned_object_id=self.source1.id ).exists()) self.assertEquals(version1, version2.parent_version) self.assertEquals(version1.mnemonic, version2.parent_version_mnemonic) self.assertIsNone(version2.previous_version) self.assertIsNone(version2.previous_version_mnemonic) self.assertEquals(2, self.source1.num_versions) version3 = SourceVersion( name='version3', mnemonic='version3', versioned_object=self.source1, previous_version=version2, created_by=self.user1, updated_by=self.user1, ) version3.full_clean() version3.save() self.assertTrue(SourceVersion.objects.filter( mnemonic='version3', versioned_object_type=ContentType.objects.get_for_model(Source), versioned_object_id=self.source1.id ).exists()) self.assertEquals(version2, version3.previous_version) self.assertEquals(version2.mnemonic, version3.previous_version_mnemonic) self.assertIsNone(version3.parent_version) self.assertIsNone(version3.parent_version_mnemonic) self.assertEquals(3, self.source1.num_versions)
def test_persist_persist_changes_negative__no_updated_by(self): mapping = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping1', ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.mappings)) kwargs = { 'parent_resource': self.source1, } Mapping.persist_new(mapping, self.user1, **kwargs) mapping = Mapping.objects.get(external_id='mapping1') source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.mappings)) self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings) mapping.to_concept = self.concept3 errors = Mapping.persist_changes(mapping, None) self.assertEquals(1, len(errors)) self.assertTrue('updated_by' in errors) source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.mappings)) self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
def test_import_job_for_change_in_data(self): stdout_stub = TestStream() stderr_stub = TestStream() mapping = Mapping( parent=self.source1, map_type='SAME-AS', from_concept=self.concept3, to_source=self.source1, to_concept_code='413532003', external_id='junk' ) kwargs = { 'parent_resource': self.source1, } Mapping.persist_new(mapping, self.user1, **kwargs) source_version = SourceVersion.get_latest_version_of(self.source1) source_version.mappings = [mapping.id] source_version.save() importer = MappingsImporter(self.source1, self.testfile, stdout_stub, stderr_stub, 'test') importer.import_mappings(total=1) self.assertTrue('mappings of 1 1 - 1 updated' in stdout_stub.getvalue()) self.assertTrue(('Updated mapping with ID ' + mapping.id) in stdout_stub.getvalue()) updated_mapping = Mapping.objects.get(to_concept_code='413532003') self.assertTrue(updated_mapping.retired) self.assertEquals(updated_mapping.external_id, '70279ABBBBBBBBBBBBBBBBBBBBBBBBBBBBBB')
def test_source_version_create_positive(self): source_version = SourceVersion( name='version1', mnemonic='version1', versioned_object=self.source1, released=True, created_by=self.user1, updated_by=self.user1, ) source_version.full_clean() source_version.save() self.assertTrue(SourceVersion.objects.filter( mnemonic='version1', versioned_object_type=ContentType.objects.get_for_model(Source), versioned_object_id=self.source1.id ).exists()) self.assertIsNone(source_version.previous_version) self.assertIsNone(source_version.previous_version_mnemonic) self.assertIsNone(source_version.parent_version) self.assertIsNone(source_version.parent_version_mnemonic) self.assertEquals(self.org1.mnemonic, source_version.parent_resource) self.assertEquals(self.org1.resource_type, source_version.parent_resource_type) self.assertEquals(source_version, SourceVersion.get_latest_version_of(self.source1)) self.assertEquals(1, self.source1.num_versions)
def test_import_job_for_change_in_data(self): stdout_stub = TestStream() source_version_latest = SourceVersion.get_latest_version_of(self.source1) concept = Concept( mnemonic='1', created_by=self.user1, updated_by=self.user1, parent=self.source1, concept_class='Diagnosis', external_id='1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA', names=[self.name]) kwargs = { 'parent_resource': self.source1, } Concept.persist_new(concept, self.user1, **kwargs) importer = ConceptsImporter(self.source1, self.testfile, 'test', stdout_stub, TestStream()) importer.import_concepts(total=1) all_concept_versions = ConceptVersion.objects.all() self.assertEquals(len(all_concept_versions), 2) concept = Concept.objects.get(mnemonic='1') latest_concept_version = [version for version in all_concept_versions if version.previous_version][0] self.assertEquals(len(latest_concept_version.names), 4) self.assertTrue(('Updated concept, replacing version ID ' + latest_concept_version.previous_version.id) in stdout_stub.getvalue()) self.assertTrue('concepts of 1 1 - 1 updated' in stdout_stub.getvalue())
def retrieve(self, request, *args, **kwargs): super(SourceRetrieveUpdateDestroyView, self).retrieve(request, *args, **kwargs) self.object = self.get_object() serializer = self.get_serializer(self.object) data = serializer.data source_version = SourceVersion.get_latest_version_of(self.object) self.includeConceptsAndMappings(request, data, source_version) return Response(data)
def initialize(self, request, path_info_segment, **kwargs): super(MappingBaseView, self).initialize(request, path_info_segment, **kwargs) if self.parent_resource: if hasattr(self.parent_resource, 'versioned_object'): self.parent_resource_version = self.parent_resource self.parent_resource = self.parent_resource.versioned_object else: self.parent_resource_version = SourceVersion.get_latest_version_of(self.parent_resource)
def __init__(self, source, concepts_file, user, output_stream, error_stream): """ Initialize mapping importer """ self.source = source self.concepts_file = concepts_file self.stdout = output_stream self.stderr = error_stream self.user = user # Retrieve latest source version and, if specified, create a new one self.source_version = SourceVersion.get_latest_version_of(self.source)
def test_persist_new_positive__same_mapping_different_source(self): mapping = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping1', ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.mappings)) kwargs = { 'parent_resource': self.source1, } errors = Mapping.persist_new(mapping, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists()) mapping = Mapping.objects.get(external_id='mapping1') source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.mappings)) self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings) # Repeat with same concepts mapping = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping2', ) kwargs = { 'parent_resource': self.source2, } source_version = SourceVersion.get_latest_version_of(self.source2) self.assertEquals(0, len(source_version.mappings)) errors = Mapping.persist_new(mapping, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertTrue(Mapping.objects.filter(external_id='mapping2').exists()) mapping = Mapping.objects.get(external_id='mapping2') source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.mappings)) self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
def test_persist_new_positive(self): kwargs = { 'parent_resource': self.userprofile1 } errors = Source.persist_new(self.new_source, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertTrue(Source.objects.filter(name='source1').exists()) source = Source.objects.get(name='source1') self.assertTrue(SourceVersion.objects.filter(versioned_object_id=source.id)) source_version = SourceVersion.objects.get(versioned_object_id=source.id) self.assertEquals(1, source.num_versions) self.assertEquals(source_version, SourceVersion.get_latest_version_of(source))
def test_import_job_for_one_record(self): stdout_stub = TestStream() importer = ConceptsImporter(self.source1, self.testfile, 'test', stdout_stub, TestStream()) importer.import_concepts(total=1) self.assertTrue('Created new concept: 1 = Diagnosis' in stdout_stub.getvalue()) self.assertTrue('Finished importing concepts!' in stdout_stub.getvalue()) inserted_concept = Concept.objects.get(mnemonic='1') self.assertEquals(inserted_concept.parent, self.source1) inserted_concept_version = ConceptVersion.objects.get(versioned_object_id=inserted_concept.id) source_version_latest = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(source_version_latest.concepts, [inserted_concept_version.id])
def test_create_concept_reference_concept_and_source_versions__negative(self): with self.assertRaises(ValidationError): concept_reference = ConceptReference( created_by=self.user1, updated_by=self.user1, parent=self.source1, mnemonic='reference1', concept=self.concept1, concept_version=self.version1, source_version=SourceVersion.get_latest_version_of(self.source1) ) concept_reference.full_clean() concept_reference.save()
def test_import_job_for_one_record(self): stdout_stub = TestStream() stderr_stub = TestStream() importer = MappingsImporter(self.source1, self.testfile, stdout_stub, stderr_stub, 'test') importer.import_mappings(total=1) self.assertTrue('Created new mapping:' in stdout_stub.getvalue()) self.assertTrue('/users/user1/sources/source1/:413532003' in stdout_stub.getvalue()) inserted_mapping = Mapping.objects.get(to_concept_code='413532003') self.assertEquals(inserted_mapping.to_source, self.source1) self.assertEquals(inserted_mapping.from_source, self.source2) mapping_ids = SourceVersion.get_latest_version_of(self.source1).mappings mapping_version = MappingVersion.objects.get(versioned_object_id=inserted_mapping.id, is_latest_version=True) self.assertEquals(mapping_ids[0], mapping_version.id)
def __init__(self, source, concepts_file, user, output_stream, error_stream, save_validation_errors=True, validation_logger=None): """ Initialize mapping importer """ self.source = source self.concepts_file = concepts_file self.stdout = output_stream self.stderr = error_stream self.user = user # Retrieve latest source version and, if specified, create a new one self.source_version = SourceVersion.get_latest_version_of(self.source) self.validation_logger = validation_logger self.save_validation_errors = save_validation_errors if self.save_validation_errors and self.validation_logger is None: self.validation_logger = ValidationLogger()
def test_get_latest_version_of(self): latest_version_identity = 'version' expected_latest_source_version = SourceVersion( name=latest_version_identity, mnemonic=latest_version_identity, versioned_object=self.source, released=True, created_by=self.user, updated_by=self.user, ) expected_latest_source_version.full_clean() expected_latest_source_version.save() self.assertEquals(SourceVersion.get_latest_version_of(self.source).name, latest_version_identity)
def retrieve(self, request, *args, **kwargs): super(SourceRetrieveUpdateDestroyView, self).retrieve(request, *args, **kwargs) self.object = self.get_object() serializer = self.get_serializer(self.object) data = serializer.data source_version = None offset = request.QUERY_PARAMS.get(OFFSET_PARAM, DEFAULT_OFFSET) try: offset = int(offset) except ValueError: offset = DEFAULT_OFFSET limit = settings.REST_FRAMEWORK.get('MAX_PAGINATE_BY', self.paginate_by) include_retired = False include_concepts = request.QUERY_PARAMS.get(INCLUDE_CONCEPTS_PARAM, False) include_mappings = request.QUERY_PARAMS.get(INCLUDE_MAPPINGS_PARAM, False) updated_since = None if include_concepts or include_mappings: source_version = SourceVersion.get_latest_version_of(self.object) paginate_by = self.get_paginate_by(EmptyQuerySet()) if paginate_by: limit = min(limit, paginate_by) include_retired = request.QUERY_PARAMS.get(INCLUDE_RETIRED_PARAM, False) updated_since = parse_updated_since_param(request) if include_concepts: source_version_concepts = source_version.concepts queryset = ConceptVersion.objects.filter(is_active=True) if not include_retired: queryset = queryset.filter(~Q(retired=True)) if updated_since: queryset = queryset.filter(updated_at__gte=updated_since) queryset = queryset.filter(id__in=source_version_concepts) queryset = queryset[offset:offset+limit] serializer = ConceptVersionDetailSerializer(queryset, many=True) data['concepts'] = serializer.data if include_mappings: all_children = source_version.mappings queryset = Mapping.objects.filter(is_active=True) if not include_retired: queryset = queryset.filter(~Q(retired=True)) queryset = queryset.filter(id__in=all_children) if updated_since: queryset = queryset.filter(updated_at__gte=updated_since) queryset = queryset[offset:offset+limit] serializer = MappingDetailSerializer(queryset, many=True) data['mappings'] = serializer.data return Response(data)
def initialize(self, request, path_info_segment, **kwargs): self.include_retired = request.QUERY_PARAMS.get(INCLUDE_RETIRED_PARAM, False) include_inverse_param = request.GET.get(INCLUDE_INVERSE_MAPPINGS_PARAM, 'false') self.include_inverse_mappings = 'true' == include_inverse_param parent_path_info = self.get_parent_in_path(path_info_segment, levels=1) if parent_path_info and '/' != parent_path_info: self.concept = self.get_object_for_path(parent_path_info, self.request) parent_path_info = self.get_parent_in_path(parent_path_info, levels=2) if parent_path_info and '/' != parent_path_info: self.parent_resource = self.get_object_for_path(parent_path_info, self.request) if hasattr(self.parent_resource, 'versioned_object'): self.parent_resource_version = self.parent_resource self.parent_resource = self.parent_resource.versioned_object else: self.parent_resource_version = SourceVersion.get_latest_version_of(self.parent_resource)
def test_update_mapping_with_invalid_record(self): mapping = Mapping(parent=self.source1, map_type="SAME-AS", from_concept=self.concept3, to_concept=self.concept1) kwargs = {"parent_resource": self.source1} Mapping.persist_new(mapping, self.user1, **kwargs) source_version = SourceVersion.get_latest_version_of(self.source1) source_version.mappings = [mapping.id] source_version.save() stderr_stub = TestStream() invalid_json_file = open("./integration_tests/fixtures/one_internal_invalid_mapping.json", "rb") importer = MappingsImporter(self.source1, invalid_json_file, TestStream(), stderr_stub, "test") importer.import_mappings(total=1) self.assertTrue( "Must specify either 'to_concept' or 'to_source' & 'to_concept_code'. Cannot specify both." in stderr_stub.getvalue() )
def test_persist_new_negative__no_parent(self): mapping = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping1', ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.mappings)) kwargs = {} errors = Mapping.persist_new(mapping, self.user1, **kwargs) self.assertEquals(1, len(errors)) self.assertTrue('non_field_errors' in errors) non_field_errors = errors['non_field_errors'] self.assertEquals(1, len(non_field_errors)) self.assertTrue('parent' in non_field_errors[0]) self.assertFalse( Mapping.objects.filter(external_id='mapping1').exists()) source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(0, len(source_version.mappings))
def initialize(self, request, path_info_segment, **kwargs): self.include_retired = request.QUERY_PARAMS.get( INCLUDE_RETIRED_PARAM, False) include_inverse_param = request.GET.get(INCLUDE_INVERSE_MAPPINGS_PARAM, 'false') self.include_inverse_mappings = 'true' == include_inverse_param parent_path_info = self.get_parent_in_path(path_info_segment, levels=1) if parent_path_info and '/' != parent_path_info: self.concept = self.get_object_for_path(parent_path_info, self.request) parent_path_info = self.get_parent_in_path(parent_path_info, levels=2) if parent_path_info and '/' != parent_path_info: self.parent_resource = self.get_object_for_path( parent_path_info, self.request) if hasattr(self.parent_resource, 'versioned_object'): self.parent_resource_version = self.parent_resource self.parent_resource = self.parent_resource.versioned_object else: self.parent_resource_version = SourceVersion.get_latest_version_of( self.parent_resource)
def test_persist_new_positive(self): mapping = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping1', ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.mappings)) kwargs = { 'parent_resource': self.source1, } errors = Mapping.persist_new(mapping, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertTrue( Mapping.objects.filter(external_id='mapping1').exists()) mapping = Mapping.objects.get(external_id='mapping1') self.assertEquals(self.source1.public_access, mapping.public_access) self.assertEquals('user1', mapping.created_by) self.assertEquals('user1', mapping.updated_by) self.assertEquals(self.source1, mapping.parent) self.assertEquals('Same As', mapping.map_type) self.assertEquals(self.concept1, mapping.from_concept) self.assertEquals(self.concept2, mapping.to_concept) self.assertEquals(self.source1, mapping.from_source) self.assertEquals(self.source1.owner_name, mapping.from_source_owner) self.assertEquals(self.source1.mnemonic, mapping.from_source_name) self.assertEquals(self.source1, mapping.get_to_source()) self.assertEquals(self.source1.owner_name, mapping.to_source_owner) self.assertEquals(self.concept2.mnemonic, mapping.get_to_concept_code()) self.assertEquals(self.concept2.display_name, mapping.get_to_concept_name()) source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.mappings)) self.assertTrue( MappingVersion.objects.get( versioned_object_id=mapping.id).id in source_version.mappings)
def test_persist_new_negative__same_mapping(self): mapping = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping1', ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.mappings)) kwargs = { 'parent_resource': self.source1, } errors = Mapping.persist_new(mapping, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists()) mapping = Mapping.objects.get(external_id='mapping1') source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.mappings)) mv = MappingVersion.objects.get(versioned_object_id=mapping.id) self.assertTrue(mv.id in source_version.mappings) # Repeat with same concepts mapping = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping2', ) kwargs = { 'parent_resource': self.source1, } errors = Mapping.persist_new(mapping, self.user1, **kwargs) self.assertEquals(1, len(errors)) self.assertEquals(1, len(errors)) self.assertTrue('__all__' in errors) non_field_errors = errors['__all__'] self.assertEquals(1, len(non_field_errors)) self.assertTrue('already exists' in non_field_errors[0]) self.assertEquals(1, len(source_version.mappings))
def test_import_job_for_one_record(self): stdout_stub = TestStream() importer = ConceptsImporter(self.source1, self.testfile, 'test', stdout_stub, TestStream(), save_validation_errors=False) importer.import_concepts(total=1) self.assertTrue( 'Created new concept: 1 = Diagnosis' in stdout_stub.getvalue()) self.assertTrue( 'Finished importing concepts!' in stdout_stub.getvalue()) inserted_concept = Concept.objects.get(mnemonic='1') self.assertEquals(inserted_concept.parent, self.source1) inserted_concept_version = ConceptVersion.objects.get( versioned_object_id=inserted_concept.id) source_version_latest = SourceVersion.get_latest_version_of( self.source1) self.assertItemsEqual(source_version_latest.get_concept_ids(), [inserted_concept_version.id])
def test_edit_mapping_make_new_version_positive(self): mapping1 = Mapping( map_type='Same As', from_concept=self.concept1, to_concept=self.concept2, external_id='mapping1', ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.mappings)) kwargs = { 'parent_resource': self.source1, } errors = Mapping.persist_new(mapping1, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertEquals( 1, len(MappingVersion.objects.filter( versioned_object_id=mapping1.id))) mapping1.map_type = 'BROADER_THAN' Mapping.persist_changes(mapping1, self.user1) self.assertEquals( 2, len(MappingVersion.objects.filter( versioned_object_id=mapping1.id))) old_version = MappingVersion.objects.get( versioned_object_id=mapping1.id, is_latest_version=False) new_version = MappingVersion.objects.get( versioned_object_id=mapping1.id, is_latest_version=True) self.assertFalse(old_version.is_latest_version) self.assertTrue(new_version.is_latest_version) self.assertEquals(new_version.map_type, 'BROADER_THAN') self.assertEquals(old_version.map_type, 'Same As')
def test_persist_new_negative__no_owner(self): concept = Concept( mnemonic='concept1', created_by=self.user1, updated_by=self.user1, parent=self.source1, concept_class='First', names=[self.name], ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.concepts)) kwargs = { 'parent_resource': self.source1, } errors = Concept.persist_new(concept, None, **kwargs) self.assertEquals(1, len(errors)) self.assertTrue('created_by' in errors) self.assertFalse(Concept.objects.filter(mnemonic='concept1').exists()) self.assertEquals(0, concept.num_versions) source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(0, len(source_version.concepts))
def test_update_mapping_with_invalid_record(self): mapping = Mapping(parent=self.source1, map_type='SAME-AS', from_concept=self.concept3, to_concept=self.concept1) kwargs = { 'parent_resource': self.source1, } Mapping.persist_new(mapping, self.user1, **kwargs) source_version = SourceVersion.get_latest_version_of(self.source1) source_version.mappings = [mapping.id] source_version.save() stderr_stub = TestStream() invalid_json_file = open( './integration_tests/fixtures/one_internal_invalid_mapping.json', 'rb') importer = MappingsImporter(self.source1, invalid_json_file, TestStream(), stderr_stub, 'test') importer.import_mappings(total=1) self.assertTrue( "Must specify either 'to_concept' or 'to_source' & 'to_concept_code'. Cannot specify both." in stderr_stub.getvalue())
def test_persist_new_positive__repeated_mnemonic(self): concept = Concept( mnemonic='concept1', created_by=self.user1, updated_by=self.user1, parent=self.source1, concept_class='First', names=[self.name], ) source_version = SourceVersion.get_latest_version_of(self.source1) self.assertEquals(0, len(source_version.concepts)) kwargs = { 'parent_resource': self.source1, } errors = Concept.persist_new(concept, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists()) self.assertFalse(concept.retired) self.assertEquals(self.name.name, concept.display_name) self.assertEquals(self.name.locale, concept.display_locale) self.assertEquals(self.source1.owner_name, concept.owner_name) self.assertEquals(self.source1.owner_type, concept.owner_type) self.assertEquals(self.source1.public_access, concept.public_access) self.assertEquals(1, concept.num_versions) concept_version = ConceptVersion.get_latest_version_of(concept) self.assertEquals(concept_version, concept_version.root_version) source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.concepts)) self.assertTrue(concept_version.id in source_version.concepts) # Repeat with same mnemonic, different parent concept = Concept( mnemonic='concept1', created_by=self.user1, updated_by=self.user1, parent=self.source2, concept_class='First', names=[self.name], ) source_version = SourceVersion.get_latest_version_of(self.source2) self.assertEquals(0, len(source_version.concepts)) kwargs = { 'parent_resource': self.source2, } errors = Concept.persist_new(concept, self.user1, **kwargs) self.assertEquals(0, len(errors)) self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists()) self.assertFalse(concept.retired) self.assertEquals(self.name.name, concept.display_name) self.assertEquals(self.name.locale, concept.display_locale) self.assertEquals(self.source2.parent_resource, concept.owner_name) self.assertEquals(self.source2.owner_type, concept.owner_type) self.assertEquals(self.source2.public_access, concept.public_access) self.assertEquals(1, concept.num_versions) concept_version = ConceptVersion.get_latest_version_of(concept) self.assertEquals(concept_version, concept_version.root_version) source_version = SourceVersion.objects.get(id=source_version.id) self.assertEquals(1, len(source_version.concepts)) self.assertTrue(concept_version.id in source_version.concepts)
def import_mappings(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs): """ Main mapping importer loop """ logger.info('Import mappings to source...') self.test_mode = test_mode # Retrieve latest source version and, if specified, create a new one self.source_version = SourceVersion.get_latest_version_of(self.source) if new_version: try: new_version = SourceVersion.for_base_object( self.source, new_version, previous_version=self.source_version) new_version.seed_concepts() new_version.seed_mappings() new_version.full_clean() new_version.save() self.source_version = new_version except Exception as exc: raise CommandError('Failed to create new source version due to %s' % exc.args[0]) # Load the JSON file line by line and import each line self.mapping_ids = set(self.source_version.mappings) self.count = 0 concepts_for_uris = Concept.objects.filter(parent_id=self.source.id) self.concepts_cache = dict((x.uri, x) for x in concepts_for_uris) for line in self.mappings_file: # Load the next JSON line self.count += 1 data = None try: data = json.loads(line) except ValueError as exc: str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line) self.stderr.write(str_log) logger.warning(str_log) self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP) # Process the import for the current JSON line if data: try: update_action = self.handle_mapping(data) self.count_action(update_action) except IllegalInputException as exc: str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data) self.stderr.write(str_log) logger.warning(str_log) self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP) except InvalidStateException as exc: str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data) self.stderr.write(str_log) logger.warning(str_log) self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP) # Simple progress bars if (self.count % 10) == 0: str_log = ImportActionHelper.get_progress_descriptor( 'mappings', self.count, total, self.action_count) self.stdout.write(str_log, ending='\r') self.stdout.flush() if (self.count % 1000) == 0: logger.info(str_log) # Done with the input file, so close it self.mappings_file.close() # Import complete - display final progress bar str_log = ImportActionHelper.get_progress_descriptor( 'mappings', self.count, total, self.action_count) self.stdout.write(str_log, ending='\r') self.stdout.flush() logger.info(str_log) # Log remaining unhandled IDs str_log = 'Remaining unhandled mapping IDs:\n' self.stdout.write(str_log, ending='\r') logger.info(str_log) str_log = ','.join(str(el) for el in self.mapping_ids) self.stdout.write(str_log, ending='\r') self.stdout.flush() logger.info(str_log) # Deactivate old records if deactivate_old_records: str_log = 'Deactivating old mappings...\n' self.stdout.write(str_log) logger.info(str_log) for mapping_id in self.mapping_ids: try: if self.remove_mapping(mapping_id): self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE) # Log the mapping deactivation str_log = 'Deactivated mapping: %s\n' % mapping_id self.stdout.write(str_log) logger.info(str_log) except InvalidStateException as exc: str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0]) self.stderr.write(str_log) logger.warning(str_log) else: str_log = 'Skipping deactivation loop...\n' self.stdout.write(str_log) logger.info(str_log) # Display final summary str_log = 'Finished importing mappings!\n' self.stdout.write(str_log) logger.info(str_log) str_log = ImportActionHelper.get_progress_descriptor( 'mappings', self.count, total, self.action_count) self.stdout.write(str_log, ending='\r') logger.info(str_log)