def test_source_version_create_negative__same_mnemonic(self):
source_version = SourceVersion(
name='version1',
mnemonic='version1',
versioned_object=self.source1,
created_by=self.user1,
updated_by=self.user1,
)
source_version.full_clean()
source_version.save()
self.assertEquals(1, self.source1.num_versions)
with self.assertRaises(ValidationError):
source_version = SourceVersion(
name='version1',
mnemonic='version1',
versioned_object=self.source1,
created_by=self.user1,
updated_by=self.user1,
)
source_version.full_clean()
source_version.save()
self.assertEquals(1, self.source1.num_versions)
def test_source_version_create_positive__same_mnemonic(self):
source_version = SourceVersion(
name='version1',
mnemonic='version1',
versioned_object=self.source1,
created_by=self.user1,
updated_by=self.user1,
)
source_version.full_clean()
source_version.save()
self.assertTrue(SourceVersion.objects.filter(
mnemonic='version1',
versioned_object_type=ContentType.objects.get_for_model(Source),
versioned_object_id=self.source1.id
).exists())
self.assertEquals(1, self.source1.num_versions)
source_version = SourceVersion(
name='version1',
mnemonic='version1',
versioned_object=self.source2,
created_by=self.user1,
updated_by=self.user1,
)
source_version.full_clean()
source_version.save()
self.assertTrue(SourceVersion.objects.filter(
mnemonic='version1',
versioned_object_type=ContentType.objects.get_for_model(Source),
versioned_object_id=self.source2.id
).exists())
self.assertEquals(1, self.source2.num_versions)
def test_persist_new_positive__earlier_source_version(self):
version1 = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(version1.mappings))
version2 = SourceVersion.for_base_object(self.source1, label='version2')
version2.save()
self.assertEquals(0, len(version2.mappings))
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
kwargs = {
'parent_resource': self.source1,
'parent_resource_version': version1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
version1 = SourceVersion.objects.get(id=version1.id)
self.assertEquals(1, len(version1.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in version1.mappings)
version2 = SourceVersion.objects.get(id=version2.id)
self.assertEquals(0, len(version2.mappings))
latest_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(latest_version.mappings))
def test_persist_changes_positive__good_parent_version(self):
version1 = SourceVersion.for_base_object(self.source1, 'version1')
version1.full_clean()
version1.save()
version2 = SourceVersion.for_base_object(self.source1, 'version2')
version2.full_clean()
version2.save()
self.assertIsNone(version2.parent_version)
mnemonic = version2.mnemonic
released = version2.released
description = version2.description
external_id = version2.external_id
id = version2.id
version2._parent_version_mnemonic = 'version1'
version2.mnemonic = "%s-prime" % mnemonic
version2.released = not released
version2.description = "%s-prime" % description
version2.external_id = "%s-prime" % external_id
errors = SourceVersion.persist_changes(version2)
self.assertEquals(0, len(errors))
version2 = SourceVersion.objects.get(id=id)
self.assertEquals(self.source1, version2.versioned_object)
self.assertEquals(2, self.source1.num_versions)
self.assertEquals(version2, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(version1, version2.parent_version)
self.assertNotEquals(mnemonic, version2.mnemonic)
self.assertNotEquals(released, version2.released)
self.assertNotEquals(description, version2.description)
self.assertNotEquals(external_id, version2.external_id)
def test_persist_changes_negative__parent_version_is_self(self):
version1 = SourceVersion.for_base_object(self.source1, 'version1', released=True)
version1.full_clean()
version1.save()
mnemonic = version1.mnemonic
released = version1.released
description = version1.description
external_id = version1.external_id
id = version1.id
version1._parent_version_mnemonic = mnemonic
version1.released = not released
version1.description = "%s-prime" % description
version1.external_id = "%s-prime" % external_id
errors = SourceVersion.persist_changes(version1)
self.assertEquals(1, len(errors))
self.assertTrue('parentVersion' in errors)
version1 = SourceVersion.objects.get(id=id)
self.assertEquals(self.source1, version1.versioned_object)
self.assertEquals(1, self.source1.num_versions)
self.assertEquals(version1, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(mnemonic, version1.mnemonic)
self.assertEquals(released, version1.released)
self.assertEquals(description, version1.description)
self.assertEquals(external_id, version1.external_id)
def test_retire_positive(self):
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.concepts))
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='First',
names=[self.name],
)
kwargs = {
'parent_resource': self.source1,
}
errors = Concept.persist_new(concept, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertFalse(concept.retired)
self.assertEquals(1, concept.num_versions)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertTrue(concept_version.is_latest_version)
self.assertFalse(concept_version.retired)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(1, len(source_version.concepts))
self.assertEquals(concept_version.id, source_version.concepts[0])
errors = Concept.retire(concept, self.user1)
self.assertFalse(errors)
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertTrue(concept.retired)
self.assertEquals(2, concept.num_versions)
previous_version = ConceptVersion.objects.get(id=concept_version.id)
self.assertFalse(previous_version.is_latest_version)
self.assertFalse(previous_version.retired)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertTrue(concept_version.is_latest_version)
self.assertTrue(concept_version.retired)
self.assertEquals(self.user1.username, concept_version.version_created_by)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(1, len(source_version.concepts))
self.assertEquals(concept_version.id, source_version.concepts[0])
self.assertEquals(
1, ConceptVersion.objects.filter(versioned_object_id=concept.id, retired=True).count())
self.assertEquals(
1, ConceptVersion.objects.filter(versioned_object_id=concept.id, retired=False).count())
errors = Concept.retire(concept, self.user1)
self.assertEquals(1, len(errors))
def test_source_version_create_negative__no_source(self):
with self.assertRaises(ValidationError):
source_version = SourceVersion(
mnemonic='version1',
name='version1',
created_by=self.user1,
updated_by=self.user1,
)
source_version.full_clean()
source_version.save()
self.assertEquals(0, self.source1.num_versions)
def test_persist_new_positive(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
self.assertEquals(self.source1.public_access, mapping.public_access)
self.assertEquals('user1', mapping.created_by)
self.assertEquals('user1', mapping.updated_by)
self.assertEquals(self.source1, mapping.parent)
self.assertEquals('Same As', mapping.map_type)
self.assertEquals(self.concept1, mapping.from_concept)
self.assertEquals(self.concept2, mapping.to_concept)
self.assertEquals(self.source1, mapping.from_source)
self.assertEquals(self.source1.owner_name, mapping.from_source_owner)
self.assertEquals(self.source1.mnemonic, mapping.from_source_name)
self.assertEquals(self.source1, mapping.get_to_source())
self.assertEquals(self.source1.owner_name, mapping.to_source_owner)
self.assertEquals(self.concept2.mnemonic, mapping.get_to_concept_code())
self.assertEquals(self.concept2.display_name, mapping.get_to_concept_name())
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
def test_edit_mapping_make_new_version_positive(self):
mapping1 = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping1, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertEquals(1,len(MappingVersion.objects.filter(versioned_object_id=mapping1.id)))
mapping1.map_type='BROADER_THAN'
Mapping.persist_changes(mapping1, self.user1)
self.assertEquals(2, len(MappingVersion.objects.filter(versioned_object_id=mapping1.id)))
old_version = MappingVersion.objects.get(versioned_object_id=mapping1.id, is_latest_version=False)
new_version= MappingVersion.objects.get(versioned_object_id=mapping1.id, is_latest_version=True)
self.assertFalse(old_version.is_latest_version)
self.assertTrue(new_version.is_latest_version)
self.assertEquals(new_version.map_type,'BROADER_THAN')
self.assertEquals(old_version.map_type,'Same As')
def test_persist_persist_changes_negative__no_updated_by(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
Mapping.persist_new(mapping, self.user1, **kwargs)
mapping = Mapping.objects.get(external_id='mapping1')
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
mapping.to_concept = self.concept3
errors = Mapping.persist_changes(mapping, None)
self.assertEquals(1, len(errors))
self.assertTrue('updated_by' in errors)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
def test_persist_persist_changes_positive(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
Mapping.persist_new(mapping, self.user1, **kwargs)
mapping = Mapping.objects.get(external_id='mapping1')
to_concept = mapping.to_concept
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
mapping.to_concept = self.concept3
errors = Mapping.persist_changes(mapping, self.user1)
self.assertEquals(0, len(errors))
mapping = Mapping.objects.get(external_id='mapping1')
self.assertEquals(self.concept3, mapping.to_concept)
self.assertNotEquals(to_concept, mapping.to_concept)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
mv = MappingVersion.objects.filter(versioned_object_id=mapping.id)
self.assertTrue(mv[1].id in source_version.mappings)
def persist_changes(cls, obj, updated_by, update_comment=None, **kwargs):
errors = dict()
obj.updated_by = updated_by
try:
obj.full_clean()
except ValidationError as e:
errors.update(e.message_dict)
return errors
persisted = False
try:
source_version = SourceVersion.get_head_of(obj.parent)
obj.save(**kwargs)
prev_latest_version = MappingVersion.objects.get(versioned_object_id=obj.id, is_latest_version=True)
prev_latest_version.is_latest_version = False
prev_latest_version.save()
new_latest_version = MappingVersion.for_mapping(obj)
new_latest_version.previous_version = prev_latest_version
new_latest_version.update_comment = update_comment
new_latest_version.mnemonic = int(prev_latest_version.mnemonic) + 1
new_latest_version.save()
source_version.update_mapping_version(new_latest_version)
persisted = True
finally:
if not persisted:
errors['non_field_errors'] = ["Failed to persist mapping."]
return errors
def test_import_job_for_change_in_data(self):
stdout_stub = TestStream()
stderr_stub = TestStream()
mapping = Mapping(
parent=self.source1,
map_type='SAME-AS',
from_concept=self.concept3,
to_source=self.source1,
to_concept_code='413532003',
external_id='junk'
)
kwargs = {
'parent_resource': self.source1,
}
Mapping.persist_new(mapping, self.user1, **kwargs)
source_version = SourceVersion.get_latest_version_of(self.source1)
source_version.mappings = [mapping.id]
source_version.save()
importer = MappingsImporter(self.source1, self.testfile, stdout_stub, stderr_stub, 'test')
importer.import_mappings(total=1)
self.assertTrue('mappings of 1 1 - 1 updated' in stdout_stub.getvalue())
self.assertTrue(('Updated mapping with ID ' + mapping.id) in stdout_stub.getvalue())
updated_mapping = Mapping.objects.get(to_concept_code='413532003')
self.assertTrue(updated_mapping.retired)
self.assertEquals(updated_mapping.external_id, '70279ABBBBBBBBBBBBBBBBBBBBBBBBBBBBBB')
def test_create_concept_reference_source_version__positive(self):
source_version = SourceVersion.get_latest_version_of(self.source1)
concept_reference = ConceptReference(
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
mnemonic='reference1',
concept=self.concept1,
source_version=source_version,
)
concept_reference.full_clean()
concept_reference.save()
self.assertTrue(ConceptReference.objects.filter(mnemonic='reference1').exists())
self.assertEquals(self.concept1.concept_class, concept_reference.concept_class)
self.assertEquals(self.concept1.datatype, concept_reference.data_type)
self.assertEquals(self.concept1.parent, concept_reference.source)
self.assertEquals(self.concept1.owner_name, concept_reference.owner_name)
self.assertEquals(self.concept1.owner_type, concept_reference.owner_type)
self.assertEquals(self.concept1.display_name, concept_reference.display_name)
self.assertEquals(self.concept1.display_locale, concept_reference.display_locale)
self.assertEquals(
'/users/user1/sources/source1/%s/concepts/concept1/' % source_version.mnemonic,
concept_reference.concept_reference_url)
self.assertFalse(concept_reference.is_current_version)
def test_persist_clone_positive(self):
self.assertEquals(2, self.concept1.num_versions)
self.assertEquals(
self.concept_version, ConceptVersion.get_latest_version_of(self.concept1))
source_version = SourceVersion.get_latest_version_of(self.source1)
source_version.update_concept_version(self.concept_version)
self.assertEquals(1, len(source_version.concepts))
self.assertEquals(self.concept_version.id, source_version.concepts[0])
version2 = self.concept_version.clone()
errors = ConceptVersion.persist_clone(version2, self.user1)
self.assertEquals(0, len(errors))
self.assertEquals(3, self.concept1.num_versions)
self.assertEquals(version2, ConceptVersion.get_latest_version_of(self.concept1))
self.assertEquals(self.concept_version.public_access, version2.public_access)
self.assertEquals(self.concept_version, version2.previous_version)
self.assertEquals(self.concept_version.root_version, version2.root_version)
self.assertEquals(self.concept_version.external_id, version2.external_id)
self.assertEquals(self.user1.username, version2.version_created_by)
source_version.update_concept_version(version2)
self.assertEquals(1, len(source_version.concepts))
self.assertEquals(version2.id, source_version.concepts[0])
def test_persist_new_positive(self):
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
parent=self.source1,
concept_class='First',
names=[self.name],
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.concepts))
kwargs = {
'parent_resource': self.source1,
}
errors = Concept.persist_new(concept, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertFalse(concept.retired)
self.assertEquals(self.name.name, concept.display_name)
self.assertEquals(self.name.locale, concept.display_locale)
self.assertEquals(self.source1.owner_name, concept.owner_name)
self.assertEquals(self.source1.owner_type, concept.owner_type)
self.assertEquals(self.source1.public_access, concept.public_access)
self.assertEquals(1, concept.num_versions)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertEquals(concept_version, concept_version.root_version)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.concepts))
self.assertTrue(concept_version.id in source_version.concepts)
self.assertEquals(concept_version.mnemonic, concept_version.id)
def test_import_job_for_change_in_data(self):
stdout_stub = TestStream()
source_version_latest = SourceVersion.get_latest_version_of(self.source1)
concept = Concept(
mnemonic='1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='Diagnosis',
external_id='1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA',
names=[self.name])
kwargs = {
'parent_resource': self.source1,
}
Concept.persist_new(concept, self.user1, **kwargs)
importer = ConceptsImporter(self.source1, self.testfile, 'test', stdout_stub, TestStream())
importer.import_concepts(total=1)
all_concept_versions = ConceptVersion.objects.all()
self.assertEquals(len(all_concept_versions), 2)
concept = Concept.objects.get(mnemonic='1')
latest_concept_version = [version for version in all_concept_versions if version.previous_version][0]
self.assertEquals(len(latest_concept_version.names), 4)
self.assertTrue(('Updated concept, replacing version ID ' + latest_concept_version.previous_version.id) in stdout_stub.getvalue())
self.assertTrue('concepts of 1 1 - 1 updated' in stdout_stub.getvalue())
def test_import_job_for_change_in_data(self):
stdout_stub = TestStream()
stderr_stub = TestStream()
mapping = Mapping(
parent=self.source1,
map_type="SAME-AS",
from_concept=self.concept3,
to_source=self.source1,
to_concept_code="413532003",
external_id="junk",
)
kwargs = {"parent_resource": self.source1}
Mapping.persist_new(mapping, self.user1, **kwargs)
source_version = SourceVersion.get_latest_version_of(self.source1)
source_version.mappings = [mapping.id]
source_version.save()
importer = MappingsImporter(self.source1, self.testfile, stdout_stub, stderr_stub, "test")
importer.import_mappings(total=1)
self.assertTrue("**** Processed 1 out of 1 mappings - 1 updated, ****" in stdout_stub.getvalue())
self.assertTrue(("Updated mapping with ID " + mapping.id) in stdout_stub.getvalue())
updated_mapping = Mapping.objects.get(to_concept_code="413532003")
self.assertTrue(updated_mapping.retired)
self.assertEquals(updated_mapping.external_id, "70279ABBBBBBBBBBBBBBBBBBBBBBBBBBBBBB")
def test_dispatch_with_head_and_versions(self):
source_version = SourceVersion(
name='version1',
mnemonic='version1',
versioned_object=self.source1,
released=True,
created_by=self.user1,
updated_by=self.user1,
)
source_version.full_clean()
source_version.save()
concept2 = Concept(
mnemonic='concept2',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='not First',
external_id='EXTID',
names=[self.name],
)
display_name = LocalizedText(
name='concept2',
locale='en'
)
concept2.names.append(display_name)
kwargs = {
'parent_resource': self.source1,
}
Concept.persist_new(concept2, self.user1, **kwargs)
self.client.login(username='******', password='******')
kwargs = {
'org': self.org1.mnemonic,
'source': self.source1.mnemonic
}
response = self.client.get(reverse('concept-create', kwargs=kwargs))
self.assertEquals(response.status_code, 200)
content = json.loads(response.content)
self.assertEquals(2, len(content))
source_head_concepts = SourceVersion.objects.get(mnemonic='HEAD', versioned_object_id=self.source1.id).concepts
self.assertEquals(2, len(source_head_concepts))
for concept in content:
self.assertTrue(concept['version'] in source_head_concepts)
def retrieve(self, request, *args, **kwargs):
super(SourceRetrieveUpdateDestroyView, self).retrieve(request, *args, **kwargs)
self.object = self.get_object()
serializer = self.get_serializer(self.object)
data = serializer.data
source_version = SourceVersion.get_latest_version_of(self.object)
self.includeConceptsAndMappings(request, data, source_version)
return Response(data)
def create_new_source_version(self, new_version):
new_source_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_source_version.seed_concepts()
new_source_version.seed_mappings()
new_source_version.full_clean()
new_source_version.save()
self.source_version = new_source_version
def initialize(self, request, path_info_segment, **kwargs):
super(MappingBaseView, self).initialize(request, path_info_segment, **kwargs)
if self.parent_resource:
if hasattr(self.parent_resource, 'versioned_object'):
self.parent_resource_version = self.parent_resource
self.parent_resource = self.parent_resource.versioned_object
else:
self.parent_resource_version = SourceVersion.get_latest_version_of(self.parent_resource)
def __init__(self, source, concepts_file, user, output_stream, error_stream): """ Initialize mapping importer """ self.source = source self.concepts_file = concepts_file self.stdout = output_stream self.stderr = error_stream self.user = user # Retrieve latest source version and, if specified, create a new one self.source_version = SourceVersion.get_latest_version_of(self.source)
def test_persist_new_positive__same_mapping_different_source(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
# Repeat with same concepts
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping2',
)
kwargs = {
'parent_resource': self.source2,
}
source_version = SourceVersion.get_latest_version_of(self.source2)
self.assertEquals(0, len(source_version.mappings))
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Mapping.objects.filter(external_id='mapping2').exists())
mapping = Mapping.objects.get(external_id='mapping2')
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in source_version.mappings)
def save_object(self, obj, **kwargs):
with transaction.commit_on_success():
request_user = self.context['request'].user
snap_serializer = SourceDetailSerializer(kwargs['versioned_object'])
obj.source_snapshot = snap_serializer.data
errors = SourceVersion.persist_new(obj, user=request_user, **kwargs)
if errors:
self._errors.update(errors)
if not errors:
update_children_for_resource_version.delay(obj.id, 'source')
def test_import_job_for_one_record(self):
stdout_stub = TestStream()
importer = ConceptsImporter(self.source1, self.testfile, 'test', stdout_stub, TestStream())
importer.import_concepts(total=1)
self.assertTrue('Created new concept: 1 = Diagnosis' in stdout_stub.getvalue())
self.assertTrue('Finished importing concepts!' in stdout_stub.getvalue())
inserted_concept = Concept.objects.get(mnemonic='1')
self.assertEquals(inserted_concept.parent, self.source1)
inserted_concept_version = ConceptVersion.objects.get(versioned_object_id=inserted_concept.id)
source_version_latest = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(source_version_latest.concepts, [inserted_concept_version.id])
def test_persist_new_positive(self):
kwargs = {
'parent_resource': self.userprofile1
}
errors = Source.persist_new(self.new_source, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Source.objects.filter(name='source1').exists())
source = Source.objects.get(name='source1')
self.assertTrue(SourceVersion.objects.filter(versioned_object_id=source.id))
source_version = SourceVersion.objects.get(versioned_object_id=source.id)
self.assertEquals(1, source.num_versions)
self.assertEquals(source_version, SourceVersion.get_latest_version_of(source))
def test_persist_new_positive__earlier_source_version(self):
version1 = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(version1.concepts))
version2 = SourceVersion.for_base_object(self.source1, label='version2')
version2.save()
self.assertEquals(0, len(version2.concepts))
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='First',
names=[self.name],
)
kwargs = {
'parent_resource': self.source1,
'parent_resource_version': version1
}
errors = Concept.persist_new(concept, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertFalse(concept.retired)
self.assertEquals(self.name.name, concept.display_name)
self.assertEquals(self.name.locale, concept.display_locale)
self.assertEquals(self.source1.owner_name, concept.owner_name)
self.assertEquals(self.source1.owner_type, concept.owner_type)
self.assertEquals(self.source1.public_access, concept.public_access)
self.assertEquals(1, concept.num_versions)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertEquals(concept_version, concept_version.root_version)
version1 = SourceVersion.objects.get(id=version1.id)
self.assertEquals(1, len(version1.concepts))
self.assertTrue(concept_version.id in version1.concepts)
version2 = SourceVersion.objects.get(id=version2.id)
self.assertEquals(0, len(version2.concepts))
self.assertFalse(concept_version.id in version2.concepts)
def test_source_version_create_positive(self):
source_version = SourceVersion(
name='version1',
mnemonic='version1',
versioned_object=self.source1,
released=True,
created_by=self.user1,
updated_by=self.user1,
)
source_version.full_clean()
source_version.save()
self.assertTrue(SourceVersion.objects.filter(
mnemonic='version1',
versioned_object_type=ContentType.objects.get_for_model(Source),
versioned_object_id=self.source1.id
).exists())
self.assertIsNone(source_version.previous_version)
self.assertIsNone(source_version.previous_version_mnemonic)
self.assertIsNone(source_version.parent_version)
self.assertIsNone(source_version.parent_version_mnemonic)
self.assertEquals(self.org1.mnemonic, source_version.parent_resource)
self.assertEquals(self.org1.resource_type, source_version.parent_resource_type)
self.assertEquals(source_version, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(1, self.source1.num_versions)
def test_import_job_for_one_record(self):
stdout_stub = TestStream()
stderr_stub = TestStream()
importer = MappingsImporter(self.source1, self.testfile, stdout_stub, stderr_stub, 'test')
importer.import_mappings(total=1)
self.assertTrue('Created new mapping:' in stdout_stub.getvalue())
self.assertTrue('/users/user1/sources/source1/:413532003' in stdout_stub.getvalue())
inserted_mapping = Mapping.objects.get(to_concept_code='413532003')
self.assertEquals(inserted_mapping.to_source, self.source1)
self.assertEquals(inserted_mapping.from_source, self.source2)
mapping_ids = SourceVersion.get_latest_version_of(self.source1).mappings
mapping_version = MappingVersion.objects.get(versioned_object_id=inserted_mapping.id, is_latest_version=True)
self.assertEquals(mapping_ids[0], mapping_version.id)
def persist_clone(cls, obj, user=None, prev_latest_version=None, **kwargs):
errors = dict()
if not user:
errors[
'version_created_by'] = 'Must specify which user is attempting to create a new concept version.'
return errors
obj.version_created_by = user.username
previous_version = obj.previous_version
previous_was_latest = previous_version.is_latest_version and obj.is_latest_version
source_version = SourceVersion.get_head_of(obj.versioned_object.parent)
persisted = False
errored_action = 'saving new mapping version'
try:
obj.save(**kwargs)
obj.mnemonic = int(prev_latest_version.mnemonic) + 1
obj.save()
errored_action = "updating 'is_latest_version' attribute on previous version"
if previous_was_latest:
previous_version.is_latest_version = False
previous_version.save()
errored_action = 'replacing previous version in latest version of source'
source_version.update_mapping_version(obj)
# Mark versioned object as updated
mapping = obj.versioned_object
mapping.save()
persisted = True
finally:
if not persisted:
source_version.update_mapping_version(obj.previous_version)
if previous_was_latest:
previous_version.is_latest_version = True
previous_version.save()
if obj.id:
obj.delete()
errors['non_field_errors'] = [
'An error occurred while %s.' % errored_action
]
return errors
def test_get_latest_version_when_no_version_exist(self):
source = Source(name='source2',
mnemonic='source2',
full_name='Source Two',
source_type='Dictionary',
public_access=ACCESS_TYPE_EDIT,
default_locale='en',
supported_locales=['en'],
website='www.source2.com',
description='This is not the first test source')
kwargs = {'parent_resource': self.profile}
Source.persist_new(source, self.user, **kwargs)
source.get_version_model().objects.get(
mnemonic='HEAD', versioned_object_id=source.id).delete()
self.assertFalse(
SourceVersion.objects.filter(
versioned_object_id=source.id).exists())
self.assertIsNone(SourceVersion.get_latest_version_of(source))
def test_persist_new_negative__no_parent(self):
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='First',
names=[self.name],
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.concepts))
errors = Concept.persist_new(concept, self.user1)
self.assertEquals(1, len(errors))
self.assertTrue('parent' in errors)
self.assertFalse(Concept.objects.filter(mnemonic='concept1').exists())
self.assertEquals(0, concept.num_versions)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(0, len(source_version.concepts))
def test_persist_new_negative__no_parent(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(1, len(errors))
self.assertTrue('non_field_errors' in errors)
non_field_errors = errors['non_field_errors']
self.assertEquals(1, len(non_field_errors))
self.assertTrue('parent' in non_field_errors[0])
self.assertFalse(Mapping.objects.filter(external_id='mapping1').exists())
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(0, len(source_version.mappings))
def test_persist_new_negative__same_mapping(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(
Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
mv = MappingVersion.objects.get(versioned_object_id=mapping.id)
self.assertTrue(mv.id in source_version.mappings)
# Repeat with same concepts
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping2',
)
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(1, len(errors))
self.assertEquals(1, len(errors))
self.assertTrue('__all__' in errors)
non_field_errors = errors['__all__']
self.assertEquals(1, len(non_field_errors))
self.assertTrue('already exists' in non_field_errors[0])
self.assertEquals(1, len(source_version.mappings))
def initialize(self, request, path_info_segment, **kwargs):
self.include_retired = request.QUERY_PARAMS.get(
INCLUDE_RETIRED_PARAM, False)
include_inverse_param = request.GET.get(INCLUDE_INVERSE_MAPPINGS_PARAM,
'false')
self.include_inverse_mappings = 'true' == include_inverse_param
parent_path_info = self.get_parent_in_path(path_info_segment, levels=1)
if parent_path_info and '/' != parent_path_info:
self.concept = self.get_object_for_path(parent_path_info,
self.request)
parent_path_info = self.get_parent_in_path(parent_path_info,
levels=2)
if parent_path_info and '/' != parent_path_info:
self.parent_resource = self.get_object_for_path(
parent_path_info, self.request)
if hasattr(self.parent_resource, 'versioned_object'):
self.parent_resource_version = self.parent_resource
self.parent_resource = self.parent_resource.versioned_object
else:
self.parent_resource_version = SourceVersion.get_latest_version_of(
self.parent_resource)
def __init__(self,
source,
concepts_file,
user,
output_stream,
error_stream,
save_validation_errors=True,
validation_logger=None):
""" Initialize mapping importer """
self.source = source
self.concepts_file = concepts_file
self.stdout = output_stream
self.stderr = error_stream
self.user = user
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_head_of(self.source)
self.validation_logger = validation_logger
self.save_validation_errors = save_validation_errors
if self.save_validation_errors and self.validation_logger is None:
self.validation_logger = ValidationLogger()
def test_persist_new_positive(self):
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(
Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
self.assertEquals(self.source1.public_access, mapping.public_access)
self.assertEquals('user1', mapping.created_by)
self.assertEquals('user1', mapping.updated_by)
self.assertEquals(self.source1, mapping.parent)
self.assertEquals('Same As', mapping.map_type)
self.assertEquals(self.concept1, mapping.from_concept)
self.assertEquals(self.concept2, mapping.to_concept)
self.assertEquals(self.source1, mapping.from_source)
self.assertEquals(self.source1.owner_name, mapping.from_source_owner)
self.assertEquals(self.source1.mnemonic, mapping.from_source_name)
self.assertEquals(self.source1, mapping.get_to_source())
self.assertEquals(self.source1.owner_name, mapping.to_source_owner)
self.assertEquals(self.concept2.mnemonic,
mapping.get_to_concept_code())
self.assertEquals(self.concept2.display_name,
mapping.get_to_concept_name())
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.mappings))
self.assertTrue(
MappingVersion.objects.get(
versioned_object_id=mapping.id).id in source_version.mappings)
def persist_changes(cls, obj, updated_by, update_comment=None, **kwargs):
errors = dict()
obj.updated_by = updated_by
try:
if obj.to_source == None:
obj._meta.unique_together = (
("parent", "map_type", "from_concept", "to_concept"),
)
else:
obj._meta.unique_together = (
("parent", "map_type", "from_concept", "to_source", "to_concept_code"),
)
obj.full_clean()
except ValidationError as e:
errors.update(e.message_dict)
return errors
persisted = False
try:
source_version = SourceVersion.get_head_of(obj.parent)
obj.save(**kwargs)
prev_latest_version = MappingVersion.objects.get(versioned_object_id=obj.id, is_latest_version=True);
prev_latest_version.is_latest_version = False
new_latest_version = MappingVersion.for_mapping(obj)
new_latest_version.previous_version = prev_latest_version
new_latest_version.update_comment = update_comment
new_latest_version.mnemonic = int(prev_latest_version.mnemonic) + 1
new_latest_version.save()
source_version.update_mapping_version(new_latest_version)
prev_latest_version.save()
persisted = True
finally:
if not persisted:
errors['non_field_errors'] = ["Failed to persist mapping."]
return errors
def test_import_job_for_one_record(self):
stdout_stub = TestStream()
importer = ConceptsImporter(self.source1,
self.testfile,
'test',
stdout_stub,
TestStream(),
save_validation_errors=False)
importer.import_concepts(total=1)
self.assertTrue(
'Created new concept: 1 = Diagnosis' in stdout_stub.getvalue())
self.assertTrue(
'Finished importing concepts!' in stdout_stub.getvalue())
inserted_concept = Concept.objects.get(mnemonic='1')
self.assertEquals(inserted_concept.parent, self.source1)
inserted_concept_version = ConceptVersion.objects.get(
versioned_object_id=inserted_concept.id)
source_version_latest = SourceVersion.get_latest_version_of(
self.source1)
self.assertItemsEqual(source_version_latest.get_concept_ids(),
[inserted_concept_version.id])
def test_edit_mapping_make_new_version_positive(self):
mapping1 = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
kwargs = {
'parent_resource': self.source1,
}
errors = Mapping.persist_new(mapping1, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertEquals(
1,
len(MappingVersion.objects.filter(
versioned_object_id=mapping1.id)))
mapping1.map_type = 'BROADER_THAN'
Mapping.persist_changes(mapping1, self.user1)
self.assertEquals(
2,
len(MappingVersion.objects.filter(
versioned_object_id=mapping1.id)))
old_version = MappingVersion.objects.get(
versioned_object_id=mapping1.id, is_latest_version=False)
new_version = MappingVersion.objects.get(
versioned_object_id=mapping1.id, is_latest_version=True)
self.assertFalse(old_version.is_latest_version)
self.assertTrue(new_version.is_latest_version)
self.assertEquals(new_version.map_type, 'BROADER_THAN')
self.assertEquals(old_version.map_type, 'Same As')
def test_update_mapping_with_invalid_record(self):
mapping = Mapping(parent=self.source1,
map_type='SAME-AS',
from_concept=self.concept3,
to_concept=self.concept1)
kwargs = {
'parent_resource': self.source1,
}
Mapping.persist_new(mapping, self.user1, **kwargs)
source_version = SourceVersion.get_latest_version_of(self.source1)
source_version.mappings = [mapping.id]
source_version.save()
stderr_stub = TestStream()
invalid_json_file = open(
'./integration_tests/fixtures/one_internal_invalid_mapping.json',
'rb')
importer = MappingsImporter(self.source1, invalid_json_file,
TestStream(), stderr_stub, 'test')
importer.import_mappings(total=1)
self.assertTrue(
"Must specify either 'to_concept' or 'to_source' & 'to_concept_code'. Cannot specify both."
in stderr_stub.getvalue())
def restore_object(self, attrs, instance=None):
version = SourceVersion()
version.mnemonic = attrs.get(self.Meta.lookup_field)
return super(SourceVersionCreateSerializer,
self).restore_object(attrs, instance=version)
def import_mappings(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs):
initial_signal_processor = haystack.signal_processor
try:
haystack.signal_processor.teardown()
haystack.signal_processor = haystack.signals.BaseSignalProcessor
import_start_time = datetime.now()
logger.info('Started import at {}'.format(import_start_time.strftime("%Y-%m-%dT%H:%M:%S")))
""" Main mapping importer loop """
logger.info('Import mappings to source...')
self.test_mode = test_mode
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_head_of(self.source)
if new_version:
try:
new_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_version.full_clean()
new_version.save()
new_version.seed_concepts()
new_version.seed_mappings()
self.source_version = new_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
self.mapping_ids = set(self.source_version.get_mapping_ids())
self.count = 0
for line in self.mappings_file:
# Load the next JSON line
self.count += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_mapping(data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bars
if (self.count % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (self.count % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.mappings_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining %s unhandled mapping IDs\n' % len(self.mapping_ids)
self.stdout.write(str_log)
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old mappings...\n'
self.stdout.write(str_log)
logger.info(str_log)
for mapping_id in self.mapping_ids:
try:
if self.remove_mapping(mapping_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated mapping: %s\n' % mapping_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing mappings!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
actions = self.action_count
update_index_required = actions.get(ImportActionHelper.IMPORT_ACTION_ADD, 0) > 0
update_index_required |= actions.get(ImportActionHelper.IMPORT_ACTION_UPDATE, 0) > 0
if update_index_required:
logger.info('Indexing objects updated since {}'.format(import_start_time.strftime("%Y-%m-%dT%H:%M:%S")))
update_index.Command().handle(start_date=import_start_time.strftime("%Y-%m-%dT%H:%M:%S"), verbosity=2,
workers=4, batchsize=100)
finally:
haystack.signal_processor = initial_signal_processor
haystack.signal_processor.setup()
def import_concepts(self,
new_version=False,
total=0,
test_mode=False,
deactivate_old_records=False,
**kwargs):
self.action_count = {}
self.test_mode = test_mode
logger.info('Import concepts to source...')
if new_version:
try:
new_source_version = SourceVersion.for_base_object(
self.source,
new_version,
previous_version=self.source_version)
new_source_version.seed_concepts()
new_source_version.seed_mappings()
new_source_version.full_clean()
new_source_version.save()
self.source_version = new_source_version
except Exception as exc:
raise CommandError(
'Failed to create new source version due to %s' %
exc.args[0])
# Load the JSON file line by line and import each line
total = total or '(Unknown)'
self.user = User.objects.filter(is_superuser=True)[0]
self.concept_version_ids = set(self.source_version.concepts)
cnt = 0
self.create_concept_versions_map()
for line in self.concepts_file:
# Load the next JSON line
cnt += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (
exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_concept(self.source, data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s\nFailed to parse line: %s. Skipping it...\n' % (
exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (
exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bar
if (cnt % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (cnt % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.concepts_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining unhandled concept versions:\n'
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
str_log = ','.join(str(el) for el in self.concept_version_ids)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old concepts...\n'
self.stdout.write(str_log)
logger.info(str_log)
for version_id in self.concept_version_ids:
try:
if self.remove_concept_version(version_id):
self.count_action(
ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated concept version: %s\n' % version_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate concept version on ID %s! %s\n' % (
version_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing concepts!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
def test_get_head_of(self):
self.assertEquals(SourceVersion.get_head_of(self.source), self.source.get_version_model().objects.get(mnemonic='HEAD', versioned_object_id=self.source.id))
def save_object(self, obj, **kwargs):
errors = SourceVersion.persist_changes(obj, **kwargs)
self._errors.update(errors)
def test_persist_new_positive__repeated_mnemonic(self):
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='First',
names=[self.name],
)
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.concepts))
kwargs = {
'parent_resource': self.source1,
}
errors = Concept.persist_new(concept, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertFalse(concept.retired)
self.assertEquals(self.name.name, concept.display_name)
self.assertEquals(self.name.locale, concept.display_locale)
self.assertEquals(self.source1.owner_name, concept.owner_name)
self.assertEquals(self.source1.owner_type, concept.owner_type)
self.assertEquals(self.source1.public_access, concept.public_access)
self.assertEquals(1, concept.num_versions)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertEquals(concept_version, concept_version.root_version)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.concepts))
self.assertTrue(concept_version.id in source_version.concepts)
# Repeat with same mnemonic, different parent
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source2,
concept_class='First',
names=[self.name],
)
source_version = SourceVersion.get_latest_version_of(self.source2)
self.assertEquals(0, len(source_version.concepts))
kwargs = {
'parent_resource': self.source2,
}
errors = Concept.persist_new(concept, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertFalse(concept.retired)
self.assertEquals(self.name.name, concept.display_name)
self.assertEquals(self.name.locale, concept.display_locale)
self.assertEquals(self.source2.parent_resource, concept.owner_name)
self.assertEquals(self.source2.owner_type, concept.owner_type)
self.assertEquals(self.source2.public_access, concept.public_access)
self.assertEquals(1, concept.num_versions)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertEquals(concept_version, concept_version.root_version)
source_version = SourceVersion.objects.get(id=source_version.id)
self.assertEquals(1, len(source_version.concepts))
self.assertTrue(concept_version.id in source_version.concepts)
def test_delete_organization_with_sources_and_collections(self):
org = Organization(mnemonic='org1',
name='My Organization',
created_by=self.user,
updated_by=self.user)
org.full_clean()
org.save()
org_id = org.id
org = Organization.objects.get(id=org.id)
user1 = UserProfile.objects.get(mnemonic=self.user.username)
org.members.append(user1.id)
user1.organizations.append(org.id)
org.save()
user1.save()
source = Source(name='source1',
mnemonic='source1',
full_name='Source One',
parent=org,
created_by=self.user,
updated_by=self.user)
source.full_clean()
source.save()
source2 = Source(name='source2',
mnemonic='source2',
full_name='Source Two',
parent=org,
created_by=self.user,
updated_by=self.user)
source2.full_clean()
source2.save()
source_version = SourceVersion(
name='version1',
mnemonic='version1',
versioned_object=source2,
released=True,
created_by=self.user,
updated_by=self.user,
)
source_version.full_clean()
source_version.save()
source_version2 = SourceVersion(
name='version2',
mnemonic='version2',
versioned_object=source2,
released=True,
created_by=self.user,
updated_by=self.user,
)
source_version2.full_clean()
source_version2.save()
collection = Collection.objects.create(name='collection1',
mnemonic='collection1',
created_by=self.user,
updated_by=self.user,
parent=org,
external_id='EXTID1')
collection_version = CollectionVersion(
name='version1',
mnemonic='version1',
versioned_object=collection,
released=True,
created_by='user1',
updated_by='user1',
)
collection_version.full_clean()
collection_version.save()
collection_version2 = CollectionVersion(
name='version2',
mnemonic='version2',
versioned_object=collection,
released=True,
created_by=self.user,
updated_by=self.user,
)
collection_version2.full_clean()
collection_version2.save()
collection2 = Collection.objects.create(name='collection2',
mnemonic='collection2',
created_by=self.user,
updated_by=self.user,
parent=org,
external_id='EXTID2')
self.assertTrue(Organization.objects.filter(id=org_id).exists())
self.assertTrue(Source.objects.filter(id=source.id).exists())
self.assertTrue(Source.objects.filter(id=source2.id).exists())
self.assertTrue(
SourceVersion.objects.filter(id=source_version.id).exists())
self.assertTrue(
SourceVersion.objects.filter(id=source_version2.id).exists())
self.assertTrue(Collection.objects.filter(id=collection.id).exists())
self.assertTrue(Collection.objects.filter(id=collection2.id).exists())
self.assertTrue(
CollectionVersion.objects.filter(
id=collection_version.id).exists())
self.assertTrue(
CollectionVersion.objects.filter(
id=collection_version2.id).exists())
org.delete()
self.assertFalse(Organization.objects.filter(id=org.id).exists())
self.assertFalse(Source.objects.filter(id=source.id).exists())
self.assertFalse(Source.objects.filter(id=source2.id).exists())
self.assertFalse(
SourceVersion.objects.filter(id=source_version.id).exists())
self.assertFalse(
SourceVersion.objects.filter(id=source_version2.id).exists())
self.assertFalse(Collection.objects.filter(id=collection.id).exists())
self.assertFalse(Collection.objects.filter(id=collection2.id).exists())
self.assertFalse(
CollectionVersion.objects.filter(
id=collection_version.id).exists())
self.assertFalse(
CollectionVersion.objects.filter(
id=collection_version2.id).exists())
#should not delete member user
self.assertTrue(
UserProfile.objects.filter(mnemonic=self.user.username).exists())
#should delete org from organizations on user
self.assertFalse(org_id in UserProfile.objects.get(
mnemonic=self.user.username).organizations)
def import_mappings(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs):
""" Main mapping importer loop """
logger.info('Import mappings to source...')
self.test_mode = test_mode
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_latest_version_of(self.source)
if new_version:
try:
new_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_version.seed_concepts()
new_version.seed_mappings()
new_version.full_clean()
new_version.save()
self.source_version = new_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
self.mapping_ids = set(self.source_version.mappings)
self.count = 0
concepts_for_uris = Concept.objects.filter(parent_id=self.source.id)
self.concepts_cache = dict((x.uri, x) for x in concepts_for_uris)
for line in self.mappings_file:
# Load the next JSON line
self.count += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_mapping(data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bars
if (self.count % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (self.count % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.mappings_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining unhandled mapping IDs:\n'
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
str_log = ','.join(str(el) for el in self.mapping_ids)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old mappings...\n'
self.stdout.write(str_log)
logger.info(str_log)
for mapping_id in self.mapping_ids:
try:
if self.remove_mapping(mapping_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated mapping: %s\n' % mapping_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing mappings!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
