def test_persist_changes_positive__good_parent_version(self):
version1 = SourceVersion.for_base_object(self.source1, 'version1')
version1.full_clean()
version1.save()
version2 = SourceVersion.for_base_object(self.source1, 'version2')
version2.full_clean()
version2.save()
self.assertIsNone(version2.parent_version)
mnemonic = version2.mnemonic
released = version2.released
description = version2.description
external_id = version2.external_id
id = version2.id
version2._parent_version_mnemonic = 'version1'
version2.mnemonic = "%s-prime" % mnemonic
version2.released = not released
version2.description = "%s-prime" % description
version2.external_id = "%s-prime" % external_id
errors = SourceVersion.persist_changes(version2)
self.assertEquals(0, len(errors))
version2 = SourceVersion.objects.get(id=id)
self.assertEquals(self.source1, version2.versioned_object)
self.assertEquals(2, self.source1.num_versions)
self.assertEquals(version2, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(version1, version2.parent_version)
self.assertNotEquals(mnemonic, version2.mnemonic)
self.assertNotEquals(released, version2.released)
self.assertNotEquals(description, version2.description)
self.assertNotEquals(external_id, version2.external_id)
def test_persist_changes_positive__seed_from_previous_over_parent(self):
version1 = SourceVersion.for_base_object(self.source1, 'version1')
version1.concepts = [1]
version1.full_clean()
version1.save()
version2 = SourceVersion.for_base_object(self.source1, 'version2')
version2.concepts = [2]
version2.full_clean()
version2.save()
self.assertIsNone(version2.previous_version)
version3 = SourceVersion.for_base_object(self.source1, 'version3')
version3.full_clean()
version3.save()
mnemonic = version3.mnemonic
released = version3.released
description = version3.description
external_id = version3.external_id
id = version3.id
version3._parent_version_mnemonic = 'version2'
version3._previous_version_mnemonic = 'version1'
version3.mnemonic = "%s-prime" % mnemonic
version3.released = not released
version3.description = "%s-prime" % description
version3.external_id = "%s-prime" % external_id
errors = SourceVersion.persist_changes(version3)
self.assertEquals(0, len(errors))
version3 = SourceVersion.objects.get(id=id)
self.assertEquals(self.source1, version3.versioned_object)
self.assertEquals(3, self.source1.num_versions)
self.assertEquals(version3, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(version1, version3.previous_version)
self.assertEquals(version2, version3.parent_version)
self.assertEquals([], version3.concepts)
self.assertNotEquals(mnemonic, version3.mnemonic)
self.assertNotEquals(released, version3.released)
self.assertNotEquals(description, version3.description)
self.assertNotEquals(external_id, version3.external_id)
errors = SourceVersion.persist_changes(version3, seed_concepts=True)
self.assertEquals(0, len(errors))
version3 = SourceVersion.objects.get(id=id)
self.assertEquals(self.source1, version3.versioned_object)
self.assertEquals(3, self.source1.num_versions)
self.assertEquals(version3, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(version2, version3.parent_version)
self.assertEquals(version1, version3.previous_version)
self.assertEquals([1], version3.concepts)
def test_persist_new_positive__earlier_source_version(self):
version1 = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(version1.mappings))
version2 = SourceVersion.for_base_object(self.source1, label='version2')
version2.save()
self.assertEquals(0, len(version2.mappings))
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
kwargs = {
'parent_resource': self.source1,
'parent_resource_version': version1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
version1 = SourceVersion.objects.get(id=version1.id)
self.assertEquals(1, len(version1.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in version1.mappings)
version2 = SourceVersion.objects.get(id=version2.id)
self.assertEquals(0, len(version2.mappings))
latest_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(latest_version.mappings))
def test_persist_changes_negative__parent_version_is_self(self):
version1 = SourceVersion.for_base_object(self.source1, 'version1', released=True)
version1.full_clean()
version1.save()
mnemonic = version1.mnemonic
released = version1.released
description = version1.description
external_id = version1.external_id
id = version1.id
version1._parent_version_mnemonic = mnemonic
version1.released = not released
version1.description = "%s-prime" % description
version1.external_id = "%s-prime" % external_id
errors = SourceVersion.persist_changes(version1)
self.assertEquals(1, len(errors))
self.assertTrue('parentVersion' in errors)
version1 = SourceVersion.objects.get(id=id)
self.assertEquals(self.source1, version1.versioned_object)
self.assertEquals(1, self.source1.num_versions)
self.assertEquals(version1, SourceVersion.get_latest_version_of(self.source1))
self.assertEquals(mnemonic, version1.mnemonic)
self.assertEquals(released, version1.released)
self.assertEquals(description, version1.description)
self.assertEquals(external_id, version1.external_id)
def test_persist_new_positive__earlier_source_version(self):
version1 = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(version1.mappings))
version2 = SourceVersion.for_base_object(self.source1, label='version2')
version2.save()
self.assertEquals(0, len(version2.mappings))
source_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(source_version.mappings))
mapping = Mapping(
map_type='Same As',
from_concept=self.concept1,
to_concept=self.concept2,
external_id='mapping1',
)
kwargs = {
'parent_resource': self.source1,
'parent_resource_version': version1,
}
errors = Mapping.persist_new(mapping, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Mapping.objects.filter(external_id='mapping1').exists())
mapping = Mapping.objects.get(external_id='mapping1')
version1 = SourceVersion.objects.get(id=version1.id)
self.assertEquals(1, len(version1.mappings))
self.assertTrue(MappingVersion.objects.get(versioned_object_id=mapping.id).id in version1.mappings)
version2 = SourceVersion.objects.get(id=version2.id)
self.assertEquals(0, len(version2.mappings))
latest_version = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(latest_version.mappings))
def create_new_source_version(self, new_version):
new_source_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_source_version.seed_concepts()
new_source_version.seed_mappings()
new_source_version.full_clean()
new_source_version.save()
self.source_version = new_source_version
def create_new_source_version(self, new_version):
new_source_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_source_version.seed_concepts()
new_source_version.seed_mappings()
new_source_version.full_clean()
new_source_version.save()
self.source_version = new_source_version
def test_for_base_object_positive(self):
version1 = SourceVersion.for_base_object(self.source1, 'version1')
version1.full_clean()
version1.save()
self.assertEquals(version1.mnemonic, 'version1')
self.assertEquals(self.source1, version1.versioned_object)
self.assertEquals(self.source1.name, version1.name)
self.assertEquals(self.source1.full_name, version1.full_name)
self.assertEquals(self.source1.source_type, version1.source_type)
self.assertEquals(self.source1.public_access, version1.public_access)
self.assertEquals(self.source1.default_locale, version1.default_locale)
self.assertEquals(self.source1.supported_locales, version1.supported_locales)
self.assertEquals(self.source1.website, version1.website)
self.assertEquals(self.source1.description, version1.description)
self.assertEquals(self.source1.external_id, version1.external_id)
self.assertFalse(version1.released)
self.assertIsNone(version1.parent_version)
self.assertIsNone(version1.previous_version)
self.assertEquals(1, self.source1.num_versions)
def test_persist_new_positive__earlier_source_version(self):
version1 = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(version1.concepts))
version2 = SourceVersion.for_base_object(self.source1,
label='version2')
version2.save()
self.assertEquals(0, len(version2.concepts))
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='First',
names=[self.name],
)
kwargs = {
'parent_resource': self.source1,
'parent_resource_version': version1
}
errors = Concept.persist_new(concept, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertFalse(concept.retired)
self.assertEquals(self.name.name, concept.display_name)
self.assertEquals(self.name.locale, concept.display_locale)
self.assertEquals(self.source1.owner_name, concept.owner_name)
self.assertEquals(self.source1.owner_type, concept.owner_type)
self.assertEquals(self.source1.public_access, concept.public_access)
self.assertEquals(1, concept.num_versions)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertEquals(concept_version, concept_version.root_version)
version1 = SourceVersion.objects.get(id=version1.id)
self.assertEquals(1, len(version1.concepts))
self.assertTrue(concept_version.id in version1.concepts)
version2 = SourceVersion.objects.get(id=version2.id)
self.assertEquals(0, len(version2.concepts))
self.assertFalse(concept_version.id in version2.concepts)
def test_persist_new_positive__earlier_source_version(self):
version1 = SourceVersion.get_latest_version_of(self.source1)
self.assertEquals(0, len(version1.concepts))
version2 = SourceVersion.for_base_object(self.source1, label='version2')
version2.save()
self.assertEquals(0, len(version2.concepts))
concept = Concept(
mnemonic='concept1',
created_by=self.user1,
updated_by=self.user1,
parent=self.source1,
concept_class='First',
names=[self.name],
)
kwargs = {
'parent_resource': self.source1,
'parent_resource_version': version1
}
errors = Concept.persist_new(concept, self.user1, **kwargs)
self.assertEquals(0, len(errors))
self.assertTrue(Concept.objects.filter(mnemonic='concept1').exists())
self.assertFalse(concept.retired)
self.assertEquals(self.name.name, concept.display_name)
self.assertEquals(self.name.locale, concept.display_locale)
self.assertEquals(self.source1.owner_name, concept.owner_name)
self.assertEquals(self.source1.owner_type, concept.owner_type)
self.assertEquals(self.source1.public_access, concept.public_access)
self.assertEquals(1, concept.num_versions)
concept_version = ConceptVersion.get_latest_version_of(concept)
self.assertEquals(concept_version, concept_version.root_version)
version1 = SourceVersion.objects.get(id=version1.id)
self.assertEquals(1, len(version1.concepts))
self.assertTrue(concept_version.id in version1.concepts)
version2 = SourceVersion.objects.get(id=version2.id)
self.assertEquals(0, len(version2.concepts))
self.assertFalse(concept_version.id in version2.concepts)
def handle(self, *args, **options):
user = self.create_admin_user()
version = 'v2019-07-01'
source_version = SourceVersion.objects.filter(mnemonic=version)
if source_version.exists():
return
org = self.create_organization(user, 'CIEL')
source = self.create_source(user, org, 'CIEL')
demo_file = open('./demo-data/ciel_' + version + '_c2k.json', 'rb')
importer = ConceptsImporter(source,
demo_file,
user,
OutputWrapper(sys.stdout),
OutputWrapper(sys.stderr),
save_validation_errors=False)
importer.import_concepts(**options)
demo_file = open('./demo-data/ciel_' + version + '_m2k.json', 'rb')
importer = MappingsImporter(source, demo_file,
OutputWrapper(sys.stdout),
OutputWrapper(sys.stderr), user)
importer.import_mappings(**options)
new_version = SourceVersion.for_base_object(source,
version,
released=True)
SourceVersion.persist_new(new_version,
versioned_object=source,
force_insert=True)
update_children_for_resource_version.delay(new_version.id, 'source')
export_source.delay(new_version.id)
def import_concepts(self,
new_version=False,
total=0,
test_mode=False,
deactivate_old_records=False,
**kwargs):
self.action_count = {}
self.test_mode = test_mode
logger.info('Import concepts to source...')
if new_version:
try:
new_source_version = SourceVersion.for_base_object(
self.source,
new_version,
previous_version=self.source_version)
new_source_version.seed_concepts()
new_source_version.seed_mappings()
new_source_version.full_clean()
new_source_version.save()
self.source_version = new_source_version
except Exception as exc:
raise CommandError(
'Failed to create new source version due to %s' %
exc.args[0])
# Load the JSON file line by line and import each line
total = total or '(Unknown)'
self.user = User.objects.filter(is_superuser=True)[0]
self.concept_version_ids = set(self.source_version.concepts)
cnt = 0
self.create_concept_versions_map()
for line in self.concepts_file:
# Load the next JSON line
cnt += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (
exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_concept(self.source, data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s\nFailed to parse line: %s. Skipping it...\n' % (
exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (
exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bar
if (cnt % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (cnt % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.concepts_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining unhandled concept versions:\n'
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
str_log = ','.join(str(el) for el in self.concept_version_ids)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old concepts...\n'
self.stdout.write(str_log)
logger.info(str_log)
for version_id in self.concept_version_ids:
try:
if self.remove_concept_version(version_id):
self.count_action(
ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated concept version: %s\n' % version_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate concept version on ID %s! %s\n' % (
version_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing concepts!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
def import_mappings(self, new_version=False, total=0, test_mode=False, **kwargs):
""" Main mapping importer loop """
logger.info('Import mappings to source...')
self.test_mode = test_mode
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_latest_version_of(self.source)
if new_version:
try:
new_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_version.seed_concepts()
new_version.seed_mappings()
new_version.full_clean()
new_version.save()
self.source_version = new_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
self.mapping_ids = set(self.source_version.mappings)
self.count = 0
for line in self.mappings_file:
# Load the next JSON line
self.count += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_mapping(data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bars
if (self.count % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (self.count % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.mappings_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining unhandled mapping IDs:\n'
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
str_log = ','.join(str(el) for el in self.mapping_ids)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if kwargs['deactivate_old_records']:
str_log = 'Deactivating old mappings...\n'
self.stdout.write(str_log)
logger.info(str_log)
for mapping_id in self.mapping_ids:
try:
if self.remove_mapping(mapping_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated mapping: %s\n' % mapping_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing mappings!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
def import_mappings(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs):
initial_signal_processor = haystack.signal_processor
try:
haystack.signal_processor.teardown()
haystack.signal_processor = haystack.signals.BaseSignalProcessor
import_start_time = datetime.now()
logger.info('Started import at {}'.format(import_start_time.strftime("%Y-%m-%dT%H:%M:%S")))
""" Main mapping importer loop """
logger.info('Import mappings to source...')
self.test_mode = test_mode
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_head_of(self.source)
if new_version:
try:
new_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_version.full_clean()
new_version.save()
new_version.seed_concepts()
new_version.seed_mappings()
self.source_version = new_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
self.mapping_ids = set(self.source_version.get_mapping_ids())
self.count = 0
for line in self.mappings_file:
# Load the next JSON line
self.count += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_mapping(data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bars
if (self.count % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (self.count % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.mappings_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining %s unhandled mapping IDs\n' % len(self.mapping_ids)
self.stdout.write(str_log)
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old mappings...\n'
self.stdout.write(str_log)
logger.info(str_log)
for mapping_id in self.mapping_ids:
try:
if self.remove_mapping(mapping_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated mapping: %s\n' % mapping_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing mappings!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
actions = self.action_count
update_index_required = actions.get(ImportActionHelper.IMPORT_ACTION_ADD, 0) > 0
update_index_required |= actions.get(ImportActionHelper.IMPORT_ACTION_UPDATE, 0) > 0
if update_index_required:
logger.info('Indexing objects updated since {}'.format(import_start_time.strftime("%Y-%m-%dT%H:%M:%S")))
update_index.Command().handle(start_date=import_start_time.strftime("%Y-%m-%dT%H:%M:%S"), verbosity=2,
workers=4, batchsize=100)
finally:
haystack.signal_processor = initial_signal_processor
haystack.signal_processor.setup()
def import_concepts(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs):
self.action_count = {}
self.test_mode = test_mode
logger.info('Import concepts to source...')
if new_version:
try:
new_source_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_source_version.seed_concepts()
new_source_version.seed_mappings()
new_source_version.full_clean()
new_source_version.save()
self.source_version = new_source_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
total = total or '(Unknown)'
self.user = User.objects.filter(is_superuser=True)[0]
self.concept_version_ids = set(self.source_version.concepts)
cnt = 0
self.create_concept_versions_map()
for line in self.concepts_file:
# Load the next JSON line
cnt += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_concept(self.source, data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s\nFailed to parse line: %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bar
if (cnt % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (cnt % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.concepts_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining unhandled concept versions:\n'
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
str_log = ','.join(str(el) for el in self.concept_version_ids)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old concepts...\n'
self.stdout.write(str_log)
logger.info(str_log)
for version_id in self.concept_version_ids:
try:
if self.remove_concept_version(version_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated concept version: %s\n' % version_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate concept version on ID %s! %s\n' % (version_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing concepts!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
def import_mappings(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs):
""" Main mapping importer loop """
logger.info('Import mappings to source...')
self.test_mode = test_mode
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_latest_version_of(self.source)
if new_version:
try:
new_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_version.seed_concepts()
new_version.seed_mappings()
new_version.full_clean()
new_version.save()
self.source_version = new_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
self.mapping_ids = set(self.source_version.mappings)
self.count = 0
concepts_for_uris = Concept.objects.filter(parent_id=self.source.id)
self.concepts_cache = dict((x.uri, x) for x in concepts_for_uris)
for line in self.mappings_file:
# Load the next JSON line
self.count += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_mapping(data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bars
if (self.count % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (self.count % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.mappings_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining unhandled mapping IDs:\n'
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
str_log = ','.join(str(el) for el in self.mapping_ids)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old mappings...\n'
self.stdout.write(str_log)
logger.info(str_log)
for mapping_id in self.mapping_ids:
try:
if self.remove_mapping(mapping_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated mapping: %s\n' % mapping_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing mappings!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
def test_for_base_object_negative__bad_parent_version(self):
with self.assertRaises(ValueError):
version1 = SourceVersion.for_base_object(self.source1, 'version1', parent_version=self.source1)
version1.full_clean()
version1.save()
self.assertEquals(0, self.source1.num_versions)
def test_for_base_object_negative__newborn_source(self):
with self.assertRaises(ValidationError):
version1 = SourceVersion.for_base_object(Source(), 'version1')
version1.full_clean()
version1.save()
