def persist_changes(cls, obj, updated_by, update_comment=None, **kwargs):
errors = dict()
obj.updated_by = updated_by
try:
obj.full_clean()
except ValidationError as e:
errors.update(e.message_dict)
return errors
persisted = False
try:
source_version = SourceVersion.get_head_of(obj.parent)
obj.save(**kwargs)
prev_latest_version = MappingVersion.objects.get(
versioned_object_id=obj.id, is_latest_version=True)
prev_latest_version.is_latest_version = False
prev_latest_version.save()
new_latest_version = MappingVersion.for_mapping(obj)
new_latest_version.previous_version = prev_latest_version
new_latest_version.update_comment = update_comment
new_latest_version.mnemonic = int(prev_latest_version.mnemonic) + 1
new_latest_version.save()
source_version.update_mapping_version(new_latest_version)
persisted = True
finally:
if not persisted:
errors['non_field_errors'] = ["Failed to persist mapping."]
return errors
def persist_changes(cls, obj, updated_by, update_comment=None, **kwargs):
errors = dict()
obj.updated_by = updated_by
try:
obj.full_clean()
except ValidationError as e:
errors.update(e.message_dict)
return errors
persisted = False
try:
source_version = SourceVersion.get_head_of(obj.parent)
obj.save(**kwargs)
prev_latest_version = MappingVersion.objects.get(versioned_object_id=obj.id, is_latest_version=True)
prev_latest_version.is_latest_version = False
prev_latest_version.save()
new_latest_version = MappingVersion.for_mapping(obj)
new_latest_version.previous_version = prev_latest_version
new_latest_version.update_comment = update_comment
new_latest_version.mnemonic = int(prev_latest_version.mnemonic) + 1
new_latest_version.save()
source_version.update_mapping_version(new_latest_version)
persisted = True
finally:
if not persisted:
errors['non_field_errors'] = ["Failed to persist mapping."]
return errors
def persist_clone(cls, obj, user=None, **kwargs):
errors = dict()
if not user:
errors[
'version_created_by'] = 'Must specify which user is attempting to create a new concept version.'
return errors
obj.version_created_by = user.username
previous_version = obj.previous_version
previous_was_latest = previous_version.is_latest_version and obj.is_latest_version
source_version = SourceVersion.get_head_of(obj.versioned_object.parent)
persisted = False
errored_action = 'saving new concept version'
try:
obj.clean()
obj.save(**kwargs)
obj.mnemonic = obj.id
obj.save()
errored_action = "updating 'is_latest_version' attribute on previous version"
if previous_was_latest:
previous_version.is_latest_version = False
previous_version.save()
errored_action = 'replacing previous version in latest version of source'
source_version.update_concept_version(obj)
# Mark versioned object as updated
concept = obj.versioned_object
concept.extras = obj.extras
concept.names = obj.names
concept.descriptions = obj.descriptions
concept.concept_class = obj.concept_class
concept.datatype = obj.datatype
concept.save()
persisted = True
except ValidationError as err:
errors.update(err.message_dict)
finally:
if not persisted:
source_version.update_concept_version(obj.previous_version)
if previous_was_latest:
previous_version.is_latest_version = True
previous_version.save()
if obj.id:
obj.delete()
errors['non_field_errors'] = [
'An error occurred while %s.' % errored_action
]
return errors
def __init__(self, source, concepts_file, user, output_stream, error_stream, save_validation_errors=True, validation_logger=None):
""" Initialize mapping importer """
self.source = source
self.concepts_file = concepts_file
self.stdout = output_stream
self.stderr = error_stream
self.user = user
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_head_of(self.source)
self.validation_logger = validation_logger
self.save_validation_errors = save_validation_errors
if self.save_validation_errors and self.validation_logger is None:
self.validation_logger = ValidationLogger()
def test_get_head_of_when_head_does_not_exist(self):
source = Source(name='source2',
mnemonic='source2',
full_name='Source Two',
source_type='Dictionary',
public_access=ACCESS_TYPE_EDIT,
default_locale='en',
supported_locales=['en'],
website='www.source2.com',
description='This is not the first test source')
kwargs = {'parent_resource': self.profile}
Source.persist_new(source, self.user, **kwargs)
source.get_version_model().objects.get(
mnemonic='HEAD', versioned_object_id=source.id).delete()
self.assertIsNone(SourceVersion.get_head_of(source))
def persist_clone(cls, obj, user=None, **kwargs):
errors = dict()
if not user:
errors['version_created_by'] = 'Must specify which user is attempting to create a new concept version.'
return errors
obj.version_created_by = user.username
previous_version = obj.previous_version
previous_was_latest = previous_version.is_latest_version and obj.is_latest_version
source_version = SourceVersion.get_head_of(obj.versioned_object.parent)
persisted = False
errored_action = 'saving new concept version'
try:
obj.clean()
obj.save(**kwargs)
obj.mnemonic = obj.id
obj.save()
errored_action = "updating 'is_latest_version' attribute on previous version"
if previous_was_latest:
previous_version.is_latest_version = False
previous_version.save()
errored_action = 'replacing previous version in latest version of source'
source_version.update_concept_version(obj)
# Mark versioned object as updated
concept = obj.versioned_object
concept.extras = obj.extras
concept.names= obj.names
concept.descriptions= obj.descriptions
concept.concept_class=obj.concept_class
concept.datatype=obj.datatype
concept.save()
persisted = True
except ValidationError as err:
errors.update(err.message_dict)
finally:
if not persisted:
source_version.update_concept_version(obj.previous_version)
if previous_was_latest:
previous_version.is_latest_version = True
previous_version.save()
if obj.id:
obj.delete()
errors['non_field_errors'] = ['An error occurred while %s.' % errored_action]
return errors
def test_get_head_of_when_head_does_not_exist(self):
source = Source(name='source2',
mnemonic='source2',
full_name='Source Two',
source_type='Dictionary',
public_access=ACCESS_TYPE_EDIT,
default_locale='en',
supported_locales=['en'],
website='www.source2.com',
description='This is not the first test source'
)
kwargs = {
'parent_resource': self.profile
}
Source.persist_new(source, self.user, **kwargs)
source.get_version_model().objects.get(mnemonic='HEAD', versioned_object_id=source.id).delete()
self.assertIsNone(SourceVersion.get_head_of(source))
def persist_clone(cls, obj, user=None, prev_latest_version=None, **kwargs):
errors = dict()
if not user:
errors[
'version_created_by'] = 'Must specify which user is attempting to create a new concept version.'
return errors
obj.version_created_by = user.username
previous_version = obj.previous_version
previous_was_latest = previous_version.is_latest_version and obj.is_latest_version
source_version = SourceVersion.get_head_of(obj.versioned_object.parent)
persisted = False
errored_action = 'saving new mapping version'
try:
obj.save(**kwargs)
obj.mnemonic = int(prev_latest_version.mnemonic) + 1
obj.save()
errored_action = "updating 'is_latest_version' attribute on previous version"
if previous_was_latest:
previous_version.is_latest_version = False
previous_version.save()
errored_action = 'replacing previous version in latest version of source'
source_version.update_mapping_version(obj)
# Mark versioned object as updated
mapping = obj.versioned_object
mapping.save()
persisted = True
finally:
if not persisted:
source_version.update_mapping_version(obj.previous_version)
if previous_was_latest:
previous_version.is_latest_version = True
previous_version.save()
if obj.id:
obj.delete()
errors['non_field_errors'] = [
'An error occurred while %s.' % errored_action
]
return errors
def __init__(self,
source,
concepts_file,
user,
output_stream,
error_stream,
save_validation_errors=True,
validation_logger=None):
""" Initialize mapping importer """
self.source = source
self.concepts_file = concepts_file
self.stdout = output_stream
self.stderr = error_stream
self.user = user
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_head_of(self.source)
self.validation_logger = validation_logger
self.save_validation_errors = save_validation_errors
if self.save_validation_errors and self.validation_logger is None:
self.validation_logger = ValidationLogger()
def persist_clone(cls, obj, user=None, prev_latest_version=None, **kwargs):
errors = dict()
if not user:
errors['version_created_by'] = 'Must specify which user is attempting to create a new concept version.'
return errors
obj.version_created_by = user.username
previous_version = obj.previous_version
previous_was_latest = previous_version.is_latest_version and obj.is_latest_version
source_version = SourceVersion.get_head_of(obj.versioned_object.parent)
persisted = False
errored_action = 'saving new mapping version'
try:
obj.save(**kwargs)
obj.mnemonic = int(prev_latest_version.mnemonic) + 1
obj.save()
errored_action = "updating 'is_latest_version' attribute on previous version"
if previous_was_latest:
previous_version.is_latest_version = False
previous_version.save()
errored_action = 'replacing previous version in latest version of source'
source_version.update_mapping_version(obj)
# Mark versioned object as updated
mapping = obj.versioned_object
mapping.save()
persisted = True
finally:
if not persisted:
source_version.update_mapping_version(obj.previous_version)
if previous_was_latest:
previous_version.is_latest_version = True
previous_version.save()
if obj.id:
obj.delete()
errors['non_field_errors'] = ['An error occurred while %s.' % errored_action]
return errors
def test_get_head_of(self):
self.assertEquals(SourceVersion.get_head_of(self.source), self.source.get_version_model().objects.get(mnemonic='HEAD', versioned_object_id=self.source.id))
def import_mappings(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs):
initial_signal_processor = haystack.signal_processor
try:
haystack.signal_processor.teardown()
haystack.signal_processor = haystack.signals.BaseSignalProcessor
import_start_time = datetime.now()
logger.info('Started import at {}'.format(import_start_time.strftime("%Y-%m-%dT%H:%M:%S")))
""" Main mapping importer loop """
logger.info('Import mappings to source...')
self.test_mode = test_mode
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_head_of(self.source)
if new_version:
try:
new_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_version.full_clean()
new_version.save()
new_version.seed_concepts()
new_version.seed_mappings()
self.source_version = new_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
self.mapping_ids = set(self.source_version.get_mapping_ids())
self.count = 0
for line in self.mappings_file:
# Load the next JSON line
self.count += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_mapping(data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bars
if (self.count % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (self.count % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.mappings_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining %s unhandled mapping IDs\n' % len(self.mapping_ids)
self.stdout.write(str_log)
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old mappings...\n'
self.stdout.write(str_log)
logger.info(str_log)
for mapping_id in self.mapping_ids:
try:
if self.remove_mapping(mapping_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated mapping: %s\n' % mapping_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing mappings!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
actions = self.action_count
update_index_required = actions.get(ImportActionHelper.IMPORT_ACTION_ADD, 0) > 0
update_index_required |= actions.get(ImportActionHelper.IMPORT_ACTION_UPDATE, 0) > 0
if update_index_required:
logger.info('Indexing objects updated since {}'.format(import_start_time.strftime("%Y-%m-%dT%H:%M:%S")))
update_index.Command().handle(start_date=import_start_time.strftime("%Y-%m-%dT%H:%M:%S"), verbosity=2,
workers=4, batchsize=100)
finally:
haystack.signal_processor = initial_signal_processor
haystack.signal_processor.setup()
def test_get_head_of(self):
self.assertEquals(SourceVersion.get_head_of(self.source), self.source.get_version_model().objects.get(mnemonic='HEAD', versioned_object_id=self.source.id))