def persist_changes(cls, obj, updated_by, update_comment=None, **kwargs): errors = dict() obj.updated_by = updated_by try: obj.full_clean() except ValidationError as e: errors.update(e.message_dict) return errors persisted = False try: source_version = SourceVersion.get_head_of(obj.parent) obj.save(**kwargs) prev_latest_version = MappingVersion.objects.get( versioned_object_id=obj.id, is_latest_version=True) prev_latest_version.is_latest_version = False prev_latest_version.save() new_latest_version = MappingVersion.for_mapping(obj) new_latest_version.previous_version = prev_latest_version new_latest_version.update_comment = update_comment new_latest_version.mnemonic = int(prev_latest_version.mnemonic) + 1 new_latest_version.save() source_version.update_mapping_version(new_latest_version) persisted = True finally: if not persisted: errors['non_field_errors'] = ["Failed to persist mapping."] return errors
def persist_changes(cls, obj, updated_by, update_comment=None, **kwargs): errors = dict() obj.updated_by = updated_by try: obj.full_clean() except ValidationError as e: errors.update(e.message_dict) return errors persisted = False try: source_version = SourceVersion.get_head_of(obj.parent) obj.save(**kwargs) prev_latest_version = MappingVersion.objects.get(versioned_object_id=obj.id, is_latest_version=True) prev_latest_version.is_latest_version = False prev_latest_version.save() new_latest_version = MappingVersion.for_mapping(obj) new_latest_version.previous_version = prev_latest_version new_latest_version.update_comment = update_comment new_latest_version.mnemonic = int(prev_latest_version.mnemonic) + 1 new_latest_version.save() source_version.update_mapping_version(new_latest_version) persisted = True finally: if not persisted: errors['non_field_errors'] = ["Failed to persist mapping."] return errors
def persist_clone(cls, obj, user=None, **kwargs): errors = dict() if not user: errors[ 'version_created_by'] = 'Must specify which user is attempting to create a new concept version.' return errors obj.version_created_by = user.username previous_version = obj.previous_version previous_was_latest = previous_version.is_latest_version and obj.is_latest_version source_version = SourceVersion.get_head_of(obj.versioned_object.parent) persisted = False errored_action = 'saving new concept version' try: obj.clean() obj.save(**kwargs) obj.mnemonic = obj.id obj.save() errored_action = "updating 'is_latest_version' attribute on previous version" if previous_was_latest: previous_version.is_latest_version = False previous_version.save() errored_action = 'replacing previous version in latest version of source' source_version.update_concept_version(obj) # Mark versioned object as updated concept = obj.versioned_object concept.extras = obj.extras concept.names = obj.names concept.descriptions = obj.descriptions concept.concept_class = obj.concept_class concept.datatype = obj.datatype concept.save() persisted = True except ValidationError as err: errors.update(err.message_dict) finally: if not persisted: source_version.update_concept_version(obj.previous_version) if previous_was_latest: previous_version.is_latest_version = True previous_version.save() if obj.id: obj.delete() errors['non_field_errors'] = [ 'An error occurred while %s.' % errored_action ] return errors
def __init__(self, source, concepts_file, user, output_stream, error_stream, save_validation_errors=True, validation_logger=None): """ Initialize mapping importer """ self.source = source self.concepts_file = concepts_file self.stdout = output_stream self.stderr = error_stream self.user = user # Retrieve latest source version and, if specified, create a new one self.source_version = SourceVersion.get_head_of(self.source) self.validation_logger = validation_logger self.save_validation_errors = save_validation_errors if self.save_validation_errors and self.validation_logger is None: self.validation_logger = ValidationLogger()
def test_get_head_of_when_head_does_not_exist(self): source = Source(name='source2', mnemonic='source2', full_name='Source Two', source_type='Dictionary', public_access=ACCESS_TYPE_EDIT, default_locale='en', supported_locales=['en'], website='www.source2.com', description='This is not the first test source') kwargs = {'parent_resource': self.profile} Source.persist_new(source, self.user, **kwargs) source.get_version_model().objects.get( mnemonic='HEAD', versioned_object_id=source.id).delete() self.assertIsNone(SourceVersion.get_head_of(source))
def persist_clone(cls, obj, user=None, **kwargs): errors = dict() if not user: errors['version_created_by'] = 'Must specify which user is attempting to create a new concept version.' return errors obj.version_created_by = user.username previous_version = obj.previous_version previous_was_latest = previous_version.is_latest_version and obj.is_latest_version source_version = SourceVersion.get_head_of(obj.versioned_object.parent) persisted = False errored_action = 'saving new concept version' try: obj.clean() obj.save(**kwargs) obj.mnemonic = obj.id obj.save() errored_action = "updating 'is_latest_version' attribute on previous version" if previous_was_latest: previous_version.is_latest_version = False previous_version.save() errored_action = 'replacing previous version in latest version of source' source_version.update_concept_version(obj) # Mark versioned object as updated concept = obj.versioned_object concept.extras = obj.extras concept.names= obj.names concept.descriptions= obj.descriptions concept.concept_class=obj.concept_class concept.datatype=obj.datatype concept.save() persisted = True except ValidationError as err: errors.update(err.message_dict) finally: if not persisted: source_version.update_concept_version(obj.previous_version) if previous_was_latest: previous_version.is_latest_version = True previous_version.save() if obj.id: obj.delete() errors['non_field_errors'] = ['An error occurred while %s.' % errored_action] return errors
def test_get_head_of_when_head_does_not_exist(self): source = Source(name='source2', mnemonic='source2', full_name='Source Two', source_type='Dictionary', public_access=ACCESS_TYPE_EDIT, default_locale='en', supported_locales=['en'], website='www.source2.com', description='This is not the first test source' ) kwargs = { 'parent_resource': self.profile } Source.persist_new(source, self.user, **kwargs) source.get_version_model().objects.get(mnemonic='HEAD', versioned_object_id=source.id).delete() self.assertIsNone(SourceVersion.get_head_of(source))
def persist_clone(cls, obj, user=None, prev_latest_version=None, **kwargs): errors = dict() if not user: errors[ 'version_created_by'] = 'Must specify which user is attempting to create a new concept version.' return errors obj.version_created_by = user.username previous_version = obj.previous_version previous_was_latest = previous_version.is_latest_version and obj.is_latest_version source_version = SourceVersion.get_head_of(obj.versioned_object.parent) persisted = False errored_action = 'saving new mapping version' try: obj.save(**kwargs) obj.mnemonic = int(prev_latest_version.mnemonic) + 1 obj.save() errored_action = "updating 'is_latest_version' attribute on previous version" if previous_was_latest: previous_version.is_latest_version = False previous_version.save() errored_action = 'replacing previous version in latest version of source' source_version.update_mapping_version(obj) # Mark versioned object as updated mapping = obj.versioned_object mapping.save() persisted = True finally: if not persisted: source_version.update_mapping_version(obj.previous_version) if previous_was_latest: previous_version.is_latest_version = True previous_version.save() if obj.id: obj.delete() errors['non_field_errors'] = [ 'An error occurred while %s.' % errored_action ] return errors
def persist_clone(cls, obj, user=None, prev_latest_version=None, **kwargs): errors = dict() if not user: errors['version_created_by'] = 'Must specify which user is attempting to create a new concept version.' return errors obj.version_created_by = user.username previous_version = obj.previous_version previous_was_latest = previous_version.is_latest_version and obj.is_latest_version source_version = SourceVersion.get_head_of(obj.versioned_object.parent) persisted = False errored_action = 'saving new mapping version' try: obj.save(**kwargs) obj.mnemonic = int(prev_latest_version.mnemonic) + 1 obj.save() errored_action = "updating 'is_latest_version' attribute on previous version" if previous_was_latest: previous_version.is_latest_version = False previous_version.save() errored_action = 'replacing previous version in latest version of source' source_version.update_mapping_version(obj) # Mark versioned object as updated mapping = obj.versioned_object mapping.save() persisted = True finally: if not persisted: source_version.update_mapping_version(obj.previous_version) if previous_was_latest: previous_version.is_latest_version = True previous_version.save() if obj.id: obj.delete() errors['non_field_errors'] = ['An error occurred while %s.' % errored_action] return errors
def test_get_head_of(self): self.assertEquals(SourceVersion.get_head_of(self.source), self.source.get_version_model().objects.get(mnemonic='HEAD', versioned_object_id=self.source.id))
def import_mappings(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs): initial_signal_processor = haystack.signal_processor try: haystack.signal_processor.teardown() haystack.signal_processor = haystack.signals.BaseSignalProcessor import_start_time = datetime.now() logger.info('Started import at {}'.format(import_start_time.strftime("%Y-%m-%dT%H:%M:%S"))) """ Main mapping importer loop """ logger.info('Import mappings to source...') self.test_mode = test_mode # Retrieve latest source version and, if specified, create a new one self.source_version = SourceVersion.get_head_of(self.source) if new_version: try: new_version = SourceVersion.for_base_object( self.source, new_version, previous_version=self.source_version) new_version.full_clean() new_version.save() new_version.seed_concepts() new_version.seed_mappings() self.source_version = new_version except Exception as exc: raise CommandError('Failed to create new source version due to %s' % exc.args[0]) # Load the JSON file line by line and import each line self.mapping_ids = set(self.source_version.get_mapping_ids()) self.count = 0 for line in self.mappings_file: # Load the next JSON line self.count += 1 data = None try: data = json.loads(line) except ValueError as exc: str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line) self.stderr.write(str_log) logger.warning(str_log) self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP) # Process the import for the current JSON line if data: try: update_action = self.handle_mapping(data) self.count_action(update_action) except IllegalInputException as exc: str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data) self.stderr.write(str_log) logger.warning(str_log) self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP) except InvalidStateException as exc: str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data) self.stderr.write(str_log) logger.warning(str_log) self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP) # Simple progress bars if (self.count % 10) == 0: str_log = ImportActionHelper.get_progress_descriptor( 'mappings', self.count, total, self.action_count) self.stdout.write(str_log, ending='\r') self.stdout.flush() if (self.count % 1000) == 0: logger.info(str_log) # Done with the input file, so close it self.mappings_file.close() # Import complete - display final progress bar str_log = ImportActionHelper.get_progress_descriptor( 'mappings', self.count, total, self.action_count) self.stdout.write(str_log, ending='\r') self.stdout.flush() logger.info(str_log) # Log remaining unhandled IDs str_log = 'Remaining %s unhandled mapping IDs\n' % len(self.mapping_ids) self.stdout.write(str_log) self.stdout.flush() logger.info(str_log) # Deactivate old records if deactivate_old_records: str_log = 'Deactivating old mappings...\n' self.stdout.write(str_log) logger.info(str_log) for mapping_id in self.mapping_ids: try: if self.remove_mapping(mapping_id): self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE) # Log the mapping deactivation str_log = 'Deactivated mapping: %s\n' % mapping_id self.stdout.write(str_log) logger.info(str_log) except InvalidStateException as exc: str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0]) self.stderr.write(str_log) logger.warning(str_log) else: str_log = 'Skipping deactivation loop...\n' self.stdout.write(str_log) logger.info(str_log) # Display final summary str_log = 'Finished importing mappings!\n' self.stdout.write(str_log) logger.info(str_log) str_log = ImportActionHelper.get_progress_descriptor( 'mappings', self.count, total, self.action_count) self.stdout.write(str_log, ending='\r') logger.info(str_log) actions = self.action_count update_index_required = actions.get(ImportActionHelper.IMPORT_ACTION_ADD, 0) > 0 update_index_required |= actions.get(ImportActionHelper.IMPORT_ACTION_UPDATE, 0) > 0 if update_index_required: logger.info('Indexing objects updated since {}'.format(import_start_time.strftime("%Y-%m-%dT%H:%M:%S"))) update_index.Command().handle(start_date=import_start_time.strftime("%Y-%m-%dT%H:%M:%S"), verbosity=2, workers=4, batchsize=100) finally: haystack.signal_processor = initial_signal_processor haystack.signal_processor.setup()