def handle_lines_in_input_file(self, total):
lines_handled = 0
for line in self.concepts_file:
# Load the next JSON line
lines_handled += 1
data = self.json_to_concept(line) # Process the import for the current JSON line
self.try_import_concept(data)
# Simple progress bar
if (lines_handled % 10) == 0:
log = ImportActionHelper.get_progress_descriptor(
'concepts', lines_handled, total, self.action_count)
self.stdout.write(log, ending='\r')
self.stdout.flush()
if (lines_handled % 1000) == 0:
logger.info(log)
# Done with the input file, so close it
self.concepts_file.close()
if self.validation_logger:
self.validation_logger.close()
# Import complete - display final progress bar
log = ImportActionHelper.get_progress_descriptor(
'concepts', lines_handled, total, self.action_count)
self.info(log, ending='\r', flush=True)
return lines_handled
def handle_lines_in_input_file(self, total):
lines_handled = 0
for line in self.concepts_file:
# Load the next JSON line
lines_handled += 1
data = self.json_to_concept(
line) # Process the import for the current JSON line
self.try_import_concept(data)
# Simple progress bar
if (lines_handled % 100) == 0:
log = ImportActionHelper.get_progress_descriptor(
'concepts', lines_handled, total, self.action_count)
self.stdout.write(log)
self.stdout.flush()
if (lines_handled % 1000) == 0:
logger.info(log)
# Done with the input file, so close it
self.concepts_file.close()
if self.validation_logger:
self.validation_logger.close()
# Import complete - display final progress bar
log = ImportActionHelper.get_progress_descriptor(
'concepts', lines_handled, total, self.action_count)
self.info(log, flush=True)
return lines_handled
def output_summary(self, lines_handled, total='Unknown'):
str_log = 'Finished importing concepts!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', lines_handled, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
def import_mappings(self, new_version=False, total=0, test_mode=False, **kwargs):
""" Main mapping importer loop """
logger.info('Import mappings to source...')
self.test_mode = test_mode
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_latest_version_of(self.source)
if new_version:
try:
new_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_version.seed_concepts()
new_version.seed_mappings()
new_version.full_clean()
new_version.save()
self.source_version = new_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
self.mapping_ids = set(self.source_version.mappings)
self.count = 0
for line in self.mappings_file:
# Load the next JSON line
self.count += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_mapping(data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bars
if (self.count % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (self.count % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.mappings_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining unhandled mapping IDs:\n'
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
str_log = ','.join(str(el) for el in self.mapping_ids)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if kwargs['deactivate_old_records']:
str_log = 'Deactivating old mappings...\n'
self.stdout.write(str_log)
logger.info(str_log)
for mapping_id in self.mapping_ids:
try:
if self.remove_mapping(mapping_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated mapping: %s\n' % mapping_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing mappings!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
def import_mappings(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs):
initial_signal_processor = haystack.signal_processor
try:
haystack.signal_processor.teardown()
haystack.signal_processor = haystack.signals.BaseSignalProcessor
import_start_time = datetime.now()
logger.info('Started import at {}'.format(import_start_time.strftime("%Y-%m-%dT%H:%M:%S")))
""" Main mapping importer loop """
logger.info('Import mappings to source...')
self.test_mode = test_mode
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_head_of(self.source)
if new_version:
try:
new_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_version.full_clean()
new_version.save()
new_version.seed_concepts()
new_version.seed_mappings()
self.source_version = new_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
self.mapping_ids = set(self.source_version.get_mapping_ids())
self.count = 0
for line in self.mappings_file:
# Load the next JSON line
self.count += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_mapping(data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bars
if (self.count % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (self.count % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.mappings_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining %s unhandled mapping IDs\n' % len(self.mapping_ids)
self.stdout.write(str_log)
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old mappings...\n'
self.stdout.write(str_log)
logger.info(str_log)
for mapping_id in self.mapping_ids:
try:
if self.remove_mapping(mapping_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated mapping: %s\n' % mapping_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing mappings!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
actions = self.action_count
update_index_required = actions.get(ImportActionHelper.IMPORT_ACTION_ADD, 0) > 0
update_index_required |= actions.get(ImportActionHelper.IMPORT_ACTION_UPDATE, 0) > 0
if update_index_required:
logger.info('Indexing objects updated since {}'.format(import_start_time.strftime("%Y-%m-%dT%H:%M:%S")))
update_index.Command().handle(start_date=import_start_time.strftime("%Y-%m-%dT%H:%M:%S"), verbosity=2,
workers=4, batchsize=100)
finally:
haystack.signal_processor = initial_signal_processor
haystack.signal_processor.setup()
def import_concepts(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs):
self.action_count = {}
self.test_mode = test_mode
logger.info('Import concepts to source...')
if new_version:
try:
new_source_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_source_version.seed_concepts()
new_source_version.seed_mappings()
new_source_version.full_clean()
new_source_version.save()
self.source_version = new_source_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
total = total or '(Unknown)'
self.user = User.objects.filter(is_superuser=True)[0]
self.concept_version_ids = set(self.source_version.concepts)
cnt = 0
self.create_concept_versions_map()
for line in self.concepts_file:
# Load the next JSON line
cnt += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_concept(self.source, data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s\nFailed to parse line: %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bar
if (cnt % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (cnt % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.concepts_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining unhandled concept versions:\n'
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
str_log = ','.join(str(el) for el in self.concept_version_ids)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old concepts...\n'
self.stdout.write(str_log)
logger.info(str_log)
for version_id in self.concept_version_ids:
try:
if self.remove_concept_version(version_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated concept version: %s\n' % version_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate concept version on ID %s! %s\n' % (version_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing concepts!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
def output_summary(self, lines_handled, total='Unknown'):
self.info('Finished importing concepts!\n')
log = ImportActionHelper.get_progress_descriptor(
'concepts', lines_handled, total, self.action_count)
self.info(log)
def import_mappings(self, new_version=False, total=0, test_mode=False, deactivate_old_records=False, **kwargs):
""" Main mapping importer loop """
logger.info('Import mappings to source...')
self.test_mode = test_mode
# Retrieve latest source version and, if specified, create a new one
self.source_version = SourceVersion.get_latest_version_of(self.source)
if new_version:
try:
new_version = SourceVersion.for_base_object(
self.source, new_version, previous_version=self.source_version)
new_version.seed_concepts()
new_version.seed_mappings()
new_version.full_clean()
new_version.save()
self.source_version = new_version
except Exception as exc:
raise CommandError('Failed to create new source version due to %s' % exc.args[0])
# Load the JSON file line by line and import each line
self.mapping_ids = set(self.source_version.mappings)
self.count = 0
concepts_for_uris = Concept.objects.filter(parent_id=self.source.id)
self.concepts_cache = dict((x.uri, x) for x in concepts_for_uris)
for line in self.mappings_file:
# Load the next JSON line
self.count += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_mapping(data)
self.count_action(update_action)
except IllegalInputException as exc:
str_log = '%s, failed to parse line %s. Skipping it...\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
except InvalidStateException as exc:
str_log = 'Source is in an invalid state!\n%s\n%s\n' % (exc.args[0], data)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Simple progress bars
if (self.count % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (self.count % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.mappings_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining unhandled mapping IDs:\n'
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
str_log = ','.join(str(el) for el in self.mapping_ids)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old mappings...\n'
self.stdout.write(str_log)
logger.info(str_log)
for mapping_id in self.mapping_ids:
try:
if self.remove_mapping(mapping_id):
self.count_action(ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated mapping: %s\n' % mapping_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate mapping on ID %s! %s\n' % (mapping_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing mappings!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'mappings', self.count, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
def import_concepts(self,
new_version=False,
total=0,
test_mode=False,
deactivate_old_records=False,
**kwargs):
self.action_count = {}
self.test_mode = test_mode
logger.info('Import concepts to source...')
if new_version:
try:
self.create_new_source_version(new_version)
except Exception as exc:
raise CommandError(
'Failed to create new source version due to %s' %
exc.args[0])
# Load the JSON file line by line and import each line
total = total or '(Unknown)'
self.user = User.objects.filter(is_superuser=True)[0]
self.concept_version_ids = set(self.source_version.concepts)
cnt = 0
self.create_concept_versions_map()
for line in self.concepts_file:
# Load the next JSON line
cnt += 1
data = None
try:
data = json.loads(line)
except ValueError as exc:
str_log = 'Skipping invalid JSON line: %s. JSON: %s\n' % (
exc.args[0], line)
self.stderr.write(str_log)
logger.warning(str_log)
self.count_action(ImportActionHelper.IMPORT_ACTION_SKIP)
# Process the import for the current JSON line
if data:
try:
update_action = self.handle_concept(self.source, data)
self.count_action(update_action)
except IllegalInputException as exc:
exc_message = '%s\nFailed to parse line: %s. Skipping it...\n' % (
exc.args[0], data)
self.handle_exception(exc_message)
except InvalidStateException as exc:
exc_message = 'Source is in an invalid state!\n%s\n%s\n' % (
exc.args[0], data)
self.handle_exception(exc_message)
except ValidationError as exc:
exc_message = '%s\nValidation failed: %s. Skipping it...\n' % (
''.join(exc.messages), data)
self.handle_exception(exc_message)
except Exception as exc:
exc_message = '%s\nUnexpected something occured: %s. Skipping it...\n' % (
exc.message, data)
self.handle_exception(exc_message)
# Simple progress bar
if (cnt % 10) == 0:
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
if (cnt % 1000) == 0:
logger.info(str_log)
# Done with the input file, so close it
self.concepts_file.close()
# Import complete - display final progress bar
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Log remaining unhandled IDs
str_log = 'Remaining unhandled concept versions:\n'
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
str_log = ','.join(str(el) for el in self.concept_version_ids)
self.stdout.write(str_log, ending='\r')
self.stdout.flush()
logger.info(str_log)
# Deactivate old records
if deactivate_old_records:
str_log = 'Deactivating old concepts...\n'
self.stdout.write(str_log)
logger.info(str_log)
for version_id in self.concept_version_ids:
try:
if self.remove_concept_version(version_id):
self.count_action(
ImportActionHelper.IMPORT_ACTION_DEACTIVATE)
# Log the mapping deactivation
str_log = 'Deactivated concept version: %s\n' % version_id
self.stdout.write(str_log)
logger.info(str_log)
except InvalidStateException as exc:
str_log = 'Failed to inactivate concept version on ID %s! %s\n' % (
version_id, exc.args[0])
self.stderr.write(str_log)
logger.warning(str_log)
else:
str_log = 'Skipping deactivation loop...\n'
self.stdout.write(str_log)
logger.info(str_log)
# Display final summary
str_log = 'Finished importing concepts!\n'
self.stdout.write(str_log)
logger.info(str_log)
str_log = ImportActionHelper.get_progress_descriptor(
'concepts', cnt, total, self.action_count)
self.stdout.write(str_log, ending='\r')
logger.info(str_log)
def output_summary(self, lines_handled, total='Unknown'):
self.info('Finished importing concepts!\n')
log = ImportActionHelper.get_progress_descriptor(
'concepts', lines_handled, total, self.action_count)
self.info(log)
