def perform_ingest(ingest_id, mount):
"""Performs the ingest for the given ingest ID
:param ingest_id: The ID of the ingest to perform
:type ingest_id: int
:param mount: The file system to mount in the form of host:/dir/path
:type mount: string
"""
# TODO: refactor to combine _get_ingest(), _get_job_exe_id(), and _set_ingesting_status() in one database
# transaction with as few queries as possible, include retries
ingest = _get_ingest(ingest_id)
job_exe_id = _get_job_exe_id(ingest)
if not os.path.exists(SCALE_INGEST_MOUNT_PATH):
logger.info('Creating %s', SCALE_INGEST_MOUNT_PATH)
os.makedirs(SCALE_INGEST_MOUNT_PATH, mode=0755)
dup_path = os.path.join(SCALE_INGEST_MOUNT_PATH, 'duplicate',
ingest.file_name)
ingest_path = os.path.join(SCALE_INGEST_MOUNT_PATH, ingest.ingest_path)
nfs_mount(mount, SCALE_INGEST_MOUNT_PATH, read_only=False)
try:
# Check condition of the ingest
ingest = _set_ingesting_status(ingest, ingest_path, dup_path)
if ingest is None:
return
logger.info('Storing %s into %s on %s', ingest_path, ingest.file_path,
ingest.workspace.name)
try:
# TODO: future refactor: before copying file, grab existing source file (no lock) or create and save model
# This guarantees that source file exists and can be used to check if file is duplicate
# After this step, the source file should be marked as is_deleted so that it can't be used yet
src_file = SourceFile.objects.store_file(
ingest_path, ingest.get_data_type_tags(), ingest.workspace,
ingest.file_path)
_complete_ingest(ingest, 'INGESTED', src_file)
_delete_ingest_file(ingest_path)
logger.info('Ingest successful: %s', ingest_path)
except DuplicateFile:
logger.warning('Duplicate file detected: %i',
ingest_id,
exc_info=True)
# TODO: future refactor: pass source file model in so source files have duplicate ingests tied to them
_complete_ingest(ingest, 'DUPLICATE', None)
_move_ingest_file(ingest_path, dup_path)
except Exception:
# TODO: have this delete the stored source file using some SourceFile.objects.delete_file method
# TODO: future refactor: pass source file model in so source files have errored ingests tied to them
# TODO: change ERRORED to FAILED
_complete_ingest(ingest, 'ERRORED', None)
raise # File remains where it is so it can be processed again
finally:
nfs_umount(SCALE_INGEST_MOUNT_PATH)
try:
cleanup_job_exe(job_exe_id)
except Exception:
logger.exception('Job Execution %i: Error cleaning up', job_exe_id)
def _cleanup(self, exe_id):
'''Cleans up the work directory for the job. This method is safe and should not throw any exceptions.
'''
try:
cleanup_job_exe(exe_id)
except Exception:
logger.exception('Job Execution %i: Error cleaning up', exe_id)
def _cleanup(self, exe_id):
"""Cleans up the work directory for the job. This method is safe and should not throw any exceptions.
"""
try:
cleanup_job_exe(exe_id)
except Exception:
logger.exception('Job Execution %i: Error cleaning up', exe_id)
def perform_ingest(ingest_id, mount):
"""Performs the ingest for the given ingest ID
:param ingest_id: The ID of the ingest to perform
:type ingest_id: int
:param mount: The file system to mount in the form of host:/dir/path
:type mount: string
"""
# TODO: refactor to combine _get_ingest(), _get_job_exe_id(), and _set_ingesting_status() in one database
# transaction with as few queries as possible, include retries
ingest = _get_ingest(ingest_id)
job_exe_id = _get_job_exe_id(ingest)
if not os.path.exists(SCALE_INGEST_MOUNT_PATH):
logger.info('Creating %s', SCALE_INGEST_MOUNT_PATH)
os.makedirs(SCALE_INGEST_MOUNT_PATH, mode=0755)
dup_path = os.path.join(SCALE_INGEST_MOUNT_PATH, 'duplicate', ingest.file_name)
ingest_path = os.path.join(SCALE_INGEST_MOUNT_PATH, ingest.ingest_path)
nfs_mount(mount, SCALE_INGEST_MOUNT_PATH, read_only=False)
try:
# Check condition of the ingest
ingest = _set_ingesting_status(ingest, ingest_path, dup_path)
if ingest is None:
return
logger.info('Storing %s into %s on %s', ingest_path, ingest.file_path, ingest.workspace.name)
try:
# TODO: future refactor: before copying file, grab existing source file (no lock) or create and save model
# This guarantees that source file exists and can be used to check if file is duplicate
# After this step, the source file should be marked as is_deleted so that it can't be used yet
src_file = SourceFile.objects.store_file(ingest_path, ingest.get_data_type_tags(), ingest.workspace,
ingest.file_path)
_complete_ingest(ingest, 'INGESTED', src_file)
_delete_ingest_file(ingest_path)
logger.info('Ingest successful: %s', ingest_path)
except DuplicateFile:
logger.warning('Duplicate file detected: %i', ingest_id, exc_info=True)
# TODO: future refactor: pass source file model in so source files have duplicate ingests tied to them
_complete_ingest(ingest, 'DUPLICATE', None)
_move_ingest_file(ingest_path, dup_path)
except Exception:
# TODO: have this delete the stored source file using some SourceFile.objects.delete_file method
# TODO: future refactor: pass source file model in so source files have errored ingests tied to them
# TODO: change ERRORED to FAILED
_complete_ingest(ingest, 'ERRORED', None)
raise # File remains where it is so it can be processed again
finally:
nfs_umount(SCALE_INGEST_MOUNT_PATH)
try:
cleanup_job_exe(job_exe_id)
except Exception:
logger.exception('Job Execution %i: Error cleaning up', job_exe_id)
def handle(self, **options):
'''See :meth:`django.core.management.base.BaseCommand.handle`.
This method starts the command.
'''
job_exe_id = options.get('job_exe_id')
logger.info('Command starting: scale_cleanup - Job Execution ID: %i', job_exe_id)
try:
cleanup_job_exe(job_exe_id)
except Exception:
logger.exception('Error performing job execution cleanup')
sys.exit(-1)
logger.info('Command completed successfully: scale_cleanup')
def handle(self, **options):
'''See :meth:`django.core.management.base.BaseCommand.handle`.
This method starts the command.
'''
job_exe_id = options.get('job_exe_id')
logger.info('Command starting: scale_cleanup - Job Execution ID: %i',
job_exe_id)
try:
cleanup_job_exe(job_exe_id)
except Exception:
logger.exception('Error performing job execution cleanup')
sys.exit(-1)
logger.info('Command completed successfully: scale_cleanup')
def _run_processor(self, strike_id, throttle):
'''Runs the given Strike processor
:param strike_id: The ID of the Strike process to run
:type strike_id: int
:param throttle: The minimum delay time in seconds before subsequent reads of the directory
:type throttle: int
'''
strike_proc = None
# TODO: figure out how to guarantee only one Strike process runs at a time
while self.running:
secs_passed = 0
try:
if not strike_proc:
strike_proc = self._init_processor(strike_id)
else:
self._reload_processor(strike_id, strike_proc)
# Process the directory and record number of seconds used
started = now()
strike_proc.mount_and_process_dir()
ended = now()
secs_passed = (ended - started).total_seconds()
except:
logger.exception('Strike processor encountered error.')
finally:
if self.running:
# If process time takes less than user-specified time, throttle
if secs_passed < throttle:
# Delay until full throttle time reached
delay = math.ceil(throttle - secs_passed)
logger.debug('Pausing for %i seconds', delay)
time.sleep(delay)
if self.job_exe_id:
cleanup_job_exe(self.job_exe_id)
logger.info('Strike processor has stopped running')
# TODO: eventually implement a REST API call to permanently stop a Strike process, which should allow this
# command line method to return 0 and complete successfully
sys.exit(1)
def _run_processor(self, strike_id, throttle):
'''Runs the given Strike processor
:param strike_id: The ID of the Strike process to run
:type strike_id: int
:param throttle: The minimum delay time in seconds before subsequent reads of the directory
:type throttle: int
'''
strike_proc = None
# TODO: figure out how to guarantee only one Strike process runs at a time
while self.running:
secs_passed = 0
try:
if not strike_proc:
strike_proc = self._init_processor(strike_id)
else:
self._reload_processor(strike_id, strike_proc)
# Process the directory and record number of seconds used
started = now()
strike_proc.mount_and_process_dir()
ended = now()
secs_passed = (ended - started).total_seconds()
except:
logger.exception('Strike processor encountered error.')
finally:
if self.running:
# If process time takes less than user-specified time, throttle
if secs_passed < throttle:
# Delay until full throttle time reached
delay = math.ceil(throttle - secs_passed)
logger.debug('Pausing for %i seconds', delay)
time.sleep(delay)
if self.job_exe_id:
cleanup_job_exe(self.job_exe_id)
logger.info('Strike processor has stopped running')
# TODO: eventually implement a REST API call to permanently stop a Strike process, which should allow this
# command line method to return 0 and complete successfully
sys.exit(1)
def perform_ingest(ingest_id, mount):
'''Performs the ingest for the given ingest ID
:param ingest_id: The ID of the ingest to perform
:type ingest_id: long
:param mount: The file system to mount in the form of host:/dir/path
:type mount: str
'''
job_exe_id = None
upload_work_dir = None
try:
ingest = Ingest.objects.select_related().get(id=ingest_id)
job_exe_id = JobExecution.objects.get_latest([ingest.job])[ingest.job.id].id
ingest_work_dir = get_ingest_work_dir(job_exe_id)
dup_path = os.path.join(ingest_work_dir, 'duplicate', ingest.file_name)
ingest_path = os.path.join(ingest_work_dir, ingest.ingest_path)
upload_work_dir = os.path.join(os.path.dirname(ingest_path), 'upload', str(ingest_id))
if not os.path.exists(ingest_work_dir):
logger.info('Creating %s', ingest_work_dir)
os.makedirs(ingest_work_dir, mode=0755)
nfs_mount(mount, ingest_work_dir, read_only=False)
# Check condition of the ingest
ingest = _set_ingesting_status(ingest, ingest_path, dup_path)
if ingest is None:
return
logger.info('Storing %s into %s on %s', ingest_path, ingest.file_path, ingest.workspace.name)
try:
src_file = SourceFile.objects.store_file(upload_work_dir, ingest_path, ingest.get_data_type_tags(),
ingest.workspace, ingest.file_path)
# Atomically store file, mark INGESTED, and run ingest trigger rules
with transaction.atomic():
# TODO: It's possible that the file will be successfully moved into the workspace but this database
# transaction might fail. This will result in a file that is in a workspace but doesn't have database
# entries. Attempts to re-ingest will result in duplicate file errors.
logger.info('Marking file as INGESTED: %i', ingest_id)
ingest.source_file = src_file
ingest.status = 'INGESTED'
ingest.ingest_ended = timezone.now()
ingest.save()
logger.debug('Checking ingest trigger rules')
for ingest_rule in get_ingest_rules():
ingest_rule.process_ingest(ingest, src_file.id)
# Delete ingest file
_delete_ingest_file(ingest_path)
logger.info('Ingest successful: %s', ingest_path)
except DuplicateFile:
logger.warning('Duplicate file detected: %i', ingest_id, exc_info=True)
ingest.status = 'DUPLICATE'
ingest.save()
_move_ingest_file(ingest_path, dup_path)
except Exception:
# TODO: have this delete the stored source file using some SourceFile.objects.delete_file method
ingest.status = 'ERRORED'
ingest.save()
raise # File remains where it is so it can be processed again
finally:
try:
if upload_work_dir and os.path.exists(upload_work_dir):
logger.info('Deleting %s', upload_work_dir)
shutil.rmtree(upload_work_dir)
except:
# Swallow exception so error from main try block isn't covered up
logger.exception('Failed to delete upload work dir %s', upload_work_dir)
if job_exe_id:
cleanup_job_exe(job_exe_id)
def perform_ingest(ingest_id, mount):
'''Performs the ingest for the given ingest ID
:param ingest_id: The ID of the ingest to perform
:type ingest_id: long
:param mount: The file system to mount in the form of host:/dir/path
:type mount: str
'''
job_exe_id = None
upload_work_dir = None
try:
# TODO: refactor to combine _get_ingest(), _get_job_exe_id(), and _set_ingesting_status() in one database
# transaction with as few queries as possible, include retries
ingest = _get_ingest(ingest_id)
job_exe_id = _get_job_exe_id(ingest)
create_job_exe_dir(job_exe_id)
ingest_work_dir = get_ingest_work_dir(job_exe_id)
dup_path = os.path.join(ingest_work_dir, 'duplicate', ingest.file_name)
ingest_path = os.path.join(ingest_work_dir, ingest.ingest_path)
upload_work_dir = os.path.join(os.path.dirname(ingest_path), 'upload', str(ingest_id))
if not os.path.exists(ingest_work_dir):
logger.info('Creating %s', ingest_work_dir)
os.makedirs(ingest_work_dir, mode=0755)
nfs_mount(mount, ingest_work_dir, read_only=False)
if not os.path.exists(upload_work_dir):
logger.info('Creating %s', upload_work_dir)
os.makedirs(upload_work_dir, mode=0755)
# Check condition of the ingest
ingest = _set_ingesting_status(ingest, ingest_path, dup_path)
if ingest is None:
return
logger.info('Storing %s into %s on %s', ingest_path, ingest.file_path, ingest.workspace.name)
try:
# TODO: future refactor: before copying file, grab existing source file (no lock) or create and save model
# This guarantees that source file exists and can be used to check if file is duplicate
# After this step, the source file should be marked as is_deleted so that it can't be used yet
src_file = SourceFile.objects.store_file(upload_work_dir, ingest_path, ingest.get_data_type_tags(),
ingest.workspace, ingest.file_path)
_complete_ingest(ingest, 'INGESTED', src_file)
_delete_ingest_file(ingest_path)
logger.info('Ingest successful: %s', ingest_path)
except DuplicateFile:
logger.warning('Duplicate file detected: %i', ingest_id, exc_info=True)
# TODO: future refactor: pass source file model in so source files have duplicate ingests tied to them
_complete_ingest(ingest, 'DUPLICATE', None)
_move_ingest_file(ingest_path, dup_path)
except Exception:
# TODO: have this delete the stored source file using some SourceFile.objects.delete_file method
# TODO: future refactor: pass source file model in so source files have errored ingests tied to them
# TODO: change ERRORED to FAILED
_complete_ingest(ingest, 'ERRORED', None)
raise # File remains where it is so it can be processed again
finally:
try:
# Try to clean up the upload directory
if upload_work_dir and os.path.exists(upload_work_dir):
upload_dir = os.path.join(upload_work_dir, 'upload')
workspace_work_dir = os.path.join(upload_work_dir, 'work')
if os.path.exists(workspace_work_dir):
ScaleFile.objects.cleanup_upload_dir(upload_dir, workspace_work_dir, ingest.workspace)
logger.info('Deleting %s', workspace_work_dir)
os.rmdir(workspace_work_dir)
if os.path.exists(upload_dir):
logger.info('Deleting %s', upload_dir)
# Delete everything in upload dir
shutil.rmtree(upload_dir)
logger.info('Deleting %s', upload_work_dir)
os.rmdir(upload_work_dir)
except:
# Swallow exception so error from main try block isn't covered up
logger.exception('Failed to delete upload work dir %s', upload_work_dir)
try:
if job_exe_id:
cleanup_job_exe(job_exe_id)
except Exception:
logger.exception('Job Execution %i: Error cleaning up', job_exe_id)
