def cleanup_upload_dir(self, upload_dir, work_dir, workspace):
"""Performs any cleanup necessary for a previous setup_upload_dir() call
:param upload_dir: Absolute path to the local directory of the files to upload
:type upload_dir: str
:param work_dir: Absolute path to a local work directory available to assist in uploading
:type work_dir: str
:param workspace: The workspace to upload files into
:type workspace: :class:`storage.models.Workspace`
"""
upload_dir = os.path.normpath(upload_dir)
work_dir = os.path.normpath(work_dir)
delete_root_dir = self._get_delete_root_dir(work_dir)
delete_work_dir = self._get_delete_work_dir(work_dir, workspace)
workspace_root_dir = self._get_workspace_root_dir(work_dir)
workspace_work_dir = self._get_workspace_work_dir(work_dir, workspace)
if os.path.exists(workspace_work_dir):
workspace.cleanup_upload_dir(upload_dir, workspace_work_dir)
logger.info('Deleting %s', workspace_work_dir)
os.rmdir(workspace_work_dir)
if not os.listdir(workspace_root_dir):
logger.info('Deleting %s', workspace_root_dir)
os.rmdir(workspace_root_dir)
if os.path.exists(delete_work_dir):
nfs_umount(delete_work_dir)
logger.info('Deleting %s', delete_work_dir)
os.rmdir(delete_work_dir)
if not os.listdir(delete_root_dir):
logger.info('Deleting %s', delete_root_dir)
os.rmdir(delete_root_dir)
def cleanup_upload_dir(self, upload_dir, work_dir, workspace):
"""Performs any cleanup necessary for a previous setup_upload_dir() call
:param upload_dir: Absolute path to the local directory of the files to upload
:type upload_dir: str
:param work_dir: Absolute path to a local work directory available to assist in uploading
:type work_dir: str
:param workspace: The workspace to upload files into
:type workspace: :class:`storage.models.Workspace`
"""
upload_dir = os.path.normpath(upload_dir)
work_dir = os.path.normpath(work_dir)
delete_root_dir = self._get_delete_root_dir(work_dir)
delete_work_dir = self._get_delete_work_dir(work_dir, workspace)
workspace_root_dir = self._get_workspace_root_dir(work_dir)
workspace_work_dir = self._get_workspace_work_dir(work_dir, workspace)
if os.path.exists(workspace_work_dir):
workspace.cleanup_upload_dir(upload_dir, workspace_work_dir)
logger.info('Deleting %s', workspace_work_dir)
os.rmdir(workspace_work_dir)
if not os.listdir(workspace_root_dir):
logger.info('Deleting %s', workspace_root_dir)
os.rmdir(workspace_root_dir)
if os.path.exists(delete_work_dir):
nfs_umount(delete_work_dir)
logger.info('Deleting %s', delete_work_dir)
os.rmdir(delete_work_dir)
if not os.listdir(delete_root_dir):
logger.info('Deleting %s', delete_root_dir)
os.rmdir(delete_root_dir)
def move_files(self, work_dir, files_to_move):
"""See :meth:`storage.brokers.broker.Broker.move_files`
"""
nfs_mount(self.mount, work_dir, False)
try:
for file_to_move in files_to_move:
old_workspace_path = file_to_move[0]
new_workspace_path = file_to_move[1]
full_old_workspace_path = os.path.join(work_dir,
old_workspace_path)
full_new_workspace_path = os.path.join(work_dir,
new_workspace_path)
full_new_workspace_dir = os.path.dirname(
full_new_workspace_path)
if not os.path.exists(full_new_workspace_dir):
logger.info('Creating %s', full_new_workspace_dir)
os.makedirs(full_new_workspace_dir, mode=0755)
logger.info('Moving %s to %s', full_old_workspace_path,
full_new_workspace_path)
shutil.move(full_old_workspace_path, full_new_workspace_path)
os.chmod(full_new_workspace_path, 0644)
finally:
nfs_umount(work_dir)
def perform_ingest(ingest_id, mount):
"""Performs the ingest for the given ingest ID
:param ingest_id: The ID of the ingest to perform
:type ingest_id: int
:param mount: The file system to mount in the form of host:/dir/path
:type mount: string
"""
# TODO: refactor to combine _get_ingest(), _get_job_exe_id(), and _set_ingesting_status() in one database
# transaction with as few queries as possible, include retries
ingest = _get_ingest(ingest_id)
job_exe_id = _get_job_exe_id(ingest)
if not os.path.exists(SCALE_INGEST_MOUNT_PATH):
logger.info('Creating %s', SCALE_INGEST_MOUNT_PATH)
os.makedirs(SCALE_INGEST_MOUNT_PATH, mode=0755)
dup_path = os.path.join(SCALE_INGEST_MOUNT_PATH, 'duplicate',
ingest.file_name)
ingest_path = os.path.join(SCALE_INGEST_MOUNT_PATH, ingest.ingest_path)
nfs_mount(mount, SCALE_INGEST_MOUNT_PATH, read_only=False)
try:
# Check condition of the ingest
ingest = _set_ingesting_status(ingest, ingest_path, dup_path)
if ingest is None:
return
logger.info('Storing %s into %s on %s', ingest_path, ingest.file_path,
ingest.workspace.name)
try:
# TODO: future refactor: before copying file, grab existing source file (no lock) or create and save model
# This guarantees that source file exists and can be used to check if file is duplicate
# After this step, the source file should be marked as is_deleted so that it can't be used yet
src_file = SourceFile.objects.store_file(
ingest_path, ingest.get_data_type_tags(), ingest.workspace,
ingest.file_path)
_complete_ingest(ingest, 'INGESTED', src_file)
_delete_ingest_file(ingest_path)
logger.info('Ingest successful: %s', ingest_path)
except DuplicateFile:
logger.warning('Duplicate file detected: %i',
ingest_id,
exc_info=True)
# TODO: future refactor: pass source file model in so source files have duplicate ingests tied to them
_complete_ingest(ingest, 'DUPLICATE', None)
_move_ingest_file(ingest_path, dup_path)
except Exception:
# TODO: have this delete the stored source file using some SourceFile.objects.delete_file method
# TODO: future refactor: pass source file model in so source files have errored ingests tied to them
# TODO: change ERRORED to FAILED
_complete_ingest(ingest, 'ERRORED', None)
raise # File remains where it is so it can be processed again
finally:
nfs_umount(SCALE_INGEST_MOUNT_PATH)
try:
cleanup_job_exe(job_exe_id)
except Exception:
logger.exception('Job Execution %i: Error cleaning up', job_exe_id)
def cleanup_download_dir(self, download_dir, work_dir):
'''See :meth:`storage.brokers.broker.Broker.cleanup_download_dir`
'''
if os.path.exists(download_dir):
logger.info('Deleting %s', download_dir)
shutil.rmtree(download_dir)
nfs_umount(work_dir)
def cleanup_job_execution(self, job_exe):
'''See :meth:`job.execution.job_exe_cleaner.JobExecutionCleaner.cleanup_job_execution`
'''
logger.info('Cleaning up a Strike job')
ingest_work_dir = get_ingest_work_dir(job_exe.id)
if os.path.exists(ingest_work_dir):
nfs_umount(ingest_work_dir)
def perform_ingest(ingest_id, mount):
"""Performs the ingest for the given ingest ID
:param ingest_id: The ID of the ingest to perform
:type ingest_id: int
:param mount: The file system to mount in the form of host:/dir/path
:type mount: string
"""
# TODO: refactor to combine _get_ingest(), _get_job_exe_id(), and _set_ingesting_status() in one database
# transaction with as few queries as possible, include retries
ingest = _get_ingest(ingest_id)
job_exe_id = _get_job_exe_id(ingest)
if not os.path.exists(SCALE_INGEST_MOUNT_PATH):
logger.info('Creating %s', SCALE_INGEST_MOUNT_PATH)
os.makedirs(SCALE_INGEST_MOUNT_PATH, mode=0755)
dup_path = os.path.join(SCALE_INGEST_MOUNT_PATH, 'duplicate', ingest.file_name)
ingest_path = os.path.join(SCALE_INGEST_MOUNT_PATH, ingest.ingest_path)
nfs_mount(mount, SCALE_INGEST_MOUNT_PATH, read_only=False)
try:
# Check condition of the ingest
ingest = _set_ingesting_status(ingest, ingest_path, dup_path)
if ingest is None:
return
logger.info('Storing %s into %s on %s', ingest_path, ingest.file_path, ingest.workspace.name)
try:
# TODO: future refactor: before copying file, grab existing source file (no lock) or create and save model
# This guarantees that source file exists and can be used to check if file is duplicate
# After this step, the source file should be marked as is_deleted so that it can't be used yet
src_file = SourceFile.objects.store_file(ingest_path, ingest.get_data_type_tags(), ingest.workspace,
ingest.file_path)
_complete_ingest(ingest, 'INGESTED', src_file)
_delete_ingest_file(ingest_path)
logger.info('Ingest successful: %s', ingest_path)
except DuplicateFile:
logger.warning('Duplicate file detected: %i', ingest_id, exc_info=True)
# TODO: future refactor: pass source file model in so source files have duplicate ingests tied to them
_complete_ingest(ingest, 'DUPLICATE', None)
_move_ingest_file(ingest_path, dup_path)
except Exception:
# TODO: have this delete the stored source file using some SourceFile.objects.delete_file method
# TODO: future refactor: pass source file model in so source files have errored ingests tied to them
# TODO: change ERRORED to FAILED
_complete_ingest(ingest, 'ERRORED', None)
raise # File remains where it is so it can be processed again
finally:
nfs_umount(SCALE_INGEST_MOUNT_PATH)
try:
cleanup_job_exe(job_exe_id)
except Exception:
logger.exception('Job Execution %i: Error cleaning up', job_exe_id)
def cleanup_job_execution(self, job_exe):
'''See :meth:`job.execution.job_exe_cleaner.JobExecutionCleaner.cleanup_job_execution`
'''
logger.info('Cleaning up a Strike job')
ingest_work_dir = get_ingest_work_dir(job_exe.id)
if os.path.exists(ingest_work_dir):
nfs_umount(ingest_work_dir)
logger.info('Deleting %s', ingest_work_dir)
os.rmdir(ingest_work_dir)
def delete_files(self, work_dir, workspace_paths):
'''See :meth:`storage.brokers.broker.Broker.delete_files`
'''
nfs_mount(self.mount, work_dir, False)
try:
for workspace_path in workspace_paths:
path_to_delete = os.path.join(work_dir, workspace_path)
if os.path.exists(path_to_delete):
logger.info('Deleting %s', path_to_delete)
os.remove(path_to_delete)
finally:
nfs_umount(work_dir)
def delete_files(self, work_dir, workspace_paths):
"""See :meth:`storage.brokers.broker.Broker.delete_files`
"""
nfs_mount(self.mount, work_dir, False)
try:
for workspace_path in workspace_paths:
path_to_delete = os.path.join(work_dir, workspace_path)
if os.path.exists(path_to_delete):
logger.info('Deleting %s', path_to_delete)
os.remove(path_to_delete)
finally:
nfs_umount(work_dir)
def mount_and_process_dir(self):
'''Mounts NFS and processes the current files in the Strike directory
'''
try:
if not os.path.exists(self.strike_dir):
logger.info('Creating %s', self.strike_dir)
os.makedirs(self.strike_dir, mode=0755)
nfs_mount(self.mount, self.strike_dir, read_only=False)
self._init_dirs()
self._process_dir()
except Exception:
logger.exception('Strike processor encountered error.')
finally:
nfs_umount(self.strike_dir)
def mount_and_process_dir(self):
'''Mounts NFS and processes the current files in the Strike directory
'''
try:
if not os.path.exists(self.strike_dir):
logger.info('Creating %s', self.strike_dir)
os.makedirs(self.strike_dir, mode=0755)
nfs_mount(self.mount, self.strike_dir, read_only=False)
self._init_dirs()
self._process_dir()
except Exception:
logger.exception('Strike processor encountered error.')
finally:
nfs_umount(self.strike_dir)
def upload_files(self, upload_dir, work_dir, files_to_upload):
'''See :meth:`storage.brokers.broker.Broker.setup_upload_dir`
'''
nfs_mount(self.mount, work_dir, False)
try:
for file_to_upload in files_to_upload:
src_path = file_to_upload[0]
workspace_path = file_to_upload[1]
full_src_path = os.path.join(upload_dir, src_path)
full_workspace_path = os.path.join(work_dir, workspace_path)
full_workspace_dir = os.path.dirname(full_workspace_path)
if not os.path.exists(full_workspace_dir):
logger.info('Creating %s', full_workspace_dir)
os.makedirs(full_workspace_dir, mode=0755)
self._copy_file(full_src_path, full_workspace_path)
finally:
nfs_umount(work_dir)
def upload_files(self, upload_dir, work_dir, files_to_upload):
"""See :meth:`storage.brokers.broker.Broker.setup_upload_dir`
"""
nfs_mount(self.mount, work_dir, False)
try:
for file_to_upload in files_to_upload:
src_path = file_to_upload[0]
workspace_path = file_to_upload[1]
full_src_path = os.path.join(upload_dir, src_path)
full_workspace_path = os.path.join(work_dir, workspace_path)
full_workspace_dir = os.path.dirname(full_workspace_path)
if not os.path.exists(full_workspace_dir):
logger.info('Creating %s', full_workspace_dir)
os.makedirs(full_workspace_dir, mode=0755)
self._copy_file(full_src_path, full_workspace_path)
os.chmod(full_workspace_path, 0644)
finally:
nfs_umount(work_dir)
def cleanup_move_dir(self, work_dir):
"""Performs any cleanup necessary for a previous move_files() call
:param work_dir: Absolute path to a local work directory available to assist in moving
:type work_dir: str
:param workspace: The workspace to upload files into
:type workspace: :class:`storage.models.Workspace`
"""
work_dir = os.path.normpath(work_dir)
workspace_root_dir = self._get_workspace_root_dir(work_dir)
if os.path.exists(workspace_root_dir):
for name in os.listdir(workspace_root_dir):
sub_dir = os.path.join(workspace_root_dir, name)
nfs_umount(sub_dir)
logger.info('Deleting %s', sub_dir)
os.rmdir(sub_dir)
logger.info('Deleting %s', workspace_root_dir)
os.rmdir(workspace_root_dir)
def cleanup_move_dir(self, work_dir):
"""Performs any cleanup necessary for a previous move_files() call
:param work_dir: Absolute path to a local work directory available to assist in moving
:type work_dir: str
:param workspace: The workspace to upload files into
:type workspace: :class:`storage.models.Workspace`
"""
work_dir = os.path.normpath(work_dir)
workspace_root_dir = self._get_workspace_root_dir(work_dir)
if os.path.exists(workspace_root_dir):
for name in os.listdir(workspace_root_dir):
sub_dir = os.path.join(workspace_root_dir, name)
nfs_umount(sub_dir)
logger.info('Deleting %s', sub_dir)
os.rmdir(sub_dir)
logger.info('Deleting %s', workspace_root_dir)
os.rmdir(workspace_root_dir)
def move_files(self, work_dir, files_to_move):
'''See :meth:`storage.brokers.broker.Broker.move_files`
'''
nfs_mount(self.mount, work_dir, False)
try:
for file_to_move in files_to_move:
old_workspace_path = file_to_move[0]
new_workspace_path = file_to_move[1]
full_old_workspace_path = os.path.join(work_dir, old_workspace_path)
full_new_workspace_path = os.path.join(work_dir, new_workspace_path)
full_new_workspace_dir = os.path.dirname(full_new_workspace_path)
if not os.path.exists(full_new_workspace_dir):
logger.info('Creating %s', full_new_workspace_dir)
os.makedirs(full_new_workspace_dir, mode=0755)
logger.info('Moving %s to %s', full_old_workspace_path, full_new_workspace_path)
shutil.move(full_old_workspace_path, full_new_workspace_path)
finally:
nfs_umount(work_dir)
def cleanup_upload_dir(self, upload_dir, work_dir):
'''See :meth:`storage.brokers.broker.Broker.cleanup_upload_dir`
'''
nfs_umount(work_dir)
def test_umount_if_not_mounted(self):
'''Tests unmounting a location that isn't currently mounted to ensure there isn't an error.'''
nfs_umount(self.mntdir) # should not throw an exception because the mount location isn't mounted
def cleanup_upload_dir(self, upload_dir, work_dir):
"""See :meth:`storage.brokers.broker.Broker.cleanup_upload_dir`
"""
nfs_umount(work_dir)
def cleanup_download_dir(self, download_dir, work_dir):
"""See :meth:`storage.brokers.broker.Broker.cleanup_download_dir`
"""
nfs_umount(work_dir)
