def move_files(self, file_moves):
"""Moves the given files to the new file system paths. Each ScaleFile model should have its related workspace
field populated. This method will update the file_path field in each ScaleFile model to the new path (it may
also change other ScaleFile fields) and save the changes in the database.
:param file_moves: List of files to move
:type file_moves: [:class:`storage.brokers.broker.FileMove`]
:raises :class:`storage.exceptions.ArchivedWorkspace`: If one of the files has a workspace that is archived
:raises :class:`storage.exceptions.DeletedFile`: If one of the files is deleted
:raises :class:`storage.exceptions.MissingRemoteMount`: If a required mount location is missing
"""
wp_dict = {} # {Workspace ID: (workspace, [file move])}
# Organize files by workspace
for file_move in file_moves:
workspace = file_move.file.workspace
if not workspace.is_active:
raise ArchivedWorkspace('%s is no longer active' %
workspace.name)
if file_move.file.is_deleted:
raise DeletedFile(file_move.file.file_name)
if workspace.id in wp_dict:
wp_list = wp_dict[workspace.id][1]
else:
wp_list = []
wp_dict[workspace.id] = (workspace, wp_list)
wp_list.append(file_move)
# Move files for each workspace
for wp_id in wp_dict:
workspace = wp_dict[wp_id][0]
wp_file_moves = wp_dict[wp_id][1]
workspace.move_files(wp_file_moves)
def download_files(self, file_downloads):
"""Downloads the given files to the given local file system paths. Each ScaleFile model should have its related
workspace field populated.
:param file_downloads: List of files to download
:type file_downloads: [:class:`storage.brokers.broker.FileDownload`]
:raises :class:`storage.exceptions.ArchivedWorkspace`: If one of the files has a workspace that is archived
:raises :class:`storage.exceptions.DeletedFile`: If one of the files is deleted
:raises :class:`storage.exceptions.MissingRemoteMount`: If a required mount location is missing
"""
wp_dict = {} # {Workspace ID: (workspace, [file download])}
# Organize files by workspace
for file_download in file_downloads:
workspace = file_download.file.workspace
if not workspace.is_active:
raise ArchivedWorkspace('%s is no longer active' %
workspace.name)
if file_download.file.is_deleted:
raise DeletedFile(file_download.file.file_name)
if workspace.id in wp_dict:
wp_list = wp_dict[workspace.id][1]
else:
wp_list = []
wp_dict[workspace.id] = (workspace, wp_list)
wp_list.append(file_download)
# Download files for each workspace
for wp_id in wp_dict:
workspace = wp_dict[wp_id][0]
wp_file_downloads = wp_dict[wp_id][1]
workspace.download_files(wp_file_downloads)
def delete_files(self, files):
"""Deletes the given files from the remove storage system. Each ScaleFile model should have its related
workspace field populated. This method will update the ScaleFile model and save the changes in the database.
:param files: List of files to delete
:type files: [:class:`storage.models.ScaleFile`]
:raises :class:`storage.exceptions.ArchivedWorkspace`: If one of the files has a workspace that is archived
:raises :class:`storage.exceptions.MissingRemoteMount`: If a required mount location is missing
"""
wp_dict = {} # {Workspace ID: (workspace, [file])}
# Organize files by workspace
for scale_file in files:
workspace = scale_file.workspace
if not workspace.is_active:
raise ArchivedWorkspace('%s is no longer active' %
workspace.name)
if workspace.id in wp_dict:
wp_list = wp_dict[workspace.id][1]
else:
wp_list = []
wp_dict[workspace.id] = (workspace, wp_list)
wp_list.append(scale_file)
# Delete files for each workspace
for wp_id in wp_dict:
workspace = wp_dict[wp_id][0]
wp_file_deletes = wp_dict[wp_id][1]
workspace.delete_files(wp_file_deletes)
def upload_files(self, workspace, file_uploads):
"""Uploads the given files from the given local file system paths into the given workspace. Each ScaleFile model
should have its file_path field populated with the relative location where the file should be stored within the
workspace. This method will update the workspace and other fields (including possibly changing file_path) in
each ScaleFile model and will save the models to the database.
:param workspace: The workspace to upload files into
:type workspace: :class:`storage.models.Workspace`
:param file_uploads: List of files to upload
:type file_uploads: [:class:`storage.brokers.broker.FileUpload`]
:returns: The list of saved file models
:rtype: [:class:`storage.models.ScaleFile`]
:raises :class:`storage.exceptions.ArchivedWorkspace`: If one of the files has a workspace that is archived
:raises :class:`storage.exceptions.MissingRemoteMount`: If a required mount location is missing
"""
if not workspace.is_active:
raise ArchivedWorkspace('%s is no longer active' % workspace.name)
file_list = []
for file_upload in file_uploads:
scale_file = file_upload.file
media_type = scale_file.media_type
# Determine file properties
file_name = os.path.basename(file_upload.local_path)
if not media_type:
media_type = get_media_type(file_name)
file_size = os.path.getsize(file_upload.local_path)
scale_file.file_name = file_name
scale_file.media_type = media_type
scale_file.file_size = file_size
scale_file.workspace = workspace
scale_file.is_deleted = False
scale_file.deleted = None
file_list.append(scale_file)
# Store files in workspace
workspace.upload_files(file_uploads)
# Populate the country list for all files that were saved
for file_upload in file_uploads:
scale_file = file_upload.file
if scale_file.pk:
scale_file.set_countries()
scale_file.save()
return file_list
def download_files(self, download_dir, work_dir, files_to_download):
"""Downloads the given Scale files into the given download directory. After all use of the downloaded files is
complete, the caller should call cleanup_download_dir(). Each ScaleFile model should have its related workspace
field populated.
:param download_dir: Absolute path to the local directory for the files to download
:type download_dir: str
:param work_dir: Absolute path to a local work directory available to assist in downloading. This directory must
not be within the download directory.
:type work_dir: str
:param files_to_download: List of tuples (Scale file, destination path relative to download directory)
:type files_to_download: list of (:class:`storage.models.ScaleFile`, str)
"""
download_dir = os.path.normpath(download_dir)
work_dir = os.path.normpath(work_dir)
# {Workspace ID: (workspace, list of (file_path, local_path))}
wp_dict = {}
# Organize files by workspace
for entry in files_to_download:
scale_file = entry[0]
download_path = entry[1]
workspace_path = scale_file.file_path
workspace = scale_file.workspace
if not workspace.is_active:
raise ArchivedWorkspace('%s is no longer active' %
workspace.name)
if scale_file.is_deleted:
raise DeletedFile('%s has been deleted' % scale_file.file_name)
if workspace.id in wp_dict:
wp_list = wp_dict[workspace.id][1]
else:
wp_list = []
wp_dict[workspace.id] = (workspace, wp_list)
wp_list.append((workspace_path, download_path))
# Retrieve files for each workspace
for wp_id in wp_dict:
workspace = wp_dict[wp_id][0]
download_file_list = wp_dict[wp_id][1]
workspace_work_dir = self._get_workspace_work_dir(
work_dir, workspace)
logger.info('Creating %s', workspace_work_dir)
os.makedirs(workspace_work_dir, mode=0755)
workspace.setup_download_dir(download_dir, workspace_work_dir)
workspace.download_files(download_dir, workspace_work_dir,
download_file_list)
def move_files(self, work_dir, files_to_move):
"""Moves the given Scale files to the new workspace location. Each ScaleFile model should have its related
workspace field populated and the caller must have obtained a model lock on each using select_for_update().
:param work_dir: Absolute path to a local work directory available to assist in moving
:type work_dir: str
:param files_to_move: List of tuples (Scale file, destination workspace path)
:type files_to_move: list of (:class:`storage.models.ScaleFile`, str)
"""
# {Workspace ID: (workspace, list of (workspace_path, new_workspace_path))}
wp_dict = {}
# Organize files by workspace
for entry in files_to_move:
scale_file = entry[0]
new_workspace_path = self._correct_workspace_path(entry[1])
workspace_path = scale_file.file_path
workspace = scale_file.workspace
if not workspace.is_active:
raise ArchivedWorkspace('%s is no longer active' %
workspace.name)
if scale_file.is_deleted:
raise DeletedFile('%s has been deleted' % scale_file.file_name)
if workspace.id in wp_dict:
wp_list = wp_dict[workspace.id][1]
else:
wp_list = []
wp_dict[workspace.id] = (workspace, wp_list)
wp_list.append((workspace_path, new_workspace_path))
# Update workspace path in model
scale_file.file_path = new_workspace_path
scale_file.save()
# Move files for each workspace
for wp_id in wp_dict:
workspace = wp_dict[wp_id][0]
move_file_list = wp_dict[wp_id][1]
workspace_work_dir = self._get_workspace_work_dir(
work_dir, workspace)
logger.info('Creating %s', workspace_work_dir)
os.makedirs(workspace_work_dir, mode=0755)
workspace.move_files(workspace_work_dir, move_file_list)