def _copy_file(self, src_path, dest_path):
"""Performs a copy from the src_path to the dest_path
:param src_path: The absolute path to the source file
:type src_path: str
:param dest_path: The absolute path to the destination
:type dest_path: str
"""
if os.path.islink(src_path):
real_path = os.path.realpath(src_path)
logger.info('%s is a link to %s', src_path, real_path)
src_path = real_path
logger.info('Copying %s to %s', src_path, dest_path)
# Attempt bbcp copy first. If it fails, we'll fallback to cp
try:
# TODO: detect bbcp location instead of assuming /usr/local and don't even try to execute if it isn't
# installed
# TODO: configuration options for the bbcp copy options such as window size
srv_src_path, srv_dest_path = self._get_mount_info(src_path, dest_path)
cmd_list = ['/usr/local/bin/bbcp',
'-s', '8', '-w', '64M', '-E', 'md5', '-o', '-y', 'd',
apply(os.path.join, srv_src_path) if srv_src_path[0] is not None else srv_src_path[1],
apply(os.path.join, srv_dest_path) if srv_dest_path[0] is not None else srv_dest_path[1]]
execute_command_line(cmd_list)
return
except OSError as e:
# errno 2 is No such file or directory..bbcp not installed. We'll be quiet about it but fallback
if e.errno != 2:
logger.exception("NFS Broker bbcp copy_file") # Ignore the error and attempt a regular cp
except:
logger.exception("NFS Broker bbcp copy_file") # Ignore the error and attempt a regular cp
logger.info('Fall back to cp for %s', src_path)
shutil.copy(src_path, dest_path)
def download_files(self, volume_path, file_downloads):
"""See :meth:`storage.brokers.broker.Broker.download_files`"""
with S3Client(self._credentials, self._region_name) as client:
for file_download in file_downloads:
# If file supports partial mount and volume is configured attempt sym-link
if file_download.partial and self._volume:
logger.debug('Partial S3 file accessed by mounted bucket.')
path_to_download = os.path.join(
volume_path, file_download.file.file_path)
logger.info('Checking path %s', path_to_download)
if not os.path.exists(path_to_download):
raise MissingFile(file_download.file.file_name)
# Create symlink to the file in the host mount
logger.info('Creating link %s -> %s',
file_download.local_path, path_to_download)
execute_command_line([
'ln', '-s', path_to_download, file_download.local_path
])
# Fall-back to default S3 file download
else:
try:
s3_object = client.get_object(
self._bucket_name, file_download.file.file_path)
except FileDoesNotExist:
raise MissingFile(file_download.file.file_name)
self._download_file(s3_object, file_download.file,
file_download.local_path)
def _copy_file(self, src_path, dest_path):
'''Performs a copy from the src_path to the dest_path
:param src_path: The absolute path to the source file
:type src_path: str
:param dest_path: The absolute path to the destination
:type dest_path: str
'''
logger.info('Copying %s to %s', src_path, dest_path)
# attempt bbcp copy first. If it fails, we'll fallback to cp
try:
# TODO: detect bbcp location instead of assuming /usr/local and don't even try to execute if it isn't installed
# TODO: configuration options for the bbcp copy options such as window size
srv_src_path, srv_dest_path = self._get_mount_info(src_path, dest_path)
cmd_list = ['/usr/local/bin/bbcp',
'-s', '8', '-w', '64M', '-E', 'md5', '-o', '-y', 'd',
apply(os.path.join, srv_src_path) if srv_src_path[0] is not None else srv_src_path[1],
apply(os.path.join, srv_dest_path) if srv_dest_path[0] is not None else srv_dest_path[1]]
execute_command_line(cmd_list)
return
except:
logger.exception("NFS Broker bbcp copy_file") # ignore the error and attempt a regular cp
logger.info('Fall back to cp for %s', src_path)
shutil.copy(src_path, dest_path)
def _copy_file(self, src_path, dest_path):
"""Performs a copy from the src_path to the dest_path
:param src_path: The absolute path to the source file
:type src_path: str
:param dest_path: The absolute path to the destination
:type dest_path: str
"""
if os.path.islink(src_path):
real_path = os.path.realpath(src_path)
logger.info('%s is a link to %s', src_path, real_path)
src_path = real_path
logger.info('Copying %s to %s', src_path, dest_path)
# attempt bbcp copy first. If it fails, we'll fallback to cp
try:
# TODO: detect bbcp location instead of assuming /usr/local and don't even try to execute if it isn't installed
# TODO: configuration options for the bbcp copy options such as window size
srv_src_path, srv_dest_path = self._get_mount_info(
src_path, dest_path)
cmd_list = [
'/usr/local/bin/bbcp', '-s', '8', '-w', '64M', '-E', 'md5',
'-o', '-y', 'd',
apply(os.path.join, srv_src_path)
if srv_src_path[0] is not None else srv_src_path[1],
apply(os.path.join, srv_dest_path)
if srv_dest_path[0] is not None else srv_dest_path[1]
]
execute_command_line(cmd_list)
return
except OSError, e:
if e.errno != 2: # errno 2 is No such file or directory..bbcp not installed. We'll be quiet about it but fallback
logger.exception("NFS Broker bbcp copy_file"
) # ignore the error and attempt a regular cp
def download_files(self, volume_path, file_downloads):
"""See :meth:`storage.brokers.broker.Broker.download_files`
"""
for file_download in file_downloads:
path_to_download = os.path.join(volume_path, file_download.file.file_path)
# Create symlink to the file in the host mount
logger.info('Creating link %s -> %s', file_download.local_path, path_to_download)
execute_command_line(['ln', '-s', path_to_download, file_download.local_path])
def store_file(self, work_dir, local_path, data_types, workspace, remote_path):
'''Stores the given local source file in the workspace
:param work_dir: Absolute path to a local work directory available to assist in storing the source file
:type work_dir: str
:param local_path: The absolute local path of the source file to store
:type local_path: str
:param data_types: The data type tags of the source file
:type data_types: list of str
:param workspace: The workspace to use for storing the source file
:type workspace: :class:`storage.models.Workspace`
:param remote_path: The relative path for storing the source file
:type remote_path: str
:returns: The model of the saved source file
:rtype: :class:`source.models.SourceFile`
'''
file_name = os.path.basename(local_path)
upload_dir = os.path.join(work_dir, 'upload')
workspace_work_dir = os.path.join(work_dir, 'work')
if not os.path.exists(upload_dir):
logger.info('Creating %s', upload_dir)
os.mkdir(upload_dir, 0755)
if not os.path.exists(workspace_work_dir):
logger.info('Creating %s', workspace_work_dir)
os.mkdir(workspace_work_dir, 0755)
upload_file_path = os.path.join(upload_dir, file_name)
ScaleFile.objects.setup_upload_dir(upload_dir, workspace_work_dir, workspace)
try:
# Check for duplicate file, else create new file
# TODO: fix race condition with many files with same name?
try:
src_file = SourceFile.objects.get(file_name=file_name)
# Duplicate files that are deleted should be stored again
if not src_file.is_deleted:
raise DuplicateFile(u'\'%s\' already exists' % file_name)
except SourceFile.DoesNotExist:
src_file = SourceFile() # New file
# Add a stable identifier based on the file name
src_file.update_uuid(file_name)
# Add tags and store the new/updated source file
for tag in data_types:
src_file.add_data_type_tag(tag)
# Link source file into upload directory and upload it
if not os.path.islink(upload_file_path):
logger.info('Creating link %s -> %s', upload_file_path, local_path)
execute_command_line(['ln', '-s', local_path, upload_file_path])
return ScaleFile.objects.upload_files(upload_dir, workspace_work_dir, workspace,
[(src_file, file_name, remote_path)])[0]
finally:
ScaleFile.objects.cleanup_upload_dir(upload_dir, workspace_work_dir, workspace)
def download_files(self, volume_path, file_downloads):
"""See :meth:`storage.brokers.broker.Broker.download_files`
"""
for file_download in file_downloads:
path_to_download = os.path.join(volume_path,
file_download.file.file_path)
# Create symlink to the file in the host mount
logger.info('Creating link %s -> %s', file_download.local_path,
path_to_download)
execute_command_line(
['ln', '-s', path_to_download, file_download.local_path])
def download_files(self, download_dir, work_dir, files_to_download):
'''See :meth:`storage.brokers.broker.Broker.download_files`
'''
for file_to_download in files_to_download:
workspace_path = file_to_download[0]
dest_path = file_to_download[1]
full_workspace_path = os.path.join(work_dir, workspace_path)
full_dest_path = os.path.join(download_dir, dest_path)
full_dest_dir = os.path.dirname(full_dest_path)
if not os.path.exists(full_dest_dir):
logger.info('Creating %s', full_dest_dir)
os.makedirs(full_dest_dir, mode=0755)
execute_command_line(['ln', '-s', full_workspace_path, full_dest_path])
def download_files(self, download_dir, work_dir, files_to_download):
"""See :meth:`storage.brokers.broker.Broker.download_files`
"""
for file_to_download in files_to_download:
workspace_path = file_to_download[0]
dest_path = file_to_download[1]
full_workspace_path = os.path.join(work_dir, workspace_path)
full_dest_path = os.path.join(download_dir, dest_path)
full_dest_dir = os.path.dirname(full_dest_path)
if not os.path.exists(full_dest_dir):
logger.info('Creating %s', full_dest_dir)
os.makedirs(full_dest_dir, mode=0755)
execute_command_line(
['ln', '-s', full_workspace_path, full_dest_path])
def download_files(self, volume_path, file_downloads):
"""See :meth:`storage.brokers.broker.Broker.download_files`"""
with S3Client(self._credentials, self._region_name) as client:
for file_download in file_downloads:
# If file supports partial mount and volume is configured attempt sym-link
if file_download.partial and self._volume:
logger.debug('Partial S3 file accessed by mounted bucket.')
path_to_download = os.path.join(volume_path, file_download.file.file_path)
# Create symlink to the file in the host mount
logger.info('Creating link %s -> %s', file_download.local_path, path_to_download)
execute_command_line(['ln', '-s', path_to_download, file_download.local_path])
# Fall-back to default S3 file download
else:
s3_object = client.get_object(self._bucket_name, file_download.file.file_path)
self._download_file(s3_object, file_download.file, file_download.local_path)
def nfs_mount(mount, mount_on, read_only=True):
'''Performs a mount of a network file system
:param mount: The network file system to mount in the form of host:/dir/path
:type mount: str
:param mount_on: The absolute directory path to mount on (must already exist)
:type mount_on: str
:param read_only: Whether the mount should be read-only
:type read_only: bool
'''
logger.info('Mounting %s on %s', mount, mount_on)
options = 'soft,' + ('ro' if read_only else 'rw') + ',lookupcache=positive'
cmd_list = ['sudo', 'mount', '-o', options, mount, mount_on]
try:
execute_command_line(cmd_list)
except Exception as ex:
raise NfsError(ex)
def nfs_umount(mounted_on):
'''Performs a umount of a network file system
:param mounted_on: The absolute directory path of the mounted file system
:type mounted_on: str
'''
logger.info('Unmounting %s', mounted_on)
cmd_list = ['sudo', 'umount', '-lf', mounted_on]
try:
execute_command_line(cmd_list)
except CommandError as ex:
if ex.returncode != 32: # ignore location not mounted error
if ex.returncode == 1 and "not mounted" in str(ex):
return # sometimes we're getting a return code of 1 instead of 32 when running the CI build
raise NfsError(ex)
except Exception as ex:
raise NfsError(ex)
def nfs_mount(mount, mount_on, read_only=True):
'''Performs a mount of a network file system
:param mount: The network file system to mount in the form of host:/dir/path
:type mount: str
:param mount_on: The absolute directory path to mount on (must already exist)
:type mount_on: str
:param read_only: Whether the mount should be read-only
:type read_only: bool
'''
logger.info('Mounting %s on %s', mount, mount_on)
options = 'soft,' + ('ro' if read_only else 'rw') + ',lookupcache=positive'
cmd_list = ['sudo', 'mount', '-o', options, mount, mount_on]
try:
execute_command_line(cmd_list)
except Exception as ex:
raise NfsError(ex)
def nfs_umount(mounted_on):
'''Performs a umount of a network file system
:param mounted_on: The absolute directory path of the mounted file system
:type mounted_on: str
'''
logger.info('Unmounting %s', mounted_on)
cmd_list = ['sudo', 'umount', '-lf', mounted_on]
try:
execute_command_line(cmd_list)
except CommandError as ex:
# Ignore location not mounted error
if ex.returncode == 32 or (ex.returncode == 1 and "not mounted" in str(ex)):
logger.info('%s was not mounted', mounted_on)
return
raise NfsError(ex)
except Exception as ex:
raise NfsError(ex)
def nfs_umount(mounted_on):
'''Performs a umount of a network file system
:param mounted_on: The absolute directory path of the mounted file system
:type mounted_on: str
'''
logger.info('Unmounting %s', mounted_on)
cmd_list = ['sudo', 'umount', '-lf', mounted_on]
try:
execute_command_line(cmd_list)
except CommandError as ex:
# Ignore location not mounted error
if ex.returncode == 32 or (ex.returncode == 1
and "not mounted" in str(ex)):
logger.info('%s was not mounted', mounted_on)
return
raise NfsError(ex)
except Exception as ex:
raise NfsError(ex)
def store_file(self, work_dir, local_path, data_types, workspace,
remote_path):
'''Stores the given local source file in the workspace
:param work_dir: Absolute path to a local work directory available to assist in storing the source file
:type work_dir: str
:param local_path: The absolute local path of the source file to store
:type local_path: str
:param data_types: The data type tags of the source file
:type data_types: list of str
:param workspace: The workspace to use for storing the source file
:type workspace: :class:`storage.models.Workspace`
:param remote_path: The relative path for storing the source file
:type remote_path: str
:returns: The model of the saved source file
:rtype: :class:`source.models.SourceFile`
'''
file_name = os.path.basename(local_path)
upload_dir = os.path.join(work_dir, 'upload')
workspace_work_dir = os.path.join(work_dir, 'work')
if not os.path.exists(upload_dir):
logger.info('Creating %s', upload_dir)
os.mkdir(upload_dir, 0755)
if not os.path.exists(workspace_work_dir):
logger.info('Creating %s', workspace_work_dir)
os.mkdir(workspace_work_dir, 0755)
upload_file_path = os.path.join(upload_dir, file_name)
ScaleFile.objects.setup_upload_dir(upload_dir, workspace_work_dir,
workspace)
try:
# Check for duplicate file, else create new file
# TODO: fix race condition with many files with same name?
try:
src_file = SourceFile.objects.get(file_name=file_name)
# Duplicate files that are deleted should be stored again
if not src_file.is_deleted:
raise DuplicateFile(u'\'%s\' already exists' % file_name)
except SourceFile.DoesNotExist:
src_file = SourceFile() # New file
# Add a stable identifier based on the file name
src_file.update_uuid(file_name)
# Add tags and store the new/updated source file
for tag in data_types:
src_file.add_data_type_tag(tag)
# Link source file into upload directory and upload it
if not os.path.islink(upload_file_path):
logger.info('Creating link %s -> %s', upload_file_path,
local_path)
execute_command_line(
['ln', '-s', local_path, upload_file_path])
return ScaleFile.objects.upload_files(
upload_dir, workspace_work_dir, workspace,
[(src_file, file_name, remote_path)])[0]
finally:
ScaleFile.objects.cleanup_upload_dir(upload_dir,
workspace_work_dir, workspace)
