def _copy_file(self, src_path, dest_path): """Performs a copy from the src_path to the dest_path :param src_path: The absolute path to the source file :type src_path: str :param dest_path: The absolute path to the destination :type dest_path: str """ if os.path.islink(src_path): real_path = os.path.realpath(src_path) logger.info('%s is a link to %s', src_path, real_path) src_path = real_path logger.info('Copying %s to %s', src_path, dest_path) # Attempt bbcp copy first. If it fails, we'll fallback to cp try: # TODO: detect bbcp location instead of assuming /usr/local and don't even try to execute if it isn't # installed # TODO: configuration options for the bbcp copy options such as window size srv_src_path, srv_dest_path = self._get_mount_info(src_path, dest_path) cmd_list = ['/usr/local/bin/bbcp', '-s', '8', '-w', '64M', '-E', 'md5', '-o', '-y', 'd', apply(os.path.join, srv_src_path) if srv_src_path[0] is not None else srv_src_path[1], apply(os.path.join, srv_dest_path) if srv_dest_path[0] is not None else srv_dest_path[1]] execute_command_line(cmd_list) return except OSError as e: # errno 2 is No such file or directory..bbcp not installed. We'll be quiet about it but fallback if e.errno != 2: logger.exception("NFS Broker bbcp copy_file") # Ignore the error and attempt a regular cp except: logger.exception("NFS Broker bbcp copy_file") # Ignore the error and attempt a regular cp logger.info('Fall back to cp for %s', src_path) shutil.copy(src_path, dest_path)
def download_files(self, volume_path, file_downloads): """See :meth:`storage.brokers.broker.Broker.download_files`""" with S3Client(self._credentials, self._region_name) as client: for file_download in file_downloads: # If file supports partial mount and volume is configured attempt sym-link if file_download.partial and self._volume: logger.debug('Partial S3 file accessed by mounted bucket.') path_to_download = os.path.join( volume_path, file_download.file.file_path) logger.info('Checking path %s', path_to_download) if not os.path.exists(path_to_download): raise MissingFile(file_download.file.file_name) # Create symlink to the file in the host mount logger.info('Creating link %s -> %s', file_download.local_path, path_to_download) execute_command_line([ 'ln', '-s', path_to_download, file_download.local_path ]) # Fall-back to default S3 file download else: try: s3_object = client.get_object( self._bucket_name, file_download.file.file_path) except FileDoesNotExist: raise MissingFile(file_download.file.file_name) self._download_file(s3_object, file_download.file, file_download.local_path)
def _copy_file(self, src_path, dest_path): '''Performs a copy from the src_path to the dest_path :param src_path: The absolute path to the source file :type src_path: str :param dest_path: The absolute path to the destination :type dest_path: str ''' logger.info('Copying %s to %s', src_path, dest_path) # attempt bbcp copy first. If it fails, we'll fallback to cp try: # TODO: detect bbcp location instead of assuming /usr/local and don't even try to execute if it isn't installed # TODO: configuration options for the bbcp copy options such as window size srv_src_path, srv_dest_path = self._get_mount_info(src_path, dest_path) cmd_list = ['/usr/local/bin/bbcp', '-s', '8', '-w', '64M', '-E', 'md5', '-o', '-y', 'd', apply(os.path.join, srv_src_path) if srv_src_path[0] is not None else srv_src_path[1], apply(os.path.join, srv_dest_path) if srv_dest_path[0] is not None else srv_dest_path[1]] execute_command_line(cmd_list) return except: logger.exception("NFS Broker bbcp copy_file") # ignore the error and attempt a regular cp logger.info('Fall back to cp for %s', src_path) shutil.copy(src_path, dest_path)
def _copy_file(self, src_path, dest_path): """Performs a copy from the src_path to the dest_path :param src_path: The absolute path to the source file :type src_path: str :param dest_path: The absolute path to the destination :type dest_path: str """ if os.path.islink(src_path): real_path = os.path.realpath(src_path) logger.info('%s is a link to %s', src_path, real_path) src_path = real_path logger.info('Copying %s to %s', src_path, dest_path) # attempt bbcp copy first. If it fails, we'll fallback to cp try: # TODO: detect bbcp location instead of assuming /usr/local and don't even try to execute if it isn't installed # TODO: configuration options for the bbcp copy options such as window size srv_src_path, srv_dest_path = self._get_mount_info( src_path, dest_path) cmd_list = [ '/usr/local/bin/bbcp', '-s', '8', '-w', '64M', '-E', 'md5', '-o', '-y', 'd', apply(os.path.join, srv_src_path) if srv_src_path[0] is not None else srv_src_path[1], apply(os.path.join, srv_dest_path) if srv_dest_path[0] is not None else srv_dest_path[1] ] execute_command_line(cmd_list) return except OSError, e: if e.errno != 2: # errno 2 is No such file or directory..bbcp not installed. We'll be quiet about it but fallback logger.exception("NFS Broker bbcp copy_file" ) # ignore the error and attempt a regular cp
def download_files(self, volume_path, file_downloads): """See :meth:`storage.brokers.broker.Broker.download_files` """ for file_download in file_downloads: path_to_download = os.path.join(volume_path, file_download.file.file_path) # Create symlink to the file in the host mount logger.info('Creating link %s -> %s', file_download.local_path, path_to_download) execute_command_line(['ln', '-s', path_to_download, file_download.local_path])
def store_file(self, work_dir, local_path, data_types, workspace, remote_path): '''Stores the given local source file in the workspace :param work_dir: Absolute path to a local work directory available to assist in storing the source file :type work_dir: str :param local_path: The absolute local path of the source file to store :type local_path: str :param data_types: The data type tags of the source file :type data_types: list of str :param workspace: The workspace to use for storing the source file :type workspace: :class:`storage.models.Workspace` :param remote_path: The relative path for storing the source file :type remote_path: str :returns: The model of the saved source file :rtype: :class:`source.models.SourceFile` ''' file_name = os.path.basename(local_path) upload_dir = os.path.join(work_dir, 'upload') workspace_work_dir = os.path.join(work_dir, 'work') if not os.path.exists(upload_dir): logger.info('Creating %s', upload_dir) os.mkdir(upload_dir, 0755) if not os.path.exists(workspace_work_dir): logger.info('Creating %s', workspace_work_dir) os.mkdir(workspace_work_dir, 0755) upload_file_path = os.path.join(upload_dir, file_name) ScaleFile.objects.setup_upload_dir(upload_dir, workspace_work_dir, workspace) try: # Check for duplicate file, else create new file # TODO: fix race condition with many files with same name? try: src_file = SourceFile.objects.get(file_name=file_name) # Duplicate files that are deleted should be stored again if not src_file.is_deleted: raise DuplicateFile(u'\'%s\' already exists' % file_name) except SourceFile.DoesNotExist: src_file = SourceFile() # New file # Add a stable identifier based on the file name src_file.update_uuid(file_name) # Add tags and store the new/updated source file for tag in data_types: src_file.add_data_type_tag(tag) # Link source file into upload directory and upload it if not os.path.islink(upload_file_path): logger.info('Creating link %s -> %s', upload_file_path, local_path) execute_command_line(['ln', '-s', local_path, upload_file_path]) return ScaleFile.objects.upload_files(upload_dir, workspace_work_dir, workspace, [(src_file, file_name, remote_path)])[0] finally: ScaleFile.objects.cleanup_upload_dir(upload_dir, workspace_work_dir, workspace)
def download_files(self, volume_path, file_downloads): """See :meth:`storage.brokers.broker.Broker.download_files` """ for file_download in file_downloads: path_to_download = os.path.join(volume_path, file_download.file.file_path) # Create symlink to the file in the host mount logger.info('Creating link %s -> %s', file_download.local_path, path_to_download) execute_command_line( ['ln', '-s', path_to_download, file_download.local_path])
def download_files(self, download_dir, work_dir, files_to_download): '''See :meth:`storage.brokers.broker.Broker.download_files` ''' for file_to_download in files_to_download: workspace_path = file_to_download[0] dest_path = file_to_download[1] full_workspace_path = os.path.join(work_dir, workspace_path) full_dest_path = os.path.join(download_dir, dest_path) full_dest_dir = os.path.dirname(full_dest_path) if not os.path.exists(full_dest_dir): logger.info('Creating %s', full_dest_dir) os.makedirs(full_dest_dir, mode=0755) execute_command_line(['ln', '-s', full_workspace_path, full_dest_path])
def download_files(self, download_dir, work_dir, files_to_download): """See :meth:`storage.brokers.broker.Broker.download_files` """ for file_to_download in files_to_download: workspace_path = file_to_download[0] dest_path = file_to_download[1] full_workspace_path = os.path.join(work_dir, workspace_path) full_dest_path = os.path.join(download_dir, dest_path) full_dest_dir = os.path.dirname(full_dest_path) if not os.path.exists(full_dest_dir): logger.info('Creating %s', full_dest_dir) os.makedirs(full_dest_dir, mode=0755) execute_command_line( ['ln', '-s', full_workspace_path, full_dest_path])
def download_files(self, volume_path, file_downloads): """See :meth:`storage.brokers.broker.Broker.download_files`""" with S3Client(self._credentials, self._region_name) as client: for file_download in file_downloads: # If file supports partial mount and volume is configured attempt sym-link if file_download.partial and self._volume: logger.debug('Partial S3 file accessed by mounted bucket.') path_to_download = os.path.join(volume_path, file_download.file.file_path) # Create symlink to the file in the host mount logger.info('Creating link %s -> %s', file_download.local_path, path_to_download) execute_command_line(['ln', '-s', path_to_download, file_download.local_path]) # Fall-back to default S3 file download else: s3_object = client.get_object(self._bucket_name, file_download.file.file_path) self._download_file(s3_object, file_download.file, file_download.local_path)
def nfs_mount(mount, mount_on, read_only=True): '''Performs a mount of a network file system :param mount: The network file system to mount in the form of host:/dir/path :type mount: str :param mount_on: The absolute directory path to mount on (must already exist) :type mount_on: str :param read_only: Whether the mount should be read-only :type read_only: bool ''' logger.info('Mounting %s on %s', mount, mount_on) options = 'soft,' + ('ro' if read_only else 'rw') + ',lookupcache=positive' cmd_list = ['sudo', 'mount', '-o', options, mount, mount_on] try: execute_command_line(cmd_list) except Exception as ex: raise NfsError(ex)
def nfs_umount(mounted_on): '''Performs a umount of a network file system :param mounted_on: The absolute directory path of the mounted file system :type mounted_on: str ''' logger.info('Unmounting %s', mounted_on) cmd_list = ['sudo', 'umount', '-lf', mounted_on] try: execute_command_line(cmd_list) except CommandError as ex: if ex.returncode != 32: # ignore location not mounted error if ex.returncode == 1 and "not mounted" in str(ex): return # sometimes we're getting a return code of 1 instead of 32 when running the CI build raise NfsError(ex) except Exception as ex: raise NfsError(ex)
def nfs_umount(mounted_on): '''Performs a umount of a network file system :param mounted_on: The absolute directory path of the mounted file system :type mounted_on: str ''' logger.info('Unmounting %s', mounted_on) cmd_list = ['sudo', 'umount', '-lf', mounted_on] try: execute_command_line(cmd_list) except CommandError as ex: # Ignore location not mounted error if ex.returncode == 32 or (ex.returncode == 1 and "not mounted" in str(ex)): logger.info('%s was not mounted', mounted_on) return raise NfsError(ex) except Exception as ex: raise NfsError(ex)
def store_file(self, work_dir, local_path, data_types, workspace, remote_path): '''Stores the given local source file in the workspace :param work_dir: Absolute path to a local work directory available to assist in storing the source file :type work_dir: str :param local_path: The absolute local path of the source file to store :type local_path: str :param data_types: The data type tags of the source file :type data_types: list of str :param workspace: The workspace to use for storing the source file :type workspace: :class:`storage.models.Workspace` :param remote_path: The relative path for storing the source file :type remote_path: str :returns: The model of the saved source file :rtype: :class:`source.models.SourceFile` ''' file_name = os.path.basename(local_path) upload_dir = os.path.join(work_dir, 'upload') workspace_work_dir = os.path.join(work_dir, 'work') if not os.path.exists(upload_dir): logger.info('Creating %s', upload_dir) os.mkdir(upload_dir, 0755) if not os.path.exists(workspace_work_dir): logger.info('Creating %s', workspace_work_dir) os.mkdir(workspace_work_dir, 0755) upload_file_path = os.path.join(upload_dir, file_name) ScaleFile.objects.setup_upload_dir(upload_dir, workspace_work_dir, workspace) try: # Check for duplicate file, else create new file # TODO: fix race condition with many files with same name? try: src_file = SourceFile.objects.get(file_name=file_name) # Duplicate files that are deleted should be stored again if not src_file.is_deleted: raise DuplicateFile(u'\'%s\' already exists' % file_name) except SourceFile.DoesNotExist: src_file = SourceFile() # New file # Add a stable identifier based on the file name src_file.update_uuid(file_name) # Add tags and store the new/updated source file for tag in data_types: src_file.add_data_type_tag(tag) # Link source file into upload directory and upload it if not os.path.islink(upload_file_path): logger.info('Creating link %s -> %s', upload_file_path, local_path) execute_command_line( ['ln', '-s', local_path, upload_file_path]) return ScaleFile.objects.upload_files( upload_dir, workspace_work_dir, workspace, [(src_file, file_name, remote_path)])[0] finally: ScaleFile.objects.cleanup_upload_dir(upload_dir, workspace_work_dir, workspace)