def put(self, source, target, source_dir=None): """ Allows to store files inside the referred RSE. :param source: path to the source file on the client file system :param target: path to the destination file on the storage :param source_dir: Path where the to be transferred files are stored in the local file system :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ full_name = source_dir + '/' + source if source_dir else source try: bucket, key = self.get_bucket_key(target, create=True) if key is None: raise exception.DestinationNotAccessible( 'Cannot get the destionation key from S3') key.set_contents_from_filename(full_name) except exception.SourceNotFound as e: raise exception.SourceNotFound(e) except Exception as e: if 'No such file' in str(e): raise exception.SourceNotFound(e) else: raise exception.ServiceUnavailable(e)
def delete(self, path): """ Deletes a file from the connected RSE. :param path: path to the to be deleted file :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ pfns = [path] if isinstance(path, string_types) else path try: pfn_chunks = [pfns[i:i + 20] for i in range(0, len(pfns), 20)] for pfn_chunk in pfn_chunks: cmd = 'lcg-del $LCGVO -v -b -l --srm-timeout 600 -D srmv2' for pfn in pfn_chunk: cmd += ' ' + pfn status, out, err = execute(cmd) if status: if self.__parse_srm_error__("SRM_INVALID_PATH", out, err): raise exception.SourceNotFound(err) raise exception.RucioException(err) except exception.SourceNotFound as error: raise exception.SourceNotFound(str(error)) except Exception as error: raise exception.ServiceUnavailable(error)
def rename(self, pfn, new_pfn): """ Allows to rename a file stored inside the connected RSE. :param path: path to the current file on the storage :param new_path: path to the new file on the storage :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ try: bucket, key = self.get_bucket_key(pfn) if key is None: raise exception.SourceNotFound( 'Cannot get the source key from S3') bucket_name, key_name = self.get_bucket_key_name(new_pfn) key.copy(bucket_name, key_name) key.delete() except exception.SourceNotFound as e: raise exception.SourceNotFound(e) except boto.exception.S3ResponseError as e: if e.status in [404, 403]: raise exception.DestinationNotAccessible(e) else: raise exception.ServiceUnavailable(e) except Exception as e: raise exception.ServiceUnavailable(e)
def get(self, path, dest, transfer_timeout=None): """ Provides access to files stored inside connected the RSE. :param path: Physical file name of requested file :param dest: Name and path of the files when stored at the client :param transfer_timeout: Transfer timeout (in seconds) :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ timeout_option = '' if transfer_timeout: timeout_option = '--sendreceive-timeout %s' % transfer_timeout try: cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 %s -D srmv2 %s file:%s' % (timeout_option, path, dest) status, out, err = execute(cmd) if status: if self.__parse_srm_error__("SRM_INVALID_PATH", out, err): raise exception.SourceNotFound(err) raise exception.RucioException(err) except exception.SourceNotFound as error: raise exception.SourceNotFound(str(error)) except Exception as error: raise exception.ServiceUnavailable(error)
def get(self, pfn, dest): """ Provides access to files stored inside connected the RSE. :param path: Physical file name of requested file :param dest: Name and path of the files when stored at the client :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ try: bucket, key = self.get_bucket_key(pfn) if key is None: raise exception.SourceNotFound( 'Cannot get the source key from S3') key.get_contents_to_filename(dest) except IOError as e: if e.errno == 2: raise exception.DestinationNotAccessible(e) else: raise exception.ServiceUnavailable(e) except exception.SourceNotFound as e: raise exception.SourceNotFound(e) except Exception as e: if os.path.exists(dest): os.remove(dest) raise exception.ServiceUnavailable(e)
def rename(self, pfn, new_pfn): """ Allows to rename a file stored inside the connected RSE. :param pfn Current physical file name :param new_pfn New physical file name :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied """ path = self.path2pfn(pfn) new_path = self.path2pfn(new_pfn) directories = new_path.split('/') headers = {'Destination': new_path} # Try the rename without testing the existence of the destination directory try: result = self.session.request('MOVE', path, verify=False, headers=headers, timeout=self.timeout, cert=self.cert) if result.status_code == 201: return elif result.status_code in [ 404, ]: raise exception.SourceNotFound() else: # Create the directories before issuing the MOVE for directory_level in reversed(list(range(1, 4))): upper_directory = "/".join(directories[:-directory_level]) self.mkdir(upper_directory) try: result = self.session.request('MOVE', path, verify=False, headers=headers, timeout=self.timeout, cert=self.cert) if result.status_code == 201: return elif result.status_code in [ 404, ]: raise exception.SourceNotFound() elif result.status_code in [ 401, ]: raise exception.RSEAccessDenied() else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error)
def put(self, source, target, source_dir=None, transfer_timeout=None, progressbar=False): """ Allows to store files inside the referred RSE. :param source Physical file name :param target Name of the file on the storage system e.g. with prefixed scope :param source_dir Path where the to be transferred files are stored in the local file system :param transfer_timeout Transfer timeout (in seconds) - dummy :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied """ path = self.path2pfn(target) full_name = source_dir + '/' + source if source_dir else source directories = path.split('/') # Try the upload without testing the existence of the destination directory try: if not os.path.exists(full_name): raise exception.SourceNotFound() it = UploadInChunks(full_name, 10000000, progressbar) result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert) if result.status_code in [200, 201]: return if result.status_code in [409, ]: raise exception.FileReplicaAlreadyExists() else: # Create the directories before issuing the PUT for directory_level in reversed(list(range(1, 4))): upper_directory = "/".join(directories[:-directory_level]) self.mkdir(upper_directory) try: if not os.path.exists(full_name): raise exception.SourceNotFound() it = UploadInChunks(full_name, 10000000, progressbar) result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert) if result.status_code in [200, 201]: return if result.status_code in [409, ]: raise exception.FileReplicaAlreadyExists() elif result.status_code in [401, ]: raise exception.RSEAccessDenied() else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except IOError as error: raise exception.SourceNotFound(error) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error) except IOError as error: raise exception.SourceNotFound(error)
def put(self, source, target, source_dir=None, transfer_timeout=None): """ Allows to store files inside the referred RSE. :param source: path to the source file on the client file system :param target: path to the destination file on the storage :param source_dir: Path where the to be transferred files are stored in the local file system :param transfer_timeout: Transfer timeout (in seconds) - dummy :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ full_name = source_dir + '/' + source if source_dir else source path = self._get_signed_url(target, operation='write') full_name = source_dir + '/' + source if source_dir else source try: if not os.path.exists(full_name): raise exception.SourceNotFound() it = UploadInChunks(full_name, 10000000, progressbar=False) result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert) if result.status_code in [200, 201]: return if result.status_code in [409, ]: raise exception.FileReplicaAlreadyExists() else: try: if not os.path.exists(full_name): raise exception.SourceNotFound() it = UploadInChunks(full_name, 10000000, progressbar=False) result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert) if result.status_code in [200, 201]: return if result.status_code in [409, ]: raise exception.FileReplicaAlreadyExists() elif result.status_code in [401, ]: raise exception.RSEAccessDenied() else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except IOError as error: raise exception.SourceNotFound(error) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except IOError as error: raise exception.SourceNotFound(error)
def mkdir(self, directory): """ Internal method to create directories :param directory Name of the directory that needs to be created :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied """ path = self.path2pfn(directory) try: result = self.session.request('MKCOL', path, verify=False, timeout=self.timeout, cert=self.cert) if result.status_code in [201, 405 ]: # Success or directory already exists return elif result.status_code in [ 404, ]: raise exception.SourceNotFound() elif result.status_code in [ 401, ]: raise exception.RSEAccessDenied() else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error)
def ls(self, filename): """ Internal method to list files/directories :param filename Name of the directory that needs to be created :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied """ path = self.path2pfn(filename) headers = {'Depth': '1'} self.exists(filename) try: result = self.session.request('PROPFIND', path, verify=False, headers=headers, timeout=self.timeout, cert=self.cert) if result.status_code in [404, ]: raise exception.SourceNotFound() elif result.status_code in [401, ]: raise exception.RSEAccessDenied() parser = Parser() parser.feed(result.text) list_files = [self.server + p_file for p_file in parser.list] try: list_files.remove(filename + '/') except ValueError: pass try: list_files.remove(filename) except ValueError: pass parser.close() return list_files except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error)
def delete(self, pfn): """ Deletes a file from the connected RSE. :param pfn Physical file name :raises ServiceUnavailable, SourceNotFound, RSEAccessDenied, ResourceTemporaryUnavailable """ path = self.path2pfn(pfn) try: result = self.session.delete(path, verify=False, timeout=self.timeout, cert=self.cert) if result.status_code in [ 204, ]: return elif result.status_code in [ 404, ]: raise exception.SourceNotFound() elif result.status_code in [401, 403]: raise exception.RSEAccessDenied() elif result.status_code in [500, 503]: raise exception.ResourceTemporaryUnavailable() else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error)
def stat(self, path): """ Returns the stats of a file. :param path: path to file :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. :raises RSEAccessDenied: in case of permission issue. :returns: a dict with two keys, filesize and adler32 of the file provided in path. """ raise NotImplementedError headers = {'Depth': '1'} dict = {} try: result = self.session.request('PROPFIND', path, verify=False, headers=headers, timeout=self.timeout, cert=self.cert) if result.status_code in [404, ]: raise exception.SourceNotFound() elif result.status_code in [401, ]: raise exception.RSEAccessDenied() if result.status_code in [400, ]: raise NotImplementedError parser = Parser() parser.feed(result.text) for file_name in parser.sizes: if '%s%s' % (self.server, file_name) == path: dict['size'] = parser.sizes[file_name] parser.close() return dict except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error) except requests.exceptions.ReadTimeout as error: raise exception.ServiceUnavailable(error)
def stat(self, pfn): """ Determines the file size in bytes of the provided file. :param pfn: The PFN the file. :returns: a dict containing the key filesize. """ try: bucket, key = self.get_bucket_key(pfn) if key is None: raise exception.SourceNotFound('Cannot get the key from S3') return {'filesize': int(key.size)} except exception.SourceNotFound as e: raise exception.SourceNotFound(e) except Exception as e: raise exception.ServiceUnavailable(e)
def put(self, filename, target, source_dir, transfer_timeout=None): """ Allows to store files inside the referred RSE. :param source: path to the source file on the client file system :param target: path to the destination file on the storage :param source_dir: Path where the to be transferred files are stored in the local file system :param transfer_timeout: Transfer timeout (in seconds) - dummy :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ self.logger(logging.DEBUG, 'rsync.put: filename: {} target: {}'.format(filename, target)) source_dir = source_dir or '.' source_url = '%s/%s' % (source_dir, filename) self.logger(logging.DEBUG, 'rsync.put: source url: {}'.format(source_url)) path = self.pfn2path(target) pathdir = os.path.dirname(path) if not os.path.exists(source_url): raise exception.SourceNotFound() try: cmd = 'ssh -p %s %s%s "mkdir -p %s" && rsync -az -e "ssh -p %s" --append-verify %s %s%s:%s' % (self.port, self.sshuser, self.hostname, pathdir, self.port, source_url, self.sshuser, self.hostname, path) self.logger(logging.DEBUG, 'rsync.put: cmd: {}'.format(cmd)) status, out, err = execute(cmd) if status: raise exception.RucioException(err) except Exception as e: raise exception.ServiceUnavailable(e)
def __gfal2_rm(self, paths): """ Uses gfal2 to remove the file. :param path: Physical file name :returns: 0 if removed successfully, other than 0 if failed :raises SourceNotFound: if the source file was not found. :raises RucioException: if it failed to remove the file. """ self.logger.debug('count: {}'.format(len(list(paths)))) ctx = self.__ctx try: for path in paths: ret = ctx.unlink(str(path)) if ret: return ret return ret except gfal2.GError as error: # pylint: disable=no-member if error.code == errno.ENOENT or 'No such file' in str(error): raise exception.SourceNotFound(error) raise exception.RucioException(error)
def get(self, pfn, dest, transfer_timeout=None): """ Provides access to files stored inside connected the RSE. :param path: Physical file name of requested file :param dest: Name and path of the files when stored at the client :param transfer_timeout: Transfer timeout (in seconds) - dummy :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ try: self.__connection.get(self.pfn2path(pfn), dest) except IOError as e: try: # To check if the error happend local or remote with open(dest, 'wb'): pass call(['rm', dest]) except IOError as e: if e.errno == 2: raise exception.DestinationNotAccessible(e) else: raise exception.ServiceUnavailable(e) if e.errno == 2: raise exception.SourceNotFound(e) else: raise exception.ServiceUnavailable(e)
def get(self, path, dest): """ Provides access to files stored inside connected the RSE. :param path: Physical file name of requested file :param dest: Name and path of the files when stored at the client :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ dest = os.path.abspath(dest) if ':' not in dest: dest = "file://" + dest try: status = self.__gfal2_copy(path, dest) if status: raise exception.RucioException() except exception.DestinationNotAccessible as error: raise exception.DestinationNotAccessible(str(error)) except exception.SourceNotFound as error: raise exception.SourceNotFound(str(error)) except Exception as error: raise exception.ServiceUnavailable(error)
def get(self, pfn, dest, transfer_timeout=None): """ Provides access to files stored inside connected the RSE. :param path: Physical file name of requested file :param dest: Name and path of the files when stored at the client :param transfer_timeout: Transfer timeout (in seconds) - dummy :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ tf = None try: tf = open(dest, 'wb') self.__s3.object_get(S3Uri(pfn), tf) # pylint: disable=no-value-for-parameter tf.close() except S3Error as e: tf.close() call(['rm', dest]) # Must be changed if resume will be supported if e.status in [404, 403]: raise exception.SourceNotFound(e) else: raise exception.ServiceUnavailable(e) except IOError as e: if e.errno == 2: raise exception.DestinationNotAccessible(e) else: raise exception.ServiceUnavailable(e)
def put(self, filename, target, source_dir, transfer_timeout=None): """ Allows to store files inside the referred RSE. :param source: path to the source file on the client file system :param target: path to the destination file on the storage :param source_dir: Path where the to be transferred files are stored in the local file system :param transfer_timeout: Transfer timeout (in seconds) - dummy :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ self.logger.debug('xrootd.put: filename: {} target: {}'.format( filename, target)) source_dir = source_dir or '.' source_url = '%s/%s' % (source_dir, filename) self.logger.debug('xrootd put: source url: {}'.format(source_url)) path = self.path2pfn(target) if not os.path.exists(source_url): raise exception.SourceNotFound() try: cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (source_url, path) self.logger.info('xrootd.put: cmd: {}'.format(cmd)) status, out, err = execute(cmd) if status != 0: raise exception.RucioException(err) except Exception as e: raise exception.ServiceUnavailable(e)
def put(self, source, target, source_dir=None, transfer_timeout=None): """ Allows to store files inside the referred RSE. :param source: path to the source file on the client file system :param target: path to the destination file on the storage :param source_dir: Path where the to be transferred files are stored in the local file system :param transfer_timeout Transfer timeout (in seconds) - dummy :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ target = self.pfn2path(target) if source_dir: sf = source_dir + '/' + source else: sf = source try: dirs = os.path.dirname(target) if not os.path.exists(dirs): os.makedirs(dirs) shutil.copy(sf, target) except IOError as e: if e.errno == 2: raise exception.SourceNotFound(e) elif not self.exists(self.rse['prefix']): path = '' for p in self.rse['prefix'].split('/'): path += p + '/' os.mkdir(path) shutil.copy(sf, self.pfn2path(target)) else: raise exception.DestinationNotAccessible(e)
def put(self, source, target, source_dir, transfer_timeout=None): """ Allows to store files inside the referred RSE. :param source: path to the source file on the client file system :param target: path to the destination file on the storage :param source_dir: Path where the to be transferred files are stored in the local file system :param transfer_timeout: Transfer timeout (in seconds) :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ source_url = '%s/%s' % (source_dir, source) if source_dir else source if not os.path.exists(source_url): raise exception.SourceNotFound() space_token = '' if self.attributes['extended_attributes'] is not None and 'space_token' in list(self.attributes['extended_attributes'].keys()): space_token = '--dst %s' % self.attributes['extended_attributes']['space_token'] timeout_option = '' if transfer_timeout: timeout_option = '--sendreceive-timeout %s' % transfer_timeout try: cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 %s -D srmv2 %s file:%s %s' % (timeout_option, space_token, source_url, target) status, out, err = execute(cmd) if status: raise exception.RucioException(err) except Exception as error: raise exception.ServiceUnavailable(error)
def get_bucket_key(self, pfn, create=False, validate=True): """ Gets boto key for a pfn :param pfn: Physical file name :param create: True if needs to create the key, False if not :returns: boto bucket and key object """ try: bucket_name, key_name = self.get_bucket_key_name(pfn) if create: try: bucket = self.__conn.get_bucket(bucket_name, validate=True) except boto.exception.S3ResponseError as e: if e.status == 404: # bucket not found bucket = self.__conn.create_bucket(bucket_name) else: raise e key = Key(bucket, key_name) else: bucket = self.__conn.get_bucket(bucket_name, validate=False) key = bucket.get_key(key_name, validate=validate) return bucket, key except boto.exception.S3ResponseError as e: if e.status == 404: raise exception.SourceNotFound(str(e)) else: raise exception.ServiceUnavailable(e)
def rename(self, pfn, new_pfn): """ Allows to rename a file stored inside the connected RSE. :param pfn Current physical file name :param new_pfn New physical file name :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ self.logger.debug('xrootd.rename: pfn: {}'.format(pfn)) if not self.exists(pfn): raise exception.SourceNotFound() try: path = self.pfn2path(pfn) new_path = self.pfn2path(new_pfn) new_dir = new_path[:new_path.rindex('/') + 1] cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mkdir -p %s' % ( self.hostname, self.port, new_dir) self.logger.info('xrootd.stat: mkdir cmd: {}'.format(cmd)) status, out, err = execute(cmd) cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mv %s %s' % ( self.hostname, self.port, path, new_path) self.logger.info('xrootd.stat: rename cmd: {}'.format(cmd)) status, out, err = execute(cmd) if status != 0: raise exception.RucioException(err) except Exception as e: raise exception.ServiceUnavailable(e)
def _get_bucket(rse, endpoint, bucket_name, operation='read'): """ Pass an endpoint and return a connection to object store. :param rse: RSE name. :param endpoint: URL endpoint string. :returns: Connection object. """ key = "%s:%s:%s" % (rse, endpoint, bucket_name) result = REGION.get(key) if type(result) is NoValue: try: logging.debug("Creating bucket object") result = None conn = _get_connection(rse, endpoint) bucket = conn.get_bucket(bucket_name) if operation == 'read': if bucket is None: raise exception.SourceNotFound( 'Bucket %s not found on %s' % (bucket_name, rse)) else: result = bucket REGION.set(key, result) else: result = conn.create_bucket(bucket_name) REGION.set(key, result) except exception.RucioException, e: raise e except:
def put(self, source, target, source_dir=None, transfer_timeout=None): """ Allows to store files inside the referred RSE. :param source: path to the source file on the client file system :param target: path to the destination file on the storage :param source_dir: Path where the to be transferred files are stored in the local file system :param transfer_timeout: Transfer timeout (in seconds) - dummy :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ if source_dir: sf = source_dir + '/' + source else: sf = source try: self.__connection.put(sf, self.pfn2path(target)) except IOError as e: try: self.__connection.execute( 'mkdir -p %s' % '/'.join(self.pfn2path(target).split('/')[0:-1])) self.__connection.put(sf, self.pfn2path(target)) except Exception as e: raise exception.DestinationNotAccessible(e) except OSError as e: if e.errno == 2: raise exception.SourceNotFound(e) else: raise exception.ServiceUnavailable(e)
def get_metadata(urls, rse): """ Pass list of urls and return their metadata. :param urls: A list of URL string. :param rse: RSE name. :returns: Dictonary of metadatas. """ result = {} for url in urls: try: endpoint, bucket_name, key_name = _get_endpoint_bucket_key(url) bucket = _get_bucket(rse, endpoint, bucket_name) metadata = None key = bucket.get_key(key_name) if key is None: metadata = exception.SourceNotFound('Key %s not found on %s' % (key_name, endpoint)) else: metadata = {'filesize': key.size} result[url] = metadata except boto.exception.S3ResponseError as e: if e.status in [404, 403]: raise exception.DestinationNotAccessible(e) else: raise exception.ServiceUnavailable(e) except exception.RucioException, e: result[url] = e except:
def get(self, path, dest, transfer_timeout=None): """ Provides access to files stored inside connected the RSE. :param path: Physical file name of requested file :param dest: Name and path of the files when stored at the client :param transfer_timeout: Transfer timeout (in seconds) :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ self.logger(logging.DEBUG, 'downloading file from {} to {}'.format(path, dest)) dest = os.path.abspath(dest) if ':' not in dest: dest = "file://" + dest try: status = self.__gfal2_copy(path, dest, transfer_timeout=transfer_timeout) if status: raise exception.RucioException() except exception.DestinationNotAccessible as error: raise exception.DestinationNotAccessible(str(error)) except exception.SourceNotFound as error: raise exception.SourceNotFound(str(error)) except Exception as error: raise exception.ServiceUnavailable(error)
def rename(url, new_url, rse): """ Rename object. :param url: URL string. :param new_url: URL string. :param rse: RSE name. """ try: endpoint, bucket_name, key_name = _get_endpoint_bucket_key(url) bucket = _get_bucket(rse, endpoint, bucket_name) key = bucket.get_key(key_name) if key is None: raise exception.SourceNotFound('Key %s not found on %s' % (key_name, endpoint)) new_endpoint, new_bucket_name, new_key_name = _get_endpoint_bucket_key( new_url) if endpoint != new_endpoint: raise exception.RucioException( "New endpont %s is different with old endpoint %s, cannot rename to different OS" % (new_endpoint, endpoint)) key.copy(new_bucket_name, new_key_name) key.delete() except boto.exception.S3ResponseError as e: if e.status in [404, 403]: raise exception.DestinationNotAccessible(e) else: raise exception.ServiceUnavailable(e) except exception.RucioException, e: raise e
def __gfal2_rename(self, path, new_path): """ Uses gfal2 to rename a file. :param path: path to the current file on the storage :param new_path: path to the new file on the storage :returns: 0 if it exists, -1 if it doesn't :raises RucioException: if failed. """ ctx = self.__ctx try: dir_name = os.path.dirname(new_path) # This function will be removed soon. gfal2 will create parent dir automatically. try: ctx.mkdir_rec(str(dir_name), 0775) except Exception: pass ret = ctx.rename(str(path), str(new_path)) return ret except gfal2.GError as error: if error.code == errno.ENOENT or 'No such file' in error.message: raise exception.SourceNotFound(error) raise exception.RucioException(error)
def get(self, path, dest, transfer_timeout=None): """ Provides access to files stored inside connected the RSE. :param path: Physical file name of requested file :param dest: Name and path of the files when stored at the client :param transfer_timeout: Transfer timeout (in seconds) - dummy :raises DestinationNotAccessible: if the destination storage was not accessible. :raises ServiceUnavailable: if some generic error occured in the library. :raises SourceNotFound: if the source file was not found on the referred storage. """ path = self._get_signed_url(path, 'read') if isinstance(path, Exception): raise path chunksize = 1024 try: result = self.session.get(path, verify=False, stream=True, timeout=self.timeout) if result and result.status_code in [ 200, ]: length = None if 'content-length' in result.headers: length = int(result.headers['content-length']) totnchunk = int(length / chunksize) + 1 with open(dest, 'wb') as f: nchunk = 0 try: if length: pbar = ProgressBar(maxval=totnchunk).start() else: print( 'Malformed HTTP response (missing content-length header). Cannot show progress bar.' ) for chunk in result.iter_content(chunksize): f.write(chunk) if length: nchunk += 1 pbar.update(nchunk) finally: if length: pbar.finish() elif result.status_code in [ 404, ]: raise exception.SourceNotFound() elif result.status_code in [401, 403]: raise exception.RSEAccessDenied() else: # catchall exception raise exception.RucioException(result.status_code, result.text) except requests.exceptions.ConnectionError as error: raise exception.ServiceUnavailable(error)