def put(self, source, target, source_dir=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
full_name = source_dir + '/' + source if source_dir else source
try:
bucket, key = self.get_bucket_key(target, create=True)
if key is None:
raise exception.DestinationNotAccessible(
'Cannot get the destionation key from S3')
key.set_contents_from_filename(full_name)
except exception.SourceNotFound as e:
raise exception.SourceNotFound(e)
except Exception as e:
if 'No such file' in str(e):
raise exception.SourceNotFound(e)
else:
raise exception.ServiceUnavailable(e)
def delete(self, path):
"""
Deletes a file from the connected RSE.
:param path: path to the to be deleted file
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
pfns = [path] if isinstance(path, string_types) else path
try:
pfn_chunks = [pfns[i:i + 20] for i in range(0, len(pfns), 20)]
for pfn_chunk in pfn_chunks:
cmd = 'lcg-del $LCGVO -v -b -l --srm-timeout 600 -D srmv2'
for pfn in pfn_chunk:
cmd += ' ' + pfn
status, out, err = execute(cmd)
if status:
if self.__parse_srm_error__("SRM_INVALID_PATH", out, err):
raise exception.SourceNotFound(err)
raise exception.RucioException(err)
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param path: path to the current file on the storage
:param new_path: path to the new file on the storage
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
bucket, key = self.get_bucket_key(pfn)
if key is None:
raise exception.SourceNotFound(
'Cannot get the source key from S3')
bucket_name, key_name = self.get_bucket_key_name(new_pfn)
key.copy(bucket_name, key_name)
key.delete()
except exception.SourceNotFound as e:
raise exception.SourceNotFound(e)
except boto.exception.S3ResponseError as e:
if e.status in [404, 403]:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
except Exception as e:
raise exception.ServiceUnavailable(e)
def get(self, path, dest, transfer_timeout=None):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds)
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
timeout_option = ''
if transfer_timeout:
timeout_option = '--sendreceive-timeout %s' % transfer_timeout
try:
cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 %s -D srmv2 %s file:%s' % (timeout_option, path, dest)
status, out, err = execute(cmd)
if status:
if self.__parse_srm_error__("SRM_INVALID_PATH", out, err):
raise exception.SourceNotFound(err)
raise exception.RucioException(err)
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def get(self, pfn, dest):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
bucket, key = self.get_bucket_key(pfn)
if key is None:
raise exception.SourceNotFound(
'Cannot get the source key from S3')
key.get_contents_to_filename(dest)
except IOError as e:
if e.errno == 2:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
except exception.SourceNotFound as e:
raise exception.SourceNotFound(e)
except Exception as e:
if os.path.exists(dest):
os.remove(dest)
raise exception.ServiceUnavailable(e)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param pfn Current physical file name
:param new_pfn New physical file name
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied
"""
path = self.path2pfn(pfn)
new_path = self.path2pfn(new_pfn)
directories = new_path.split('/')
headers = {'Destination': new_path}
# Try the rename without testing the existence of the destination directory
try:
result = self.session.request('MOVE',
path,
verify=False,
headers=headers,
timeout=self.timeout,
cert=self.cert)
if result.status_code == 201:
return
elif result.status_code in [
404,
]:
raise exception.SourceNotFound()
else:
# Create the directories before issuing the MOVE
for directory_level in reversed(list(range(1, 4))):
upper_directory = "/".join(directories[:-directory_level])
self.mkdir(upper_directory)
try:
result = self.session.request('MOVE',
path,
verify=False,
headers=headers,
timeout=self.timeout,
cert=self.cert)
if result.status_code == 201:
return
elif result.status_code in [
404,
]:
raise exception.SourceNotFound()
elif result.status_code in [
401,
]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code,
result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def put(self, source, target, source_dir=None, transfer_timeout=None, progressbar=False):
""" Allows to store files inside the referred RSE.
:param source Physical file name
:param target Name of the file on the storage system e.g. with prefixed scope
:param source_dir Path where the to be transferred files are stored in the local file system
:param transfer_timeout Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied
"""
path = self.path2pfn(target)
full_name = source_dir + '/' + source if source_dir else source
directories = path.split('/')
# Try the upload without testing the existence of the destination directory
try:
if not os.path.exists(full_name):
raise exception.SourceNotFound()
it = UploadInChunks(full_name, 10000000, progressbar)
result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert)
if result.status_code in [200, 201]:
return
if result.status_code in [409, ]:
raise exception.FileReplicaAlreadyExists()
else:
# Create the directories before issuing the PUT
for directory_level in reversed(list(range(1, 4))):
upper_directory = "/".join(directories[:-directory_level])
self.mkdir(upper_directory)
try:
if not os.path.exists(full_name):
raise exception.SourceNotFound()
it = UploadInChunks(full_name, 10000000, progressbar)
result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert)
if result.status_code in [200, 201]:
return
if result.status_code in [409, ]:
raise exception.FileReplicaAlreadyExists()
elif result.status_code in [401, ]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except IOError as error:
raise exception.SourceNotFound(error)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
except IOError as error:
raise exception.SourceNotFound(error)
def put(self, source, target, source_dir=None, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
full_name = source_dir + '/' + source if source_dir else source
path = self._get_signed_url(target, operation='write')
full_name = source_dir + '/' + source if source_dir else source
try:
if not os.path.exists(full_name):
raise exception.SourceNotFound()
it = UploadInChunks(full_name, 10000000, progressbar=False)
result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert)
if result.status_code in [200, 201]:
return
if result.status_code in [409, ]:
raise exception.FileReplicaAlreadyExists()
else:
try:
if not os.path.exists(full_name):
raise exception.SourceNotFound()
it = UploadInChunks(full_name, 10000000, progressbar=False)
result = self.session.put(path, data=IterableToFileAdapter(it), verify=False, allow_redirects=True, timeout=self.timeout, cert=self.cert)
if result.status_code in [200, 201]:
return
if result.status_code in [409, ]:
raise exception.FileReplicaAlreadyExists()
elif result.status_code in [401, ]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except IOError as error:
raise exception.SourceNotFound(error)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except IOError as error:
raise exception.SourceNotFound(error)
def mkdir(self, directory):
""" Internal method to create directories
:param directory Name of the directory that needs to be created
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied
"""
path = self.path2pfn(directory)
try:
result = self.session.request('MKCOL',
path,
verify=False,
timeout=self.timeout,
cert=self.cert)
if result.status_code in [201, 405
]: # Success or directory already exists
return
elif result.status_code in [
404,
]:
raise exception.SourceNotFound()
elif result.status_code in [
401,
]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def ls(self, filename):
""" Internal method to list files/directories
:param filename Name of the directory that needs to be created
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied
"""
path = self.path2pfn(filename)
headers = {'Depth': '1'}
self.exists(filename)
try:
result = self.session.request('PROPFIND', path, verify=False, headers=headers, timeout=self.timeout, cert=self.cert)
if result.status_code in [404, ]:
raise exception.SourceNotFound()
elif result.status_code in [401, ]:
raise exception.RSEAccessDenied()
parser = Parser()
parser.feed(result.text)
list_files = [self.server + p_file for p_file in parser.list]
try:
list_files.remove(filename + '/')
except ValueError:
pass
try:
list_files.remove(filename)
except ValueError:
pass
parser.close()
return list_files
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def delete(self, pfn):
""" Deletes a file from the connected RSE.
:param pfn Physical file name
:raises ServiceUnavailable, SourceNotFound, RSEAccessDenied, ResourceTemporaryUnavailable
"""
path = self.path2pfn(pfn)
try:
result = self.session.delete(path,
verify=False,
timeout=self.timeout,
cert=self.cert)
if result.status_code in [
204,
]:
return
elif result.status_code in [
404,
]:
raise exception.SourceNotFound()
elif result.status_code in [401, 403]:
raise exception.RSEAccessDenied()
elif result.status_code in [500, 503]:
raise exception.ResourceTemporaryUnavailable()
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def stat(self, path):
"""
Returns the stats of a file.
:param path: path to file
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
:raises RSEAccessDenied: in case of permission issue.
:returns: a dict with two keys, filesize and adler32 of the file provided in path.
"""
raise NotImplementedError
headers = {'Depth': '1'}
dict = {}
try:
result = self.session.request('PROPFIND', path, verify=False, headers=headers, timeout=self.timeout, cert=self.cert)
if result.status_code in [404, ]:
raise exception.SourceNotFound()
elif result.status_code in [401, ]:
raise exception.RSEAccessDenied()
if result.status_code in [400, ]:
raise NotImplementedError
parser = Parser()
parser.feed(result.text)
for file_name in parser.sizes:
if '%s%s' % (self.server, file_name) == path:
dict['size'] = parser.sizes[file_name]
parser.close()
return dict
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def stat(self, pfn):
""" Determines the file size in bytes of the provided file.
:param pfn: The PFN the file.
:returns: a dict containing the key filesize.
"""
try:
bucket, key = self.get_bucket_key(pfn)
if key is None:
raise exception.SourceNotFound('Cannot get the key from S3')
return {'filesize': int(key.size)}
except exception.SourceNotFound as e:
raise exception.SourceNotFound(e)
except Exception as e:
raise exception.ServiceUnavailable(e)
def put(self, filename, target, source_dir, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger(logging.DEBUG, 'rsync.put: filename: {} target: {}'.format(filename, target))
source_dir = source_dir or '.'
source_url = '%s/%s' % (source_dir, filename)
self.logger(logging.DEBUG, 'rsync.put: source url: {}'.format(source_url))
path = self.pfn2path(target)
pathdir = os.path.dirname(path)
if not os.path.exists(source_url):
raise exception.SourceNotFound()
try:
cmd = 'ssh -p %s %s%s "mkdir -p %s" && rsync -az -e "ssh -p %s" --append-verify %s %s%s:%s' % (self.port, self.sshuser, self.hostname, pathdir, self.port, source_url, self.sshuser, self.hostname, path)
self.logger(logging.DEBUG, 'rsync.put: cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def __gfal2_rm(self, paths):
"""
Uses gfal2 to remove the file.
:param path: Physical file name
:returns: 0 if removed successfully, other than 0 if failed
:raises SourceNotFound: if the source file was not found.
:raises RucioException: if it failed to remove the file.
"""
self.logger.debug('count: {}'.format(len(list(paths))))
ctx = self.__ctx
try:
for path in paths:
ret = ctx.unlink(str(path))
if ret:
return ret
return ret
except gfal2.GError as error: # pylint: disable=no-member
if error.code == errno.ENOENT or 'No such file' in str(error):
raise exception.SourceNotFound(error)
raise exception.RucioException(error)
def get(self, pfn, dest, transfer_timeout=None):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
self.__connection.get(self.pfn2path(pfn), dest)
except IOError as e:
try: # To check if the error happend local or remote
with open(dest, 'wb'):
pass
call(['rm', dest])
except IOError as e:
if e.errno == 2:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
if e.errno == 2:
raise exception.SourceNotFound(e)
else:
raise exception.ServiceUnavailable(e)
def get(self, path, dest):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
dest = os.path.abspath(dest)
if ':' not in dest:
dest = "file://" + dest
try:
status = self.__gfal2_copy(path, dest)
if status:
raise exception.RucioException()
except exception.DestinationNotAccessible as error:
raise exception.DestinationNotAccessible(str(error))
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def get(self, pfn, dest, transfer_timeout=None):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
tf = None
try:
tf = open(dest, 'wb')
self.__s3.object_get(S3Uri(pfn), tf) # pylint: disable=no-value-for-parameter
tf.close()
except S3Error as e:
tf.close()
call(['rm', dest]) # Must be changed if resume will be supported
if e.status in [404, 403]:
raise exception.SourceNotFound(e)
else:
raise exception.ServiceUnavailable(e)
except IOError as e:
if e.errno == 2:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
def put(self, filename, target, source_dir, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger.debug('xrootd.put: filename: {} target: {}'.format(
filename, target))
source_dir = source_dir or '.'
source_url = '%s/%s' % (source_dir, filename)
self.logger.debug('xrootd put: source url: {}'.format(source_url))
path = self.path2pfn(target)
if not os.path.exists(source_url):
raise exception.SourceNotFound()
try:
cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (source_url, path)
self.logger.info('xrootd.put: cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status != 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def put(self, source, target, source_dir=None, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
target = self.pfn2path(target)
if source_dir:
sf = source_dir + '/' + source
else:
sf = source
try:
dirs = os.path.dirname(target)
if not os.path.exists(dirs):
os.makedirs(dirs)
shutil.copy(sf, target)
except IOError as e:
if e.errno == 2:
raise exception.SourceNotFound(e)
elif not self.exists(self.rse['prefix']):
path = ''
for p in self.rse['prefix'].split('/'):
path += p + '/'
os.mkdir(path)
shutil.copy(sf, self.pfn2path(target))
else:
raise exception.DestinationNotAccessible(e)
def put(self, source, target, source_dir, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds)
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
source_url = '%s/%s' % (source_dir, source) if source_dir else source
if not os.path.exists(source_url):
raise exception.SourceNotFound()
space_token = ''
if self.attributes['extended_attributes'] is not None and 'space_token' in list(self.attributes['extended_attributes'].keys()):
space_token = '--dst %s' % self.attributes['extended_attributes']['space_token']
timeout_option = ''
if transfer_timeout:
timeout_option = '--sendreceive-timeout %s' % transfer_timeout
try:
cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 %s -D srmv2 %s file:%s %s' % (timeout_option, space_token, source_url, target)
status, out, err = execute(cmd)
if status:
raise exception.RucioException(err)
except Exception as error:
raise exception.ServiceUnavailable(error)
def get_bucket_key(self, pfn, create=False, validate=True):
"""
Gets boto key for a pfn
:param pfn: Physical file name
:param create: True if needs to create the key, False if not
:returns: boto bucket and key object
"""
try:
bucket_name, key_name = self.get_bucket_key_name(pfn)
if create:
try:
bucket = self.__conn.get_bucket(bucket_name, validate=True)
except boto.exception.S3ResponseError as e:
if e.status == 404: # bucket not found
bucket = self.__conn.create_bucket(bucket_name)
else:
raise e
key = Key(bucket, key_name)
else:
bucket = self.__conn.get_bucket(bucket_name, validate=False)
key = bucket.get_key(key_name, validate=validate)
return bucket, key
except boto.exception.S3ResponseError as e:
if e.status == 404:
raise exception.SourceNotFound(str(e))
else:
raise exception.ServiceUnavailable(e)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param pfn Current physical file name
:param new_pfn New physical file name
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger.debug('xrootd.rename: pfn: {}'.format(pfn))
if not self.exists(pfn):
raise exception.SourceNotFound()
try:
path = self.pfn2path(pfn)
new_path = self.pfn2path(new_pfn)
new_dir = new_path[:new_path.rindex('/') + 1]
cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mkdir -p %s' % (
self.hostname, self.port, new_dir)
self.logger.info('xrootd.stat: mkdir cmd: {}'.format(cmd))
status, out, err = execute(cmd)
cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mv %s %s' % (
self.hostname, self.port, path, new_path)
self.logger.info('xrootd.stat: rename cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status != 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def _get_bucket(rse, endpoint, bucket_name, operation='read'):
"""
Pass an endpoint and return a connection to object store.
:param rse: RSE name.
:param endpoint: URL endpoint string.
:returns: Connection object.
"""
key = "%s:%s:%s" % (rse, endpoint, bucket_name)
result = REGION.get(key)
if type(result) is NoValue:
try:
logging.debug("Creating bucket object")
result = None
conn = _get_connection(rse, endpoint)
bucket = conn.get_bucket(bucket_name)
if operation == 'read':
if bucket is None:
raise exception.SourceNotFound(
'Bucket %s not found on %s' % (bucket_name, rse))
else:
result = bucket
REGION.set(key, result)
else:
result = conn.create_bucket(bucket_name)
REGION.set(key, result)
except exception.RucioException, e:
raise e
except:
def put(self, source, target, source_dir=None, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
if source_dir:
sf = source_dir + '/' + source
else:
sf = source
try:
self.__connection.put(sf, self.pfn2path(target))
except IOError as e:
try:
self.__connection.execute(
'mkdir -p %s' %
'/'.join(self.pfn2path(target).split('/')[0:-1]))
self.__connection.put(sf, self.pfn2path(target))
except Exception as e:
raise exception.DestinationNotAccessible(e)
except OSError as e:
if e.errno == 2:
raise exception.SourceNotFound(e)
else:
raise exception.ServiceUnavailable(e)
def get_metadata(urls, rse):
"""
Pass list of urls and return their metadata.
:param urls: A list of URL string.
:param rse: RSE name.
:returns: Dictonary of metadatas.
"""
result = {}
for url in urls:
try:
endpoint, bucket_name, key_name = _get_endpoint_bucket_key(url)
bucket = _get_bucket(rse, endpoint, bucket_name)
metadata = None
key = bucket.get_key(key_name)
if key is None:
metadata = exception.SourceNotFound('Key %s not found on %s' %
(key_name, endpoint))
else:
metadata = {'filesize': key.size}
result[url] = metadata
except boto.exception.S3ResponseError as e:
if e.status in [404, 403]:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
except exception.RucioException, e:
result[url] = e
except:
def get(self, path, dest, transfer_timeout=None):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds)
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger(logging.DEBUG,
'downloading file from {} to {}'.format(path, dest))
dest = os.path.abspath(dest)
if ':' not in dest:
dest = "file://" + dest
try:
status = self.__gfal2_copy(path,
dest,
transfer_timeout=transfer_timeout)
if status:
raise exception.RucioException()
except exception.DestinationNotAccessible as error:
raise exception.DestinationNotAccessible(str(error))
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def rename(url, new_url, rse):
"""
Rename object.
:param url: URL string.
:param new_url: URL string.
:param rse: RSE name.
"""
try:
endpoint, bucket_name, key_name = _get_endpoint_bucket_key(url)
bucket = _get_bucket(rse, endpoint, bucket_name)
key = bucket.get_key(key_name)
if key is None:
raise exception.SourceNotFound('Key %s not found on %s' %
(key_name, endpoint))
new_endpoint, new_bucket_name, new_key_name = _get_endpoint_bucket_key(
new_url)
if endpoint != new_endpoint:
raise exception.RucioException(
"New endpont %s is different with old endpoint %s, cannot rename to different OS"
% (new_endpoint, endpoint))
key.copy(new_bucket_name, new_key_name)
key.delete()
except boto.exception.S3ResponseError as e:
if e.status in [404, 403]:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
except exception.RucioException, e:
raise e
def __gfal2_rename(self, path, new_path):
"""
Uses gfal2 to rename a file.
:param path: path to the current file on the storage
:param new_path: path to the new file on the storage
:returns: 0 if it exists, -1 if it doesn't
:raises RucioException: if failed.
"""
ctx = self.__ctx
try:
dir_name = os.path.dirname(new_path)
# This function will be removed soon. gfal2 will create parent dir automatically.
try:
ctx.mkdir_rec(str(dir_name), 0775)
except Exception:
pass
ret = ctx.rename(str(path), str(new_path))
return ret
except gfal2.GError as error:
if error.code == errno.ENOENT or 'No such file' in error.message:
raise exception.SourceNotFound(error)
raise exception.RucioException(error)
def get(self, path, dest, transfer_timeout=None):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
path = self._get_signed_url(path, 'read')
if isinstance(path, Exception):
raise path
chunksize = 1024
try:
result = self.session.get(path,
verify=False,
stream=True,
timeout=self.timeout)
if result and result.status_code in [
200,
]:
length = None
if 'content-length' in result.headers:
length = int(result.headers['content-length'])
totnchunk = int(length / chunksize) + 1
with open(dest, 'wb') as f:
nchunk = 0
try:
if length:
pbar = ProgressBar(maxval=totnchunk).start()
else:
print(
'Malformed HTTP response (missing content-length header). Cannot show progress bar.'
)
for chunk in result.iter_content(chunksize):
f.write(chunk)
if length:
nchunk += 1
pbar.update(nchunk)
finally:
if length:
pbar.finish()
elif result.status_code in [
404,
]:
raise exception.SourceNotFound()
elif result.status_code in [401, 403]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
