def get(self, path, dest, transfer_timeout=None):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds)
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger(logging.DEBUG,
'downloading file from {} to {}'.format(path, dest))
dest = os.path.abspath(dest)
if ':' not in dest:
dest = "file://" + dest
try:
status = self.__gfal2_copy(path,
dest,
transfer_timeout=transfer_timeout)
if status:
raise exception.RucioException()
except exception.DestinationNotAccessible as error:
raise exception.DestinationNotAccessible(str(error))
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def get(self, pfn, dest, transfer_timeout=None):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
self.__connection.get(self.pfn2path(pfn), dest)
except IOError as e:
try: # To check if the error happend local or remote
with open(dest, 'wb'):
pass
call(['rm', dest])
except IOError as e:
if e.errno == 2:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
if e.errno == 2:
raise exception.SourceNotFound(e)
else:
raise exception.ServiceUnavailable(e)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param path: path to the current file on the storage
:param new_path: path to the new file on the storage
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
bucket, key = self.get_bucket_key(pfn)
if key is None:
raise exception.SourceNotFound(
'Cannot get the source key from S3')
bucket_name, key_name = self.get_bucket_key_name(new_pfn)
key.copy(bucket_name, key_name)
key.delete()
except exception.SourceNotFound as e:
raise exception.SourceNotFound(e)
except boto.exception.S3ResponseError as e:
if e.status in [404, 403]:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
except Exception as e:
raise exception.ServiceUnavailable(e)
def get(self, path, dest):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
dest = os.path.abspath(dest)
if ':' not in dest:
dest = "file://" + dest
try:
status = self.__gfal2_copy(path, dest)
if status:
raise exception.RucioException()
except exception.DestinationNotAccessible as error:
raise exception.DestinationNotAccessible(str(error))
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def get(self, pfn, dest):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
bucket, key = self.get_bucket_key(pfn)
if key is None:
raise exception.SourceNotFound(
'Cannot get the source key from S3')
key.get_contents_to_filename(dest)
except IOError as e:
if e.errno == 2:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
except exception.SourceNotFound as e:
raise exception.SourceNotFound(e)
except Exception as e:
if os.path.exists(dest):
os.remove(dest)
raise exception.ServiceUnavailable(e)
def put(self, source, target, source_dir=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
full_name = source_dir + '/' + source if source_dir else source
try:
bucket, key = self.get_bucket_key(target, create=True)
if key is None:
raise exception.DestinationNotAccessible(
'Cannot get the destionation key from S3')
key.set_contents_from_filename(full_name)
except exception.SourceNotFound as e:
raise exception.SourceNotFound(e)
except Exception as e:
if 'No such file' in str(e):
raise exception.SourceNotFound(e)
else:
raise exception.ServiceUnavailable(e)
def rename(url, new_url, rse):
"""
Rename object.
:param url: URL string.
:param new_url: URL string.
:param rse: RSE name.
"""
try:
endpoint, bucket_name, key_name = _get_endpoint_bucket_key(url)
bucket = _get_bucket(rse, endpoint, bucket_name)
key = bucket.get_key(key_name)
if key is None:
raise exception.SourceNotFound('Key %s not found on %s' %
(key_name, endpoint))
new_endpoint, new_bucket_name, new_key_name = _get_endpoint_bucket_key(
new_url)
if endpoint != new_endpoint:
raise exception.RucioException(
"New endpont %s is different with old endpoint %s, cannot rename to different OS"
% (new_endpoint, endpoint))
key.copy(new_bucket_name, new_key_name)
key.delete()
except boto.exception.S3ResponseError as e:
if e.status in [404, 403]:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
except exception.RucioException, e:
raise e
def get_metadata(urls, rse):
"""
Pass list of urls and return their metadata.
:param urls: A list of URL string.
:param rse: RSE name.
:returns: Dictonary of metadatas.
"""
result = {}
for url in urls:
try:
endpoint, bucket_name, key_name = _get_endpoint_bucket_key(url)
bucket = _get_bucket(rse, endpoint, bucket_name)
metadata = None
key = bucket.get_key(key_name)
if key is None:
metadata = exception.SourceNotFound('Key %s not found on %s' %
(key_name, endpoint))
else:
metadata = {'filesize': key.size}
result[url] = metadata
except boto.exception.S3ResponseError as e:
if e.status in [404, 403]:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
except exception.RucioException, e:
result[url] = e
except:
def put(self, source, target, source_dir=None, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
target = self.pfn2path(target)
if source_dir:
sf = source_dir + '/' + source
else:
sf = source
try:
dirs = os.path.dirname(target)
if not os.path.exists(dirs):
os.makedirs(dirs)
shutil.copy(sf, target)
except IOError as e:
if e.errno == 2:
raise exception.SourceNotFound(e)
elif not self.exists(self.rse['prefix']):
path = ''
for p in self.rse['prefix'].split('/'):
path += p + '/'
os.mkdir(path)
shutil.copy(sf, self.pfn2path(target))
else:
raise exception.DestinationNotAccessible(e)
def put(self, source, target, source_dir=None, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
if source_dir:
sf = source_dir + '/' + source
else:
sf = source
try:
self.__connection.put(sf, self.pfn2path(target))
except IOError as e:
try:
self.__connection.execute(
'mkdir -p %s' %
'/'.join(self.pfn2path(target).split('/')[0:-1]))
self.__connection.put(sf, self.pfn2path(target))
except Exception as e:
raise exception.DestinationNotAccessible(e)
except OSError as e:
if e.errno == 2:
raise exception.SourceNotFound(e)
else:
raise exception.ServiceUnavailable(e)
def get(self, pfn, dest, transfer_timeout=None):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
tf = None
try:
tf = open(dest, 'wb')
self.__s3.object_get(S3Uri(pfn), tf) # pylint: disable=no-value-for-parameter
tf.close()
except S3Error as e:
tf.close()
call(['rm', dest]) # Must be changed if resume will be supported
if e.status in [404, 403]:
raise exception.SourceNotFound(e)
else:
raise exception.ServiceUnavailable(e)
except IOError as e:
if e.errno == 2:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
def get_signed_urls(urls, rse, operation='read'):
"""
Pass list of urls and return their signed urls.
:param urls: A list of URL string.
:param rse: RSE name.
:returns: Dictionary of Signed URLs.
"""
result = {}
for url in urls:
try:
endpoint, bucket_name, key_name = _get_endpoint_bucket_key(url)
signed_url = None
if operation == 'read':
# signed_url = conn.generate_url(3600, 'GET', bucket_name, key_name, query_auth=True, force_http=False)
bucket = _get_bucket(rse, endpoint, bucket_name)
key = bucket.get_key(key_name)
if key is None:
signed_url = exception.SourceNotFound(
'Key %s not found on %s' % (key_name, endpoint))
else:
try:
signed_url = key.generate_url(3600,
'GET',
query_auth=True,
merge_meta=False,
force_http=False)
except TypeError:
# merge_meta option is not supported
signed_url = key.generate_url(3600,
'GET',
query_auth=True,
force_http=False)
else:
conn = _get_connection(rse, endpoint)
_get_bucket(rse, endpoint, bucket_name, operation='write')
signed_url = conn.generate_url(3600,
'PUT',
bucket_name,
key_name,
query_auth=True,
force_http=False)
result[url] = signed_url
except boto.exception.S3ResponseError as e:
if e.status in [404, 403]:
result[url] = exception.DestinationNotAccessible(e)
else:
result[url] = exception.ServiceUnavailable(e)
except exception.RucioException as e:
result[url] = e
except:
result[url] = exception.RucioException(
"Failed to get signed url for %s, error: %s" %
(url, traceback.format_exc()))
return result
def put(self, source, target, source_dir, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds)
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger(logging.DEBUG,
'uploading file from {} to {}'.format(source, target))
source_url = '%s/%s' % (source_dir, source) if source_dir else source
source_url = os.path.abspath(source_url)
if not os.path.exists(source_url):
raise exception.SourceNotFound()
if ':' not in source_url:
source_url = "file://" + source_url
space_token = None
if self.attributes[
'extended_attributes'] is not None and 'space_token' in list(
self.attributes['extended_attributes'].keys()):
space_token = self.attributes['extended_attributes']['space_token']
try:
status = self.__gfal2_copy(str(source_url),
str(target),
None,
space_token,
transfer_timeout=transfer_timeout)
if status:
raise exception.RucioException()
except exception.DestinationNotAccessible as error:
raise exception.DestinationNotAccessible(str(error))
except exception.SourceNotFound as error:
raise exception.DestinationNotAccessible(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param path: path to the current file on the storage
:param new_path: path to the new file on the storage
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
self.__s3.object_move(S3Uri(pfn), S3Uri(new_pfn))
except S3Error as e:
if e.status in [404, 403]:
if self.exists(pfn):
raise exception.SourceNotFound(e)
else:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
def rename(self, path, new_path):
"""
Allows to rename a file stored inside the connected RSE.
:param path: path to the current file on the storage
:param new_path: path to the new file on the storage
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
status = self.__gfal2_rename(path, new_path)
if status:
raise exception.RucioException()
except exception.DestinationNotAccessible as error:
raise exception.DestinationNotAccessible(str(error))
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def put(self, source, target, source_dir=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
full_name = source_dir + '/' + source if source_dir else source
try:
self.__s3.object_put(full_name, S3Uri(target))
except S3Error as e:
if e.info['Code'] in ['NoSuchBucket', "AccessDenied"]:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
except InvalidFileError as error:
raise exception.SourceNotFound(error)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param path: path to the current file on the storage
:param new_path: path to the new file on the storage
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
path = self.pfn2path(pfn)
new_path = self.pfn2path(new_pfn)
try:
if not os.path.exists(os.path.dirname(new_path)):
os.makedirs(os.path.dirname(new_path))
os.rename(path, new_path)
except IOError as e:
if e.errno == 2:
if self.exists(self.pfn2path(path)):
raise exception.SourceNotFound(e)
else:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param path: path to the current file on the storage
:param new_path: path to the new file on the storage
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
try:
path = self.pfn2path(pfn)
new_path = self.pfn2path(new_pfn)
self.__connection.execute('mkdir -p %s' %
'/'.join(new_path.split('/')[0:-1]))
self.__connection.execute('mv %s %s' % (path, new_path))
except IOError as e:
if e.errno == 2:
if self.exists(path):
raise exception.SourceNotFound(e)
else:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)