def set_rse_transfer_limits(rse_id,
activity,
rse_expression=None,
max_transfers=0,
transfers=0,
waitings=0,
volume=0,
session=None):
"""
Set RSE transfer limits.
:param rse_id: The RSE id.
:param activity: The activity.
:param rse_expression: RSE expression string.
:param max_transfers: Maximum transfers.
:param transfers: Current number of tranfers.
:param waitings: Current number of waitings.
:param volume: Maximum transfer volume in bytes.
:param session: The database session in use.
:returns: True if successful, otherwise false.
"""
try:
rse_tr_limit = models.RSETransferLimit(rse_id=rse_id,
activity=activity,
rse_expression=rse_expression,
max_transfers=max_transfers,
transfers=transfers,
waitings=waitings,
volume=volume)
rse_tr_limit = session.merge(rse_tr_limit)
rowcount = rse_tr_limit.save(session=session)
return rowcount
except IntegrityError as error:
raise exception.RucioException(error.args)
def put(self, source, target, source_dir, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds)
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
source_url = '%s/%s' % (source_dir, source) if source_dir else source
if not os.path.exists(source_url):
raise exception.SourceNotFound()
space_token = ''
if self.attributes['extended_attributes'] is not None and 'space_token' in list(self.attributes['extended_attributes'].keys()):
space_token = '--dst %s' % self.attributes['extended_attributes']['space_token']
timeout_option = ''
if transfer_timeout:
timeout_option = '--sendreceive-timeout %s' % transfer_timeout
try:
cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 %s -D srmv2 %s file:%s %s' % (timeout_option, space_token, source_url, target)
status, out, err = execute(cmd)
if status:
raise exception.RucioException(err)
except Exception as error:
raise exception.ServiceUnavailable(error)
def delete(self, path):
"""
Deletes a file from the connected RSE.
:param path: path to the to be deleted file
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
pfns = [path] if isinstance(path, string_types) else path
try:
pfn_chunks = [pfns[i:i + 20] for i in range(0, len(pfns), 20)]
for pfn_chunk in pfn_chunks:
cmd = 'lcg-del $LCGVO -v -b -l --srm-timeout 600 -D srmv2'
for pfn in pfn_chunk:
cmd += ' ' + pfn
status, out, err = execute(cmd)
if status:
if self.__parse_srm_error__("SRM_INVALID_PATH", out, err):
raise exception.SourceNotFound(err)
raise exception.RucioException(err)
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def update_distances(src_rse_id=None,
dest_rse_id=None,
parameters=None,
session=None):
"""
Update distances with the given RSE ids.
:param src_rse_id: The source RSE ID.
:param dest_rse_id: The destination RSE ID.
:param parameters: A dictionnary with property
:param session: The database session to use.
"""
params = {}
for key in parameters:
if key in [
'ranking', 'agis_distance', 'geoip_distance', 'active',
'submitted', 'finished', 'failed', 'transfer_speed',
'packet_loss', 'latency', 'mbps_file', 'mbps_link',
'queued_total', 'done_1h', 'done_6h'
]:
params[key] = parameters[key]
try:
query = session.query(Distance)
if src_rse_id:
query = query.filter(Distance.src_rse_id == src_rse_id)
if dest_rse_id:
query = query.filter(Distance.dest_rse_id == dest_rse_id)
query.update(params)
except IntegrityError as error:
raise exception.RucioException(error.args)
def get(self, path, dest, transfer_timeout=None):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds)
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
timeout_option = ''
if transfer_timeout:
timeout_option = '--sendreceive-timeout %s' % transfer_timeout
try:
cmd = 'lcg-cp $LCGVO -v -b --srm-timeout 3600 %s -D srmv2 %s file:%s' % (timeout_option, path, dest)
status, out, err = execute(cmd)
if status:
if self.__parse_srm_error__("SRM_INVALID_PATH", out, err):
raise exception.SourceNotFound(err)
raise exception.RucioException(err)
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def __gfal2_rm(self, paths):
"""
Uses gfal2 to remove the file.
:param path: Physical file name
:returns: 0 if removed successfully, other than 0 if failed
:raises SourceNotFound: if the source file was not found.
:raises RucioException: if it failed to remove the file.
"""
self.logger.debug('count: {}'.format(len(list(paths))))
ctx = self.__ctx
try:
for path in paths:
ret = ctx.unlink(str(path))
if ret:
return ret
return ret
except gfal2.GError as error: # pylint: disable=no-member
if error.code == errno.ENOENT or 'No such file' in str(error):
raise exception.SourceNotFound(error)
raise exception.RucioException(error)
def _get_credentials(rse, endpoint):
"""
Pass an endpoint and return its credentials.
:param endpoint: URL endpoint string.
:param rse: RSE name.
:returns: Dictionary of credentials.
"""
key = '%s_%s' % (rse, endpoint)
result = REGION.get(key)
if type(result) is NoValue:
try:
logging.debug("Loading account credentials")
result = config.get_rse_credentials(None)
if result and rse in result:
result = result[rse]
result['is_secure'] = result['is_secure'][endpoint]
REGION.set(key, result)
else:
raise Exception("Failed to load account credentials")
logging.debug("Loaded account credentials")
except KeyError as e:
raise exception.CannotAuthenticate(
'RSE %s endpoint %s not in rse account cfg: %s' %
(rse, endpoint, e))
except:
raise exception.RucioException(
"Failed to load credentials for RSE(%s) endpoint(%s), error: %s"
% (rse, endpoint, traceback.format_exc()))
return result
def get_metadata(urls, rse):
"""
Pass list of urls and return their metadata.
:param urls: A list of URL string.
:param rse: RSE name.
:returns: Dictonary of metadatas.
"""
result = {}
for url in urls:
try:
endpoint, bucket_name, key_name = _get_endpoint_bucket_key(url)
bucket = _get_bucket(rse, endpoint, bucket_name)
metadata = None
key = bucket.get_key(key_name)
if key is None:
metadata = exception.SourceNotFound('Key %s not found on %s' %
(key_name, endpoint))
else:
metadata = {'filesize': key.size}
result[url] = metadata
except boto.exception.S3ResponseError as e:
if e.status in [404, 403]:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
except exception.RucioException as e:
result[url] = e
except:
result[url] = exception.RucioException(
"Failed to get metadata for %s, error: %s" %
(endpoint, traceback.format_exc()))
return result
def rename(url, new_url, rse):
"""
Rename object.
:param url: URL string.
:param new_url: URL string.
:param rse: RSE name.
"""
try:
endpoint, bucket_name, key_name = _get_endpoint_bucket_key(url)
bucket = _get_bucket(rse, endpoint, bucket_name)
key = bucket.get_key(key_name)
if key is None:
raise exception.SourceNotFound('Key %s not found on %s' %
(key_name, endpoint))
new_endpoint, new_bucket_name, new_key_name = _get_endpoint_bucket_key(
new_url)
if endpoint != new_endpoint:
raise exception.RucioException(
"New endpont %s is different with old endpoint %s, cannot rename to different OS"
% (new_endpoint, endpoint))
key.copy(new_bucket_name, new_key_name)
key.delete()
except boto.exception.S3ResponseError as e:
if e.status in [404, 403]:
raise exception.DestinationNotAccessible(e)
else:
raise exception.ServiceUnavailable(e)
except exception.RucioException, e:
raise e
def delete(self, pfn):
""" Deletes a file from the connected RSE.
:param pfn Physical file name
:raises ServiceUnavailable, SourceNotFound, RSEAccessDenied, ResourceTemporaryUnavailable
"""
path = self.path2pfn(pfn)
try:
result = self.session.delete(path,
verify=False,
timeout=self.timeout,
cert=self.cert)
if result.status_code in [
204,
]:
return
elif result.status_code in [
404,
]:
raise exception.SourceNotFound()
elif result.status_code in [401, 403]:
raise exception.RSEAccessDenied()
elif result.status_code in [500, 503]:
raise exception.ResourceTemporaryUnavailable()
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def add_account_attribute(account, key, value, session=None):
"""
Add an attribute for the given account name.
:param key: the key for the new attribute.
:param value: the value for the new attribute.
:param account: the account to add the attribute to.
:param session: The database session in use.
"""
query = session.query(models.Account).filter_by(account=account, status=AccountStatus.ACTIVE)
try:
query.one()
except exc.NoResultFound:
raise exception.AccountNotFound("Account ID '{0}' does not exist".format(account))
new_attr = models.AccountAttrAssociation(account=account, key=key, value=value)
try:
new_attr.save(session=session)
except IntegrityError as error:
if match('.*IntegrityError.*ORA-00001: unique constraint.*ACCOUNT_ATTR_MAP_PK.*violated.*', error.args[0]) \
or match('.*IntegrityError.*1062, "Duplicate entry.*for key.*', error.args[0]) \
or error.args[0] == "(IntegrityError) column account/key is not unique" \
or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]):
raise exception.Duplicate('Key {0} already exist for account {1}!'.format(key, account))
except Exception:
raise exception.RucioException(str(format_exc()))
def exists(self, pfn):
""" Checks if the requested file is known by the referred RSE.
:param pfn Physical file name
:returns: True if the file exists, False if it doesn't
:raise ServiceUnavailable, RSEAccessDenied
"""
path = self.path2pfn(pfn)
try:
result = self.session.request('HEAD',
path,
verify=False,
timeout=self.timeout,
cert=self.cert)
if result.status_code == 200:
return True
elif result.status_code in [
401,
]:
raise exception.RSEAccessDenied()
elif result.status_code in [
404,
]:
return False
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
def get_rse_transfer_limits(rse_id=None, activity=None, session=None):
"""
Get RSE transfer limits.
:param rse_id: The RSE id.
:param activity: The activity.
:returns: A dictionary with the limits {'limit.activity': {'limit.rse_id': {'max_transfers': limit.max_transfers, 'transfers': 0, 'waitings': 0, 'volume': 1}}}.
"""
try:
query = session.query(models.RSETransferLimit)
if rse_id:
query = query.filter_by(rse_id=rse_id)
if activity:
query = query.filter_by(activity=activity)
limits = {}
for limit in query:
if limit.activity not in limits:
limits[limit.activity] = {}
limits[limit.activity][limit.rse_id] = {'max_transfers': limit.max_transfers,
'transfers': limit.transfers,
'waitings': limit.waitings,
'volume': limit.volume}
return limits
except IntegrityError as error:
raise exception.RucioException(error.args)
def put(self, filename, target, source_dir, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger(logging.DEBUG, 'rsync.put: filename: {} target: {}'.format(filename, target))
source_dir = source_dir or '.'
source_url = '%s/%s' % (source_dir, filename)
self.logger(logging.DEBUG, 'rsync.put: source url: {}'.format(source_url))
path = self.pfn2path(target)
pathdir = os.path.dirname(path)
if not os.path.exists(source_url):
raise exception.SourceNotFound()
try:
cmd = 'ssh -p %s %s%s "mkdir -p %s" && rsync -az -e "ssh -p %s" --append-verify %s %s%s:%s' % (self.port, self.sshuser, self.hostname, pathdir, self.port, source_url, self.sshuser, self.hostname, path)
self.logger(logging.DEBUG, 'rsync.put: cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def put(self, filename, target, source_dir, transfer_timeout=None):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:param transfer_timeout: Transfer timeout (in seconds) - dummy
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger.debug('xrootd.put: filename: {} target: {}'.format(
filename, target))
source_dir = source_dir or '.'
source_url = '%s/%s' % (source_dir, filename)
self.logger.debug('xrootd put: source url: {}'.format(source_url))
path = self.path2pfn(target)
if not os.path.exists(source_url):
raise exception.SourceNotFound()
try:
cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (source_url, path)
self.logger.info('xrootd.put: cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status != 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param pfn Current physical file name
:param new_pfn New physical file name
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger.debug('xrootd.rename: pfn: {}'.format(pfn))
if not self.exists(pfn):
raise exception.SourceNotFound()
try:
path = self.pfn2path(pfn)
new_path = self.pfn2path(new_pfn)
new_dir = new_path[:new_path.rindex('/') + 1]
cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mkdir -p %s' % (
self.hostname, self.port, new_dir)
self.logger.info('xrootd.stat: mkdir cmd: {}'.format(cmd))
status, out, err = execute(cmd)
cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mv %s %s' % (
self.hostname, self.port, path, new_path)
self.logger.info('xrootd.stat: rename cmd: {}'.format(cmd))
status, out, err = execute(cmd)
if status != 0:
raise exception.RucioException(err)
except Exception as e:
raise exception.ServiceUnavailable(e)
def __gfal2_rename(self, path, new_path):
"""
Uses gfal2 to rename a file.
:param path: path to the current file on the storage
:param new_path: path to the new file on the storage
:returns: 0 if it exists, -1 if it doesn't
:raises RucioException: if failed.
"""
ctx = self.__ctx
try:
dir_name = os.path.dirname(new_path)
# This function will be removed soon. gfal2 will create parent dir automatically.
try:
ctx.mkdir_rec(str(dir_name), 0775)
except Exception:
pass
ret = ctx.rename(str(path), str(new_path))
return ret
except gfal2.GError as error: # pylint: disable=no-member
if error.code == errno.ENOENT or 'No such file' in error.message:
raise exception.SourceNotFound(error)
raise exception.RucioException(error)
def get_space_usage(self):
"""
Get RSE space usage information.
:returns: a list with dict containing 'totalsize' and 'unusedsize'
:raises ServiceUnavailable: if some generic error occured in the library.
"""
endpoint_basepath = self.path2pfn(self.attributes['prefix'])
space_token = None
if self.attributes[
'extended_attributes'] is not None and 'space_token' in list(
self.attributes['extended_attributes'].keys()):
space_token = self.attributes['extended_attributes']['space_token']
if space_token is None or space_token == "":
raise exception.RucioException(
"Space token is not defined for protocol: %s" %
(self.attributes['scheme']))
try:
totalsize, unusedsize = self.__gfal2_get_space_usage(
endpoint_basepath, space_token)
return totalsize, unusedsize
except Exception as error:
raise exception.ServiceUnavailable(error)
def get_distances(src_rse_id=None, dest_rse_id=None, session=None):
"""
Get distances between rses.
:param src_rse_id: The source RSE ID.
:param dest_rse_id: The destination RSE ID.
:param session: The database session to use.
:returns distance: List of dictionaries.
"""
try:
query = session.query(Distance)
if src_rse_id:
query = query.filter(Distance.src_rse_id == src_rse_id)
if dest_rse_id:
query = query.filter(Distance.dest_rse_id == dest_rse_id)
distances = []
tmp = query.all()
if tmp:
for t in tmp:
t2 = dict(t)
t2.pop('_sa_instance_state')
distances.append(t2)
return distances
except IntegrityError, e:
raise exception.RucioException(e.args)
def get(self, path, dest):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
dest = os.path.abspath(dest)
if ':' not in dest:
dest = "file://" + dest
try:
status = self.__gfal2_copy(path, dest)
if status:
raise exception.RucioException()
except exception.DestinationNotAccessible as error:
raise exception.DestinationNotAccessible(str(error))
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def update_distances(src_rse_id=None, dest_rse_id=None, ranking=None, agis_distance=None, geoip_distance=None,
active=None, submitted=None, finished=None, failed=None, transfer_speed=None, session=None):
"""
Update distances with the given RSE ids.
:param src_rse_id: The source RSE ID.
:param dest_rse_id: The destination RSE ID.
:param ranking: Ranking as an integer.
:param agis_distance: AGIS Distance as an integer.
:param geoip_distance: GEOIP Distance as an integer.
:param active: Active FTS transfers as an integer.
:param submitted: Submitted FTS transfers as an integer.
:param finished: Finished FTS transfers as an integer.
:param failed: Failed FTS transfers as an integer.
:param transfer_speed: FTS transfer speed as an integer.
:param session: The database session to use.
"""
try:
distance = {'ranking': ranking, 'agis_distance': agis_distance, 'geoip_distance': geoip_distance,
'active': active, 'submitted': submitted, 'finished': finished, 'failed': failed,
'transfer_speed': transfer_speed}
query = session.query(Distance)
if src_rse_id:
query = query.filter(Distance.src_rse_id == src_rse_id)
if dest_rse_id:
query = query.filter(Distance.dest_rse_id == dest_rse_id)
query.update(distance)
except IntegrityError, e:
raise exception.RucioException(e.args)
def add_distance(src_rse_id, dest_rse_id, ranking=None, agis_distance=None, geoip_distance=None,
active=None, submitted=None, finished=None, failed=None, transfer_speed=None, session=None):
"""
Add a src-dest distance.
:param src_rse_id: The source RSE ID.
:param dest_rse_id: The destination RSE ID.
:param ranking: Ranking as an integer.
:param agis_distance: AGIS Distance as an integer.
:param geoip_distance: GEOIP Distance as an integer.
:param active: Active FTS transfers as an integer.
:param submitted: Submitted FTS transfers as an integer.
:param finished: Finished FTS transfers as an integer.
:param failed: Failed FTS transfers as an integer.
:param transfer_speed: FTS transfer speed as an integer.
:param session: The database session to use.
"""
try:
new_distance = Distance(src_rse_id=src_rse_id, dest_rse_id=dest_rse_id, ranking=ranking, agis_distance=agis_distance, geoip_distance=geoip_distance,
active=active, submitted=submitted, finished=finished, failed=failed, transfer_speed=transfer_speed)
new_distance.save(session=session)
except IntegrityError:
raise exception.Duplicate('Distance from %s to %s already exists!' % (src_rse_id, dest_rse_id))
except DatabaseError, e:
raise exception.RucioException(e.args)
def get(self, path, dest, transfer_timeout=None):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds)
:raises RucioException: Passthrough of gfal-copy error message.
"""
dest = os.path.abspath(dest)
if ':' not in dest:
dest = "file://" + dest
cmd = 'gfal-copy -vf -p -t %s -T %s %s %s' % (
transfer_timeout, transfer_timeout, path, dest)
self.logger(logging.DEBUG, 'Command: ' + cmd)
cmd = cmd.split()
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if p.returncode:
self.logger(logging.DEBUG, 'Error STDOUT: ' + str(stdout))
self.logger(logging.DEBUG, 'Error STDERR: ' + str(stderr))
raise exception.RucioException(str(stderr))
def export_distances(session=None):
"""
Export distances between all the RSEs using RSE ids.
:param session: The database session to use.
:returns distance: dictionary of dictionaries with all the distances.
"""
distances = {}
try:
rse_src = aliased(RSE)
rse_dest = aliased(RSE)
query = session.query(Distance, rse_src.id, rse_dest.id)\
.join(rse_src, rse_src.id == Distance.src_rse_id)\
.join(rse_dest, rse_dest.id == Distance.dest_rse_id)
for result in query.all():
distance = result[0]
src_id = result[1]
dst_id = result[2]
if src_id not in distances:
distances[src_id] = {}
distances[src_id][dst_id] = {}
distance['distance'] = distance['agis_distance']
distances[src_id][dst_id] = distance.to_dict()
del distances[src_id][dst_id]['_sa_instance_state']
return distances
except IntegrityError as error:
raise exception.RucioException(error.args)
def get(self, path, dest, transfer_timeout=None):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:param transfer_timeout: Transfer timeout (in seconds)
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
self.logger(logging.DEBUG,
'downloading file from {} to {}'.format(path, dest))
dest = os.path.abspath(dest)
if ':' not in dest:
dest = "file://" + dest
try:
status = self.__gfal2_copy(path,
dest,
transfer_timeout=transfer_timeout)
if status:
raise exception.RucioException()
except exception.DestinationNotAccessible as error:
raise exception.DestinationNotAccessible(str(error))
except exception.SourceNotFound as error:
raise exception.SourceNotFound(str(error))
except Exception as error:
raise exception.ServiceUnavailable(error)
def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_token=None, logger=_logger):
"""
Instanciates the protocol defined for the given operation.
:param rse_settings: RSE attributes
:param operation: Intended operation for this protocol
:param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation
:param domain: Optional specification of the domain
:param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication
:returns: An instance of the requested protocol
"""
# Verify feasibility of Protocol
operation = operation.lower()
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
if domain and domain not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain)
protocol_attr = select_protocol(rse_settings, operation, scheme, domain)
# Instantiate protocol
comp = protocol_attr['impl'].split('.')
mod = __import__('.'.join(comp[:-1]))
for n in comp[1:]:
try:
mod = getattr(mod, n)
except AttributeError as e:
logger.debug('Protocol implementations not supported.')
raise exception.RucioException(str(e)) # TODO: provide proper rucio exception
protocol_attr['auth_token'] = auth_token
protocol = mod(protocol_attr, rse_settings, logger=logger)
return protocol
def mkdir(self, directory):
""" Internal method to create directories
:param directory Name of the directory that needs to be created
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied
"""
path = self.path2pfn(directory)
try:
result = self.session.request('MKCOL',
path,
verify=False,
timeout=self.timeout,
cert=self.cert)
if result.status_code in [201, 405
]: # Success or directory already exists
return
elif result.status_code in [
404,
]:
raise exception.SourceNotFound()
elif result.status_code in [
401,
]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code, result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)
def _get_bucket(rse, endpoint, bucket_name, operation='read'):
"""
Pass an endpoint and return a connection to object store.
:param rse: RSE name.
:param endpoint: URL endpoint string.
:returns: Connection object.
"""
key = "%s:%s:%s" % (rse, endpoint, bucket_name)
result = REGION.get(key)
if type(result) is NoValue:
try:
logging.debug("Creating bucket object")
result = None
conn = _get_connection(rse, endpoint)
bucket = conn.get_bucket(bucket_name)
if operation == 'read':
if bucket is None:
raise exception.SourceNotFound(
'Bucket %s not found on %s' % (bucket_name, rse))
else:
result = bucket
REGION.set(key, result)
else:
result = conn.create_bucket(bucket_name)
REGION.set(key, result)
except exception.RucioException as e:
raise e
except:
raise exception.RucioException(
"Failed to get bucket on RSE(%s), error: %s" %
(rse, traceback.format_exc()))
return result
def add_vo(vo, description, password, email, session=None):
"""
Add a VO and setup a new root user.
New root user will have account name 'root' and a userpass identity with username: '******' and password from the rootpass parameter
:param vo: 3-letter unique tag for a VO.
:param descrition: Descriptive string for the VO (e.g. Full name).
:param email: Contact email for the VO.
:param password: The password to set for the root user of the new VO
:param session: The db session in use.
"""
if len(vo) != 3:
raise exception.RucioException('Invalid VO tag, must be 3 chars.')
new_vo = models.VO(vo=vo, description=description, email=email)
try:
new_vo.save(session=session)
except IntegrityError:
raise exception.Duplicate('VO {} already exists!'.format(vo))
except DatabaseError as error:
raise exception.RucioException(error.args)
from rucio.core.account import add_account, list_identities
from rucio.core.identity import add_account_identity
new_root = InternalAccount('root', vo=vo)
add_account(account=new_root,
type=AccountType.from_sym('SERVICE'),
email=email,
session=session)
add_account_identity(identity='root@{}'.format(vo),
type=IdentityType.from_sym('userpass'),
account=new_root,
email=email,
default=False,
password=password,
session=session)
for ident in list_identities(account=InternalAccount('super_root',
vo='def'),
session=session):
add_account_identity(identity=ident['identity'],
type=ident['type'],
account=new_root,
email='',
session=session)
def rename(self, pfn, new_pfn):
""" Allows to rename a file stored inside the connected RSE.
:param pfn Current physical file name
:param new_pfn New physical file name
:raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound, RSEAccessDenied
"""
path = self.path2pfn(pfn)
new_path = self.path2pfn(new_pfn)
directories = new_path.split('/')
headers = {'Destination': new_path}
# Try the rename without testing the existence of the destination directory
try:
result = self.session.request('MOVE',
path,
verify=False,
headers=headers,
timeout=self.timeout,
cert=self.cert)
if result.status_code == 201:
return
elif result.status_code in [
404,
]:
raise exception.SourceNotFound()
else:
# Create the directories before issuing the MOVE
for directory_level in reversed(list(range(1, 4))):
upper_directory = "/".join(directories[:-directory_level])
self.mkdir(upper_directory)
try:
result = self.session.request('MOVE',
path,
verify=False,
headers=headers,
timeout=self.timeout,
cert=self.cert)
if result.status_code == 201:
return
elif result.status_code in [
404,
]:
raise exception.SourceNotFound()
elif result.status_code in [
401,
]:
raise exception.RSEAccessDenied()
else:
# catchall exception
raise exception.RucioException(result.status_code,
result.text)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ConnectionError as error:
raise exception.ServiceUnavailable(error)
except requests.exceptions.ReadTimeout as error:
raise exception.ServiceUnavailable(error)