def __lfns2pfns_client(self, lfns):
""" Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if neccessary.
:param scope: list of DIDs
:returns: dict with scope:name as keys and PFN as value (in case of errors the Rucio exception si assigned to the key)
"""
client = ReplicaClient()
pfns = {}
lfns = [lfns] if type(lfns) == dict else lfns
for lfn in lfns:
scope = lfn['scope']
name = lfn['name']
replicas = [r for r in client.list_replicas([{'scope': scope, 'name': name}, ], schemes=[self.attributes['scheme'], ])] # schemes is used to narrow down the response message.
if len(replicas) > 1:
pfns['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('This operation can only be performed for files.')
if not len(replicas):
pfns['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('File not found.')
pfns['%s:%s' % (scope, name)] = replicas[0]['rses'][self.rse['rse']][0] if (self.rse['rse'] in replicas[0]['rses'].keys()) else exception.RSEOperationNotSupported('Replica not found on given RSE.')
return pfns
def get_protocols_ordered(rse_settings, operation, scheme=None, domain='wan'):
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation %s is not supported' % operation)
if domain and domain not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'Domain %s not supported' % domain)
candidates = _get_possible_protocols(rse_settings, operation, scheme,
domain)
candidates.sort(key=lambda k: k['domains'][domain][operation])
return candidates
def create_protocol(rse_settings, operation, scheme=None):
""" Instanciates the protocol defined for the given operation.
:param rse_attr: RSE attributes
:param operation: the intended operation for this protocol
:param scheme: optional filter if no specific protocol is defined in rse_setting for the provided operation
:returns: an instance of the requested protocol
"""
# Verify feasibility of Protocol
operation = operation.lower()
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
rse_settings['domain'] = [rse_settings['domain']] if type(rse_settings['domain']) is not list else rse_settings['domain']
for domain in rse_settings['domain']:
if domain.lower() not in utils.rse_supported_protocol_domains():
raise exception.RSEOperationNotSupported('Domain %s not supported' % rse_settings['domain'])
if rse_settings['%s_protocol' % operation] == DEFAULT_PROTOCOL:
protocol_attr = select_protocol(rse_settings, operation, scheme)
else:
protocol_attr = rse_settings['%s_protocol' % operation]
for d in rse_settings['domain']:
if protocol_attr['domains'][d][operation] == 0:
raise exception.RSEOperationNotSupported('Operation %s for domain %s not supported by %s' % (operation, rse_settings['domain'], protocol_attr['scheme']))
# Instanciate protocol
comp = protocol_attr['impl'].split('.')
mod = __import__('.'.join(comp[:-1]))
for n in comp[1:]:
try:
mod = getattr(mod, n)
except AttributeError:
print 'Protocol implementation not found'
raise # TODO: provide proper rucio exception
protocol = mod(protocol_attr, rse_settings)
return protocol
def select_protocol(rse_settings, operation, scheme=None, domain='wan'):
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation %s is not supported' % operation)
if domain and domain not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'Domain %s not supported' % domain)
candidates = _get_possible_protocols(rse_settings, operation, scheme,
domain)
# Shuffle candidates to load-balance over equal sources
random.shuffle(candidates)
return min(candidates, key=lambda k: k['domains'][domain][operation])
def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_token=None, protocol_attr=None, logger=logging.log):
"""
Instanciates the protocol defined for the given operation.
:param rse_settings: RSE attributes
:param operation: Intended operation for this protocol
:param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation
:param domain: Optional specification of the domain
:param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication
:param protocol_attr: Optionally passing the full protocol availability information to correctly select WAN/LAN
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
:returns: An instance of the requested protocol
"""
# Verify feasibility of Protocol
operation = operation.lower()
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
if domain and domain not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain)
if not protocol_attr:
protocol_attr = select_protocol(rse_settings, operation, scheme, domain)
else:
candidates = _get_possible_protocols(rse_settings, operation, scheme, domain)
if protocol_attr not in candidates:
raise exception.RSEProtocolNotSupported('Protocol %s operation %s on domain %s not supported' % (protocol_attr, operation, domain))
# Instantiate protocol
comp = protocol_attr['impl'].split('.')
prefix = '.'.join(comp[-2:]) + ': '
logger = formatted_logger(logger, prefix + "%s")
mod = __import__('.'.join(comp[:-1]))
for n in comp[1:]:
try:
mod = getattr(mod, n)
except AttributeError as e:
logger(logging.DEBUG, 'Protocol implementations not supported.')
raise exception.RucioException(str(e)) # TODO: provide proper rucio exception
protocol_attr['auth_token'] = auth_token
protocol = mod(protocol_attr, rse_settings, logger=logger)
return protocol
def create_protocol(rse_settings,
operation,
scheme=None,
domain='wan',
auth_token=None,
logger=_logger):
"""
Instanciates the protocol defined for the given operation.
:param rse_settings: RSE attributes
:param operation: Intended operation for this protocol
:param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation
:param domain: Optional specification of the domain
:param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication
:returns: An instance of the requested protocol
"""
# Verify feasibility of Protocol
operation = operation.lower()
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation %s is not supported' % operation)
if domain and domain not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'Domain %s not supported' % domain)
protocol_attr = select_protocol(rse_settings, operation, scheme, domain)
# Instantiate protocol
comp = protocol_attr['impl'].split('.')
mod = __import__('.'.join(comp[:-1]))
for n in comp[1:]:
try:
mod = getattr(mod, n)
except AttributeError as e:
logger.debug('Protocol implementations not supported.')
raise exception.RucioException(
str(e)) # TODO: provide proper rucio exception
protocol_attr['auth_token'] = auth_token
protocol = mod(protocol_attr, rse_settings, logger=logger)
return protocol
def del_rse(rse, session=None):
"""
Disable a rse with the given rse name.
:param rse: the rse name.
:param session: The database session in use.
"""
old_rse = None
try:
old_rse = session.query(models.RSE).filter_by(rse=rse).one()
if not rse_is_empty(rse=rse, session=session):
raise exception.RSEOperationNotSupported(
'RSE \'%s\' is not empty' % rse)
except sqlalchemy.orm.exc.NoResultFound:
raise exception.RSENotFound('RSE \'%s\' cannot be found' % rse)
old_rse.delete(session=session)
try:
del_rse_attribute(rse=rse, key=rse, session=session)
except exception.RSEAttributeNotFound:
pass
def upload(rse_settings,
lfns,
source_dir=None,
force_pfn=None,
force_scheme=None,
transfer_timeout=None):
"""
Uploads a file to the connected storage.
Providing a list indicates the bulk mode.
:param lfns: a single dict or a list with dicts containing 'scope' and 'name'.
Examples:
[
{'name': '1_rse_local_put.raw', 'scope': 'user.jdoe', 'filesize': 42, 'adler32': '87HS3J968JSNWID'},
{'name': '2_rse_local_put.raw', 'scope': 'user.jdoe', 'filesize': 4711, 'adler32': 'RSSMICETHMISBA837464F'}
]
If the 'filename' key is present, it will be used by Rucio as the actual name of the file on disk (separate from the Rucio 'name').
:param source_dir: path to the local directory including the source files
:param force_pfn: use the given PFN -- can lead to dark data, use sparingly
:param force_scheme: use the given protocol scheme, overriding the protocol priority in the RSE description
:param transfer_timeout: set this timeout (in seconds) for the transfers, for protocols that support it
:returns: True/False for a single file or a dict object with 'scope:name' as keys and True or the exception as value for each file in bulk mode
:raises RSENotConnected: no connection to a specific storage has been established
:raises SourceNotFound: local source file can not be found
:raises DestinationNotAccessible: remote destination directory is not accessible
:raises ServiceUnavailable: for any other reason
"""
ret = {}
gs = True # gs represents the global status which indicates if every operation worked in bulk mode
protocol = create_protocol(rse_settings, 'write', scheme=force_scheme)
protocol.connect()
protocol_delete = create_protocol(rse_settings, 'delete')
protocol_delete.connect()
lfns = [lfns] if not type(lfns) is list else lfns
for lfn in lfns:
base_name = lfn.get('filename', lfn['name'])
name = lfn.get('name', base_name)
scope = lfn['scope']
if 'adler32' not in lfn:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Missing checksum for file %s:%s' % (lfn['scope'], name))
continue
if 'filesize' not in lfn:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Missing filesize for file %s:%s' % (lfn['scope'], name))
continue
if force_pfn:
pfn = force_pfn
else:
pfn = list(protocol.lfns2pfns(make_valid_did(lfn)).values())[0]
if isinstance(pfn, exception.RucioException):
raise pfn
# First check if renaming operation is supported
if protocol.renaming:
# Check if file replica is already on the storage system
if protocol.overwrite is False and protocol.exists(pfn):
ret['%s:%s' % (
scope, name
)] = exception.FileReplicaAlreadyExists(
'File %s in scope %s already exists on storage as PFN %s' %
(name, scope, pfn))
gs = False
else:
if protocol.exists(
'%s.rucio.upload' % pfn
): # Check for left over of previous unsuccessful attempts
try:
protocol_delete.delete('%s.rucio.upload' % list(
protocol_delete.lfns2pfns(
make_valid_did(lfn)).values())[0])
except Exception as e:
ret['%s:%s' % (
scope, name
)] = exception.RSEOperationNotSupported(
'Unable to remove temporary file %s.rucio.upload: %s'
% (pfn, str(e)))
gs = False
continue
try: # Try uploading file
protocol.put(base_name,
'%s.rucio.upload' % pfn,
source_dir,
transfer_timeout=transfer_timeout)
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
valid = None
try: # Get metadata of file to verify if upload was successful
try:
stats = protocol.stat('%s.rucio.upload' % pfn)
if ('adler32' in stats) and ('adler32' in lfn):
valid = stats['adler32'] == lfn['adler32']
if (valid is None) and ('filesize'
in stats) and ('filesize'
in lfn):
valid = stats['filesize'] == lfn['filesize']
except exception.RSEChecksumUnavailable as e:
if rse_settings['verify_checksum'] is False:
valid = True
else:
raise exception.RucioException(
'Checksum not validated')
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
if valid: # The upload finished successful and the file can be renamed
try:
protocol.rename('%s.rucio.upload' % pfn, pfn)
ret['%s:%s' % (scope, name)] = True
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
else:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Replica %s is corrupted.' % pfn)
else:
# Check if file replica is already on the storage system
if protocol.overwrite is False and protocol.exists(pfn):
ret['%s:%s' % (
scope, name
)] = exception.FileReplicaAlreadyExists(
'File %s in scope %s already exists on storage as PFN %s' %
(name, scope, pfn))
gs = False
else:
try: # Try uploading file
protocol.put(base_name,
pfn,
source_dir,
transfer_timeout=transfer_timeout)
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
valid = None
try: # Get metadata of file to verify if upload was successful
try:
stats = protocol.stat(pfn)
if ('adler32' in stats) and ('adler32' in lfn):
valid = stats['adler32'] == lfn['adler32']
if (valid is None) and ('filesize'
in stats) and ('filesize'
in lfn):
valid = stats['filesize'] == lfn['filesize']
except exception.RSEChecksumUnavailable as e:
if rse_settings['verify_checksum'] is False:
valid = True
else:
raise exception.RucioException(
'Checksum not validated')
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
if not valid:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Replica %s is corrupted.' % pfn)
protocol.close()
protocol_delete.close()
if len(ret) == 1:
for x in ret:
if isinstance(ret[x], Exception):
raise ret[x]
else:
return {'success': ret[x], 'pfn': pfn}
return [gs, ret]
def upload(rse_settings,
lfns,
source_dir=None,
force_pfn=None,
force_scheme=None,
transfer_timeout=None,
delete_existing=False,
sign_service=None,
auth_token=None):
"""
Uploads a file to the connected storage.
Providing a list indicates the bulk mode.
:rse_settings: RSE attributes
:param lfns: a single dict or a list with dicts containing 'scope' and 'name'.
Examples:
[
{'name': '1_rse_local_put.raw', 'scope': 'user.jdoe', 'filesize': 42, 'adler32': '87HS3J968JSNWID'},
{'name': '2_rse_local_put.raw', 'scope': 'user.jdoe', 'filesize': 4711, 'adler32': 'RSSMICETHMISBA837464F'}
]
If the 'filename' key is present, it will be used by Rucio as the actual name of the file on disk (separate from the Rucio 'name').
:param source_dir: path to the local directory including the source files
:param force_pfn: use the given PFN -- can lead to dark data, use sparingly
:param force_scheme: use the given protocol scheme, overriding the protocol priority in the RSE description
:param transfer_timeout: set this timeout (in seconds) for the transfers, for protocols that support it
:param sign_service: use the given service (e.g. gcs, s3, swift) to sign the URL
:param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication
:returns: True/False for a single file or a dict object with 'scope:name' as keys and True or the exception as value for each file in bulk mode
:raises RSENotConnected: no connection to a specific storage has been established
:raises SourceNotFound: local source file can not be found
:raises DestinationNotAccessible: remote destination directory is not accessible
:raises ServiceUnavailable: for any other reason
"""
ret = {}
gs = True # gs represents the global status which indicates if every operation worked in bulk mode
protocol = create_protocol(rse_settings,
'write',
scheme=force_scheme,
auth_token=auth_token)
protocol.connect()
protocol_delete = create_protocol(rse_settings,
'delete',
auth_token=auth_token)
protocol_delete.connect()
lfns = [lfns] if not type(lfns) is list else lfns
for lfn in lfns:
base_name = lfn.get('filename', lfn['name'])
name = lfn.get('name', base_name)
scope = lfn['scope']
if 'adler32' not in lfn:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Missing checksum for file %s:%s' % (lfn['scope'], name))
continue
if 'filesize' not in lfn:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Missing filesize for file %s:%s' % (lfn['scope'], name))
continue
if force_pfn:
pfn = force_pfn
readpfn = force_pfn
else:
pfn = list(protocol.lfns2pfns(make_valid_did(lfn)).values())[0]
if isinstance(pfn, exception.RucioException):
raise pfn
readpfn = pfn
if sign_service is not None:
# need a separate signed URL for read operations (exists and stat)
readpfn = __get_signed_url(rse_settings['rse'], sign_service,
'read', pfn) # NOQA pylint: disable=undefined-variable
pfn = __get_signed_url(rse_settings['rse'], sign_service,
'write', pfn) # NOQA pylint: disable=undefined-variable
# First check if renaming operation is supported
if protocol.renaming:
# Check if file replica is already on the storage system
if protocol.overwrite is False and delete_existing is False and protocol.exists(
pfn):
ret['%s:%s' % (
scope, name
)] = exception.FileReplicaAlreadyExists(
'File %s in scope %s already exists on storage as PFN %s' %
(name, scope, pfn))
gs = False
else:
if protocol.exists(
'%s.rucio.upload' % pfn
): # Check for left over of previous unsuccessful attempts
try:
protocol_delete.delete('%s.rucio.upload' % list(
protocol_delete.lfns2pfns(
make_valid_did(lfn)).values())[0])
except Exception as e:
ret['%s:%s' % (
scope, name
)] = exception.RSEOperationNotSupported(
'Unable to remove temporary file %s.rucio.upload: %s'
% (pfn, str(e)))
gs = False
continue
if delete_existing:
if protocol.exists(
'%s' % pfn
): # Check for previous completed uploads that have to be removed before upload
try:
protocol_delete.delete('%s' % list(
protocol_delete.lfns2pfns(
make_valid_did(lfn)).values())[0])
except Exception as e:
ret['%s:%s' %
(scope,
name)] = exception.RSEOperationNotSupported(
'Unable to remove file %s: %s' %
(pfn, str(e)))
gs = False
continue
try: # Try uploading file
protocol.put(base_name,
'%s.rucio.upload' % pfn,
source_dir,
transfer_timeout=transfer_timeout)
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
valid = None
try: # Get metadata of file to verify if upload was successful
try:
stats = _retry_protocol_stat(protocol,
'%s.rucio.upload' % pfn)
# Verify all supported checksums and keep rack of the verified ones
verified_checksums = []
for checksum_name in GLOBALLY_SUPPORTED_CHECKSUMS:
if (checksum_name in stats) and (checksum_name
in lfn):
verified_checksums.append(
stats[checksum_name] == lfn[checksum_name])
# Upload is successful if at least one checksum was found
valid = any(verified_checksums)
if not valid and ('filesize' in stats) and ('filesize'
in lfn):
valid = stats['filesize'] == lfn['filesize']
except NotImplementedError:
if rse_settings['verify_checksum'] is False:
valid = True
else:
raise exception.RucioException(
'Checksum not validated')
except exception.RSEChecksumUnavailable as e:
if rse_settings['verify_checksum'] is False:
valid = True
else:
raise exception.RucioException(
'Checksum not validated')
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
if valid: # The upload finished successful and the file can be renamed
try:
protocol.rename('%s.rucio.upload' % pfn, pfn)
ret['%s:%s' % (scope, name)] = True
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
else:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Replica %s is corrupted.' % pfn)
else:
# Check if file replica is already on the storage system
if protocol.overwrite is False and delete_existing is False and protocol.exists(
readpfn):
ret['%s:%s' % (
scope, name
)] = exception.FileReplicaAlreadyExists(
'File %s in scope %s already exists on storage as PFN %s' %
(name, scope, pfn))
gs = False
else:
try: # Try uploading file
protocol.put(base_name,
pfn,
source_dir,
transfer_timeout=transfer_timeout)
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
valid = None
try: # Get metadata of file to verify if upload was successful
try:
stats = _retry_protocol_stat(protocol, pfn)
# Verify all supported checksums and keep rack of the verified ones
verified_checksums = []
for checksum_name in GLOBALLY_SUPPORTED_CHECKSUMS:
if (checksum_name in stats) and (checksum_name
in lfn):
verified_checksums.append(
stats[checksum_name] == lfn[checksum_name])
# Upload is successful if at least one checksum was found
valid = any(verified_checksums)
if not valid and ('filesize' in stats) and ('filesize'
in lfn):
valid = stats['filesize'] == lfn['filesize']
except NotImplementedError:
if rse_settings['verify_checksum'] is False:
valid = True
else:
raise exception.RucioException(
'Checksum not validated')
except exception.RSEChecksumUnavailable as e:
if rse_settings['verify_checksum'] is False:
valid = True
else:
raise exception.RucioException(
'Checksum not validated')
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
if not valid:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Replica %s is corrupted.' % pfn)
protocol.close()
protocol_delete.close()
if len(ret) == 1:
for x in ret:
if isinstance(ret[x], Exception):
raise ret[x]
else:
return {'success': ret[x], 'pfn': pfn}
return {0: gs, 1: ret, 'success': gs, 'pfn': pfn}
def add_protocol(rse, parameter, session=None):
"""
Add a protocol to an existing RSE. If entries with equal or less priority for
an operation exist, the existing one will be reorded (i.e. +1).
:param rse: the name of the new rse.
:param parameter: parameters of the new protocol entry.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEOperationNotSupported: If no scheme supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the scheme is to big or below zero.
:raises Duplicate: If scheme with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
# Insert new protocol entry
parameter['rse_id'] = rid
# Default values
parameter['port'] = parameter.get('port', 0)
parameter['hostname'] = parameter.get('hostname', 'localhost')
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in parameter.keys():
for s in parameter['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'The protocol domain \'%s\' is not defined in the schema.'
% s)
for op in parameter['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation \'%s\' not defined in schema.' % (op))
op_name = op if op == 'third_party_copy' else ''.join(
[op, '_', s]).lower()
if parameter['domains'][s][op] < 0:
raise exception.RSEProtocolPriorityError(
'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.'
% (parameter['domains'][s][op], op, s))
parameter[op_name] = parameter['domains'][s][op]
del parameter['domains']
if ('extended_attributes'
in parameter) and parameter['extended_attributes']:
try:
parameter['extended_attributes'] = json.dumps(
parameter['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if parameter['scheme'] == 'srm':
if ('extended_attributes'
not in parameter) or ('web_service_path'
not in parameter['extended_attributes']):
raise exception.InvalidObject(
'Missing values! For SRM, extended_attributes and web_service_path must be specified'
)
try:
new_protocol = models.RSEProtocols()
new_protocol.update(parameter)
new_protocol.save(session=session)
except (IntegrityError, FlushError, OperationalError) as error:
if ('UNIQUE constraint failed' in error.args[0]) or ('conflicts with persistent instance' in error.args[0]) \
or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \
or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0])\
or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]):
raise exception.Duplicate(
'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.'
% (parameter['scheme'], parameter['port'], rse,
parameter['hostname']))
elif 'may not be NULL' in error.args[0] \
or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \
or match('.*OperationalError.*cannot be null.*', error.args[0]):
raise exception.InvalidObject('Missing values!')
raise error
return new_protocol
def update_protocols(rse, scheme, data, hostname, port, session=None):
"""
Updates an existing protocol entry for an RSE. If necessary, priorities for read,
write, and delete operations of other protocol entires will be updated too.
:param rse: the name of the new rse.
:param scheme: Protocol identifer.
:param data: Dict with new values (keys must match column names in the database).
:param hostname: Hostname defined for the scheme, used if more than one scheme
is registered with the same identifier.
:param port: The port registered for the hostename, used if more than one scheme
is regsitered with the same identifier and hostname.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE.
:raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero.
:raises KeyNotFound: Invalid data for update provided.
:raises Duplicate: If protocol with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in data:
for s in data['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'The protocol domain \'%s\' is not defined in the schema.'
% s)
for op in data['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation \'%s\' not defined in schema.' % (op))
op_name = op
if op != 'third_party_copy':
op_name = ''.join([op, '_', s])
no = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= 0)).\
count()
if not 0 <= data['domains'][s][op] <= no:
raise exception.RSEProtocolPriorityError(
'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.'
% (data['domains'][s][op], op, s))
data[op_name] = data['domains'][s][op]
del data['domains']
if 'extended_attributes' in data:
try:
data['extended_attributes'] = json.dumps(
data['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
terms = [
models.RSEProtocols.rse_id == rid,
models.RSEProtocols.scheme == scheme,
models.RSEProtocols.hostname == hostname,
models.RSEProtocols.port == port
]
try:
up = session.query(models.RSEProtocols).filter(*terms).first()
if up is None:
msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % (
rse, scheme, hostname, port)
raise exception.RSEProtocolNotSupported(msg)
# Preparing gaps if priority is updated
for domain in utils.rse_supported_protocol_domains():
for op in utils.rse_supported_protocol_operations():
op_name = op
if op != 'third_party_copy':
op_name = ''.join([op, '_', domain])
if op_name in data:
prots = []
if (not getattr(up, op_name)) and data[
op_name]: # reactivate protocol e.g. from 0 to 1
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name])).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(up, op_name) and (
not data[op_name]
): # deactivate protocol e.g. from 1 to 0
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
elif getattr(
up, op_name
) > data[op_name]: # shift forward e.g. from 5 to 2
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name],
getattr(models.RSEProtocols, op_name) < getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(
up, op_name
) < data[op_name]: # shift backward e.g. from 1 to 3
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) <= data[op_name],
getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
for p in prots:
p.update({op_name: val})
val += 1
up.update(data, flush=True, session=session)
except (IntegrityError, OperationalError) as error:
if 'UNIQUE'.lower() in error.args[0].lower(
) or 'Duplicate' in error.args[
0]: # Covers SQLite, Oracle and MySQL error
raise exception.Duplicate(
'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.'
% (scheme, port, rse, hostname))
elif 'may not be NULL' in error.args[
0] or "cannot be null" in error.args[0]:
raise exception.InvalidObject('Missing values: %s' % error.args[0])
raise error
except DatabaseError as error:
if match(
'.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*',
error.args[0]):
raise exception.InvalidObject('Invalid values !')
raise error
def upload(rse_settings, lfns, source_dir=None, force_pfn=None):
"""
Uploads a file to the connected storage.
Providing a list indicates the bulk mode.
:param lfns: a single dict or a list with dicts containing 'scope' and 'name'. E.g. [{'name': '1_rse_local_put.raw', 'scope': 'user.jdoe', 'filesize': 42, 'adler32': '87HS3J968JSNWID'},
{'name': '2_rse_local_put.raw', 'scope': 'user.jdoe', 'filesize': 4711, 'adler32': 'RSSMICETHMISBA837464F'}]
:param source_dir: path to the local directory including the source files
:param force_pfn: use the given PFN -- can lead to dark data, use sparingly
:returns: True/False for a single file or a dict object with 'scope:name' as keys and True or the exception as value for each file in bulk mode
:raises RSENotConnected: no connection to a specific storage has been established
:raises SourceNotFound: local source file can not be found
:raises DestinationNotAccessible: remote destination directory is not accessible
:raises ServiceUnavailable: for any other reason
"""
ret = {}
gs = True # gs represents the global status which indicates if every operation worked in bulk mode
protocol = create_protocol(rse_settings, 'write')
protocol.connect()
protocol_delete = create_protocol(rse_settings, 'delete')
protocol_delete.connect()
lfns = [lfns] if not type(lfns) is list else lfns
for lfn in lfns:
name = lfn['name']
scope = lfn['scope']
if 'adler32' not in lfn:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Missing checksum for file %s:%s' %
(lfn['scope'], lfn['name']))
continue
if 'filesize' not in lfn:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Missing filesize for file %s:%s' %
(lfn['scope'], lfn['name']))
continue
if force_pfn:
pfn = force_pfn
else:
pfn = protocol.lfns2pfns(lfn).values()[0]
# First check if renaming operation is supported
if protocol.renaming:
# Check if file replica is already on the storage system
if protocol.overwrite is False and protocol.exists(pfn):
ret['%s:%s' %
(scope, name)] = exception.FileReplicaAlreadyExists(
'File %s in scope %s already exists on storage' %
(name, scope))
gs = False
else:
if protocol.exists(
'%s.rucio.upload' % pfn
): # Check for left over of previous unsuccessful attempts
try:
protocol_delete.delete(
'%s.rucio.upload',
protocol_delete.lfns2pfns(lfn).values()[0])
except Exception as e:
ret['%s:%s' % (
scope, name
)] = exception.RSEOperationNotSupported(
'Unable to remove temporary file %s.rucio.upload: %s'
% (pfn, str(e)))
try: # Try uploading file
protocol.put(name, '%s.rucio.upload' % pfn, source_dir)
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
valid = None
try: # Get metadata of file to verify if upload was successful
stats = protocol.stat('%s.rucio.upload' % pfn)
if ('adler32' in stats) and ('adler32' in lfn):
valid = stats['adler32'] == lfn['adler32']
if (valid is None) and ('filesize'
in stats) and ('filesize' in lfn):
valid = stats['filesize'] == lfn['filesize']
except NotImplementedError:
valid = False
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
if valid: # The upload finished successful and the file can be renamed
try:
protocol.rename('%s.rucio.upload' % pfn, pfn)
ret['%s:%s' % (scope, name)] = True
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
else:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Replica %s is corrupted.' % pfn)
else:
# Check if file replica is already on the storage system
if protocol.overwrite is False and protocol.exists(pfn):
ret['%s:%s' %
(scope, name)] = exception.FileReplicaAlreadyExists(
'File %s in scope %s already exists on storage' %
(name, scope))
gs = False
else:
try: # Try uploading file
protocol.put(name, pfn, source_dir)
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
valid = None
try: # Get metadata of file to verify if upload was successful
stats = protocol.stat(pfn)
if ('adler32' in stats) and ('adler32' in lfn):
valid = stats['adler32'] == lfn['adler32']
if (valid is None) and ('filesize'
in stats) and ('filesize' in lfn):
valid = stats['filesize'] == lfn['filesize']
except NotImplementedError:
valid = True # If the protocol doesn't support stat of a file, we agreed on assuming that the file was uploaded without error
except Exception as e:
gs = False
ret['%s:%s' % (scope, name)] = e
continue
if not valid:
gs = False
ret['%s:%s' % (scope, name)] = exception.RucioException(
'Replica %s is corrupted.' % pfn)
protocol.close()
protocol_delete.close()
if len(ret) == 1:
for x in ret:
if isinstance(ret[x], Exception):
raise ret[x]
else:
return {'success': ret[x], 'pfn': pfn}
return [gs, ret]