def create_protocol(rse_settings, operation, scheme=None, domain='wan'):
"""
Instanciates the protocol defined for the given operation.
:param rse_attr: RSE attributes
:param operation: Intended operation for this protocol
:param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation
:param domain: Optional specification of the domain
:returns: An instance of the requested protocol
"""
# Verify feasibility of Protocol
operation = operation.lower()
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation %s is not supported' % operation)
if domain and domain not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'Domain %s not supported' % domain)
protocol_attr = select_protocol(rse_settings, operation, scheme, domain)
# Instantiate protocol
comp = protocol_attr['impl'].split('.')
mod = __import__('.'.join(comp[:-1]))
for n in comp[1:]:
try:
mod = getattr(mod, n)
except AttributeError:
print('Protocol implementation not found')
raise # TODO: provide proper rucio exception
protocol = mod(protocol_attr, rse_settings)
return protocol
def _get_possible_protocols(rse_settings, operation, scheme=None):
""" Filter the list of available protocols or provided by the supported ones.
:param rse_settings: The rse settings.
:param operation: The operation (write, read).
:param scheme: optional filter if no specific protocol is defined in
rse_setting for the provided operation.
:returns: The list of possible protocols.
"""
operation = operation.lower()
candidates = rse_settings['protocols']
if type(rse_settings['domain']) is not list:
raise exception.RSEProtocolDomainNotSupported(
'Domain setting must be list.')
for d in rse_settings['domain']:
if d not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'Domain %s is not supported'
' by Rucio.' % d)
# convert scheme to list, if given as string
if scheme and not isinstance(scheme, list):
scheme = scheme.split(',')
tbr = []
for protocol in candidates:
# Check if scheme given and filter if so
if scheme and protocol['scheme'] not in scheme:
tbr.append(protocol)
continue
filtered = True
for d in rse_settings['domain']:
if protocol['domains'][d][operation] != 0:
filtered = False
if filtered:
tbr.append(protocol)
if len(candidates) <= len(tbr):
raise exception.RSEProtocolNotSupported(
'No protocol for provided settings'
' found : %s.' % str(rse_settings))
return [c for c in candidates if c not in tbr]
def get_protocols_ordered(rse_settings, operation, scheme=None, domain='wan'):
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
if domain and domain not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain)
candidates = _get_possible_protocols(rse_settings, operation, scheme, domain)
candidates.sort(key=lambda k: k['domains'][domain][operation])
return candidates
def select_protocol(rse_settings, operation, scheme=None, domain='wan'):
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
if domain and domain not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain)
candidates = _get_possible_protocols(rse_settings, operation, scheme, domain)
# Shuffle candidates to load-balance over equal sources
random.shuffle(candidates)
return min(candidates, key=lambda k: k['domains'][domain][operation])
def select_protocol(rse_settings, operation, scheme=None):
operation = operation.lower()
candidates = copy.copy(rse_settings['protocols'])
if type(rse_settings['domain']) is not list:
raise exception.RSEProtocolDomainNotSupported('Domain setting must be list.')
for d in rse_settings['domain']:
if d not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported('Domain %s is not supported by Rucio.' % d)
tbr = list()
for protocol in candidates:
# Check if scheme given and filter if so
if scheme:
if not isinstance(scheme, list):
scheme = scheme.split(',')
if protocol['scheme'] not in scheme:
tbr.append(protocol)
continue
for d in rse_settings['domain']:
if protocol['domains'][d][operation] == 0:
tbr.append(protocol)
break
for r in tbr:
candidates.remove(r)
if not len(candidates):
raise exception.RSEProtocolNotSupported('No protocol for provided settings found : %s.' % str(rse_settings))
# Select the one with the highest priority
candidates = sorted(candidates, key=lambda k: k['scheme'])
best_choice = candidates[0]
candidates.remove(best_choice)
domain = rse_settings['domain'][0]
for p in candidates:
if p['domains'][domain][operation] < best_choice['domains'][domain][operation]:
best_choice = p
return best_choice
def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_token=None, protocol_attr=None, logger=logging.log):
"""
Instanciates the protocol defined for the given operation.
:param rse_settings: RSE attributes
:param operation: Intended operation for this protocol
:param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation
:param domain: Optional specification of the domain
:param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication
:param protocol_attr: Optionally passing the full protocol availability information to correctly select WAN/LAN
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
:returns: An instance of the requested protocol
"""
# Verify feasibility of Protocol
operation = operation.lower()
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
if domain and domain not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain)
if not protocol_attr:
protocol_attr = select_protocol(rse_settings, operation, scheme, domain)
else:
candidates = _get_possible_protocols(rse_settings, operation, scheme, domain)
if protocol_attr not in candidates:
raise exception.RSEProtocolNotSupported('Protocol %s operation %s on domain %s not supported' % (protocol_attr, operation, domain))
# Instantiate protocol
comp = protocol_attr['impl'].split('.')
prefix = '.'.join(comp[-2:]) + ': '
logger = formatted_logger(logger, prefix + "%s")
mod = __import__('.'.join(comp[:-1]))
for n in comp[1:]:
try:
mod = getattr(mod, n)
except AttributeError as e:
logger(logging.DEBUG, 'Protocol implementations not supported.')
raise exception.RucioException(str(e)) # TODO: provide proper rucio exception
protocol_attr['auth_token'] = auth_token
protocol = mod(protocol_attr, rse_settings, logger=logger)
return protocol
def add_protocol(rse, parameter, session=None):
"""
Add a protocol to an existing RSE. If entries with equal or less priority for
an operation exist, the existing one will be reorded (i.e. +1).
:param rse: the name of the new rse.
:param parameter: parameters of the new protocol entry.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEOperationNotSupported: If no scheme supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the scheme is to big or below zero.
:raises Duplicate: If scheme with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
# Insert new protocol entry
parameter['rse_id'] = rid
# Default values
parameter['port'] = parameter.get('port', 0)
parameter['hostname'] = parameter.get('hostname', 'localhost')
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in parameter.keys():
for s in parameter['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'The protocol domain \'%s\' is not defined in the schema.'
% s)
for op in parameter['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation \'%s\' not defined in schema.' % (op))
op_name = op if op == 'third_party_copy' else ''.join(
[op, '_', s]).lower()
if parameter['domains'][s][op] < 0:
raise exception.RSEProtocolPriorityError(
'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.'
% (parameter['domains'][s][op], op, s))
parameter[op_name] = parameter['domains'][s][op]
del parameter['domains']
if ('extended_attributes'
in parameter) and parameter['extended_attributes']:
try:
parameter['extended_attributes'] = json.dumps(
parameter['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if parameter['scheme'] == 'srm':
if ('extended_attributes'
not in parameter) or ('web_service_path'
not in parameter['extended_attributes']):
raise exception.InvalidObject(
'Missing values! For SRM, extended_attributes and web_service_path must be specified'
)
try:
new_protocol = models.RSEProtocols()
new_protocol.update(parameter)
new_protocol.save(session=session)
except (IntegrityError, FlushError, OperationalError) as error:
if ('UNIQUE constraint failed' in error.args[0]) or ('conflicts with persistent instance' in error.args[0]) \
or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \
or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0])\
or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]):
raise exception.Duplicate(
'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.'
% (parameter['scheme'], parameter['port'], rse,
parameter['hostname']))
elif 'may not be NULL' in error.args[0] \
or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \
or match('.*OperationalError.*cannot be null.*', error.args[0]):
raise exception.InvalidObject('Missing values!')
raise error
return new_protocol
def update_protocols(rse, scheme, data, hostname, port, session=None):
"""
Updates an existing protocol entry for an RSE. If necessary, priorities for read,
write, and delete operations of other protocol entires will be updated too.
:param rse: the name of the new rse.
:param scheme: Protocol identifer.
:param data: Dict with new values (keys must match column names in the database).
:param hostname: Hostname defined for the scheme, used if more than one scheme
is registered with the same identifier.
:param port: The port registered for the hostename, used if more than one scheme
is regsitered with the same identifier and hostname.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE.
:raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero.
:raises KeyNotFound: Invalid data for update provided.
:raises Duplicate: If protocol with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in data:
for s in data['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'The protocol domain \'%s\' is not defined in the schema.'
% s)
for op in data['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation \'%s\' not defined in schema.' % (op))
op_name = op
if op != 'third_party_copy':
op_name = ''.join([op, '_', s])
no = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= 0)).\
count()
if not 0 <= data['domains'][s][op] <= no:
raise exception.RSEProtocolPriorityError(
'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.'
% (data['domains'][s][op], op, s))
data[op_name] = data['domains'][s][op]
del data['domains']
if 'extended_attributes' in data:
try:
data['extended_attributes'] = json.dumps(
data['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
terms = [
models.RSEProtocols.rse_id == rid,
models.RSEProtocols.scheme == scheme,
models.RSEProtocols.hostname == hostname,
models.RSEProtocols.port == port
]
try:
up = session.query(models.RSEProtocols).filter(*terms).first()
if up is None:
msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % (
rse, scheme, hostname, port)
raise exception.RSEProtocolNotSupported(msg)
# Preparing gaps if priority is updated
for domain in utils.rse_supported_protocol_domains():
for op in utils.rse_supported_protocol_operations():
op_name = op
if op != 'third_party_copy':
op_name = ''.join([op, '_', domain])
if op_name in data:
prots = []
if (not getattr(up, op_name)) and data[
op_name]: # reactivate protocol e.g. from 0 to 1
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name])).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(up, op_name) and (
not data[op_name]
): # deactivate protocol e.g. from 1 to 0
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
elif getattr(
up, op_name
) > data[op_name]: # shift forward e.g. from 5 to 2
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name],
getattr(models.RSEProtocols, op_name) < getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(
up, op_name
) < data[op_name]: # shift backward e.g. from 1 to 3
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) <= data[op_name],
getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
for p in prots:
p.update({op_name: val})
val += 1
up.update(data, flush=True, session=session)
except (IntegrityError, OperationalError) as error:
if 'UNIQUE'.lower() in error.args[0].lower(
) or 'Duplicate' in error.args[
0]: # Covers SQLite, Oracle and MySQL error
raise exception.Duplicate(
'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.'
% (scheme, port, rse, hostname))
elif 'may not be NULL' in error.args[
0] or "cannot be null" in error.args[0]:
raise exception.InvalidObject('Missing values: %s' % error.args[0])
raise error
except DatabaseError as error:
if match(
'.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*',
error.args[0]):
raise exception.InvalidObject('Invalid values !')
raise error
def find_matching_scheme(rse_settings_dest,
rse_settings_src,
operation_src,
operation_dest,
domain=None,
scheme=None):
"""
Find the best matching scheme between two RSEs
:param rse_settings_dest: RSE settings for the destination RSE.
:param rse_settings_src: RSE settings for the src RSE.
:param operation_src: Source Operation such as read, write.
:param operation_dest: Dest Operation such as read, write.
:param domain: Domain such as lan, wan.
:param scheme: List of supported schemes.
:returns: Tuple of matching schemes (dest_scheme, src_scheme).
"""
operation_src = operation_src.lower()
operation_dest = operation_dest.lower()
src_candidates = copy.copy(rse_settings_src['protocols'])
dest_candidates = copy.copy(rse_settings_dest['protocols'])
if not domain:
if type(rse_settings_dest['domain']) is not list:
raise exception.RSEProtocolDomainNotSupported(
'Domain setting must be list.')
domain = rse_settings_dest['domain'][0]
# Clean up src_candidates
tbr = list()
for protocol in src_candidates:
# Check if scheme given and filter if so
if scheme:
if not isinstance(scheme, list):
scheme = scheme.split(',')
if protocol['scheme'] not in scheme:
tbr.append(protocol)
continue
if protocol['domains'].get(domain, {}).get(operation_src, 1) == 0:
tbr.append(protocol)
for r in tbr:
src_candidates.remove(r)
# Clean up dest_candidates
tbr = list()
for protocol in dest_candidates:
# Check if scheme given and filter if so
if scheme:
if not isinstance(scheme, list):
scheme = scheme.split(',')
if protocol['scheme'] not in scheme:
tbr.append(protocol)
continue
if protocol['domains'].get(domain, {}).get(operation_dest, 1) == 0:
tbr.append(protocol)
for r in tbr:
dest_candidates.remove(r)
if not len(src_candidates) or not len(dest_candidates):
raise exception.RSEProtocolNotSupported(
'No protocol for provided settings found : %s.' %
str(rse_settings_dest))
# Select the one with the highest priority
dest_candidates = sorted(
dest_candidates, key=lambda k: k['domains'][domain][operation_dest])
src_candidates = sorted(src_candidates,
key=lambda k: k['domains'][domain][operation_src])
for dest_protocol in dest_candidates:
for src_protocol in src_candidates:
if __check_compatible_scheme(dest_protocol['scheme'],
src_protocol['scheme']):
return (dest_protocol['scheme'], src_protocol['scheme'])
raise exception.RSEProtocolNotSupported(
'No protocol for provided settings found : %s.' %
str(rse_settings_dest))