def select_protocol(rse_settings, operation, scheme=None):
operation = operation.lower()
candidates = copy.copy(rse_settings['protocols'])
if type(rse_settings['domain']) is not list:
raise exception.RSEProtocolDomainNotSupported('Domain setting must be list.')
for d in rse_settings['domain']:
if d not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported('Domain %s is not supported by Rucio.' % d)
tbr = list()
for protocol in candidates:
# Check if scheme given and filter if so
if scheme and protocol['scheme'] != scheme:
tbr.append(protocol)
continue
# Check if operation in domain is supported
for d in rse_settings['domain']:
if protocol['domains'][d][operation] == 0:
tbr.append(protocol)
break
for r in tbr:
candidates.remove(r)
if not len(candidates):
raise exception.RSEProtocolNotSupported('No protocol for provided settings found : %s.' % str(rse_settings))
# Select the one with the highest priority
best_choice = candidates[0]
candidates.remove(best_choice)
domain = rse_settings['domain'][0]
for p in candidates:
if p['domains'][domain][operation] < best_choice['domains'][domain][operation]:
best_choice = p
return best_choice
def del_protocols(rse, scheme, hostname=None, port=None, session=None):
"""
Deletes an existing protocol entry for an RSE.
:param rse: the name of the new rse.
:param scheme: Protocol identifer.
:param hostname: Hostname defined for the scheme, used if more than one scheme
is registered with the same identifier.
:param port: The port registered for the hostename, used if more than one scheme
is regsitered with the same identifier and hostname.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEProtocolNotSupported: If no macthing scheme was found for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
terms = [models.RSEProtocols.rse_id == rid, models.RSEProtocols.scheme == scheme]
if hostname:
terms.append(models.RSEProtocols.hostname == hostname)
if port:
terms.append(models.RSEProtocols.port == port)
p = session.query(models.RSEProtocols).filter(*terms)
if not p.all():
msg = 'RSE \'%s\' does not support protocol \'%s\'' % (rse, scheme)
msg += ' for hostname \'%s\'' % hostname if hostname else ''
msg += ' on port \'%s\'' % port if port else ''
raise exception.RSEProtocolNotSupported(msg)
for row in p:
row.delete(session=session)
# Filling gaps in protocol priorities
for domain in utils.rse_supported_protocol_domains():
for op in utils.rse_supported_protocol_operations():
op_name = ''.join([op, '_', domain])
prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) > 0
)).order_by(getattr(models.RSEProtocols, op_name).asc())
i = 1
for p in prots:
p.update({op_name: i})
i += 1
def create_protocol(rse_settings, operation, scheme=None):
""" Instanciates the protocol defined for the given operation.
:param rse_attr: RSE attributes
:param operation: the intended operation for this protocol
:param scheme: optional filter if no specific protocol is defined in rse_setting for the provided operation
:returns: an instance of the requested protocol
"""
# Verify feasibility of Protocol
operation = operation.lower()
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation %s is not supported' % operation)
rse_settings['domain'] = [rse_settings['domain']] if type(
rse_settings['domain']) is not list else rse_settings['domain']
for domain in rse_settings['domain']:
if domain.lower() not in utils.rse_supported_protocol_domains():
raise exception.RSEOperationNotSupported(
'Domain %s not supported' % rse_settings['domain'])
if rse_settings['%s_protocol' % operation] == DEFAULT_PROTOCOL:
protocol_attr = select_protocol(rse_settings, operation, scheme)
else:
protocol_attr = rse_settings['%s_protocol' % operation]
for d in rse_settings['domain']:
if protocol_attr['domains'][d][operation] == 0:
raise exception.RSEOperationNotSupported(
'Operation %s for domain %s not supported by %s' %
(operation, rse_settings['domain'],
protocol_attr['scheme']))
# Instanciate protocol
comp = protocol_attr['impl'].split('.')
mod = __import__('.'.join(comp[:-1]))
for n in comp[1:]:
try:
mod = getattr(mod, n)
except AttributeError:
print 'Protocol implementation not found'
raise # TODO: provide proper rucio exception
protocol = mod(protocol_attr, rse_settings)
return protocol
def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_token=None, protocol_attr=None, logger=logging.log):
"""
Instanciates the protocol defined for the given operation.
:param rse_settings: RSE attributes
:param operation: Intended operation for this protocol
:param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation
:param domain: Optional specification of the domain
:param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication
:param protocol_attr: Optionally passing the full protocol availability information to correctly select WAN/LAN
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
:returns: An instance of the requested protocol
"""
# Verify feasibility of Protocol
operation = operation.lower()
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
if domain and domain not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain)
if not protocol_attr:
protocol_attr = select_protocol(rse_settings, operation, scheme, domain)
else:
candidates = _get_possible_protocols(rse_settings, operation, scheme, domain)
if protocol_attr not in candidates:
raise exception.RSEProtocolNotSupported('Protocol %s operation %s on domain %s not supported' % (protocol_attr, operation, domain))
# Instantiate protocol
comp = protocol_attr['impl'].split('.')
prefix = '.'.join(comp[-2:]) + ': '
logger = formatted_logger(logger, prefix + "%s")
mod = __import__('.'.join(comp[:-1]))
for n in comp[1:]:
try:
mod = getattr(mod, n)
except AttributeError as e:
logger(logging.DEBUG, 'Protocol implementations not supported.')
raise exception.RucioException(str(e)) # TODO: provide proper rucio exception
protocol_attr['auth_token'] = auth_token
protocol = mod(protocol_attr, rse_settings, logger=logger)
return protocol
def create_protocol(rse_settings,
operation,
scheme=None,
domain='wan',
auth_token=None,
logger=_logger):
"""
Instanciates the protocol defined for the given operation.
:param rse_settings: RSE attributes
:param operation: Intended operation for this protocol
:param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation
:param domain: Optional specification of the domain
:param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication
:returns: An instance of the requested protocol
"""
# Verify feasibility of Protocol
operation = operation.lower()
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation %s is not supported' % operation)
if domain and domain not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'Domain %s not supported' % domain)
protocol_attr = select_protocol(rse_settings, operation, scheme, domain)
# Instantiate protocol
comp = protocol_attr['impl'].split('.')
mod = __import__('.'.join(comp[:-1]))
for n in comp[1:]:
try:
mod = getattr(mod, n)
except AttributeError as e:
logger.debug('Protocol implementations not supported.')
raise exception.RucioException(
str(e)) # TODO: provide proper rucio exception
protocol_attr['auth_token'] = auth_token
protocol = mod(protocol_attr, rse_settings, logger=logger)
return protocol
def select_protocol(rse_settings, operation, scheme=None):
operation = operation.lower()
candidates = copy.copy(rse_settings['protocols'])
if type(rse_settings['domain']) is not list:
raise exception.RSEProtocolDomainNotSupported('Domain setting must be list.')
for d in rse_settings['domain']:
if d not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported('Domain %s is not supported by Rucio.' % d)
tbr = list()
for protocol in candidates:
# Check if scheme given and filter if so
if scheme:
if not isinstance(scheme, list):
scheme = scheme.split(',')
if protocol['scheme'] not in scheme:
tbr.append(protocol)
continue
for d in rse_settings['domain']:
if protocol['domains'][d][operation] == 0:
tbr.append(protocol)
break
for r in tbr:
candidates.remove(r)
if not len(candidates):
raise exception.RSEProtocolNotSupported('No protocol for provided settings found : %s.' % str(rse_settings))
# Select the one with the highest priority
candidates = sorted(candidates, key=lambda k: k['scheme'])
best_choice = candidates[0]
candidates.remove(best_choice)
domain = rse_settings['domain'][0]
for p in candidates:
if p['domains'][domain][operation] < best_choice['domains'][domain][operation]:
best_choice = p
return best_choice
def create_protocol(rse_settings, operation, scheme=None):
""" Instanciates the protocol defined for the given operation.
:param rse_attr: RSE attributes
:param operation: the intended operation for this protocol
:param scheme: optional filter if no specific protocol is defined in rse_setting for the provided operation
:returns: an instance of the requested protocol
"""
# Verify feasibility of Protocol
operation = operation.lower()
if operation not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
rse_settings['domain'] = [rse_settings['domain']] if type(rse_settings['domain']) is not list else rse_settings['domain']
for domain in rse_settings['domain']:
if domain.lower() not in utils.rse_supported_protocol_domains():
raise exception.RSEOperationNotSupported('Domain %s not supported' % rse_settings['domain'])
if rse_settings['%s_protocol' % operation] == DEFAULT_PROTOCOL:
protocol_attr = select_protocol(rse_settings, operation, scheme)
else:
protocol_attr = rse_settings['%s_protocol' % operation]
for d in rse_settings['domain']:
if protocol_attr['domains'][d][operation] == 0:
raise exception.RSEOperationNotSupported('Operation %s for domain %s not supported by %s' % (operation, rse_settings['domain'], protocol_attr['scheme']))
# Instanciate protocol
comp = protocol_attr['impl'].split('.')
mod = __import__('.'.join(comp[:-1]))
for n in comp[1:]:
try:
mod = getattr(mod, n)
except AttributeError:
print 'Protocol implementation not found'
raise # TODO: provide proper rucio exception
protocol = mod(protocol_attr, rse_settings)
return protocol
def get_rse_protocols(rse, schemes=None, session=None):
"""
Returns protocol information. Parameter combinations are: (operation OR default) XOR scheme.
:param rse: The name of the rse.
:param schemes: a list of schemes to filter by.
:param session: The database session.
:returns: A dict with RSE information and supported protocols
:raises RSENotFound: If RSE is not found.
"""
_rse = get_rse(rse=rse, session=session)
if not _rse:
raise exception.RSENotFound('RSE \'%s\' not found')
lfn2pfn_algorithms = get_rse_attribute('lfn2pfn_algorithm',
rse_id=_rse.id,
session=session)
# Resolve LFN2PFN default algorithm as soon as possible. This way, we can send back the actual
# algorithm name in response to REST queries.
lfn2pfn_algorithm = get_lfn2pfn_algorithm_default()
if lfn2pfn_algorithms:
lfn2pfn_algorithm = lfn2pfn_algorithms[0]
# Copy verify_checksum from the attributes, later: assume True if not specified
verify_checksum = get_rse_attribute('verify_checksum',
rse_id=_rse.id,
session=session)
read = True if _rse.availability & 4 else False
write = True if _rse.availability & 2 else False
delete = True if _rse.availability & 1 else False
info = {
'id': _rse.id,
'rse': _rse.rse,
'availability_read': read,
'availability_write': write,
'availability_delete': delete,
'domain': utils.rse_supported_protocol_domains(),
'protocols': list(),
'deterministic': _rse.deterministic,
'lfn2pfn_algorithm': lfn2pfn_algorithm,
'rse_type': str(_rse.rse_type),
'credentials': None,
'volatile': _rse.volatile,
'verify_checksum': verify_checksum[0] if verify_checksum else True,
'staging_area': _rse.staging_area
}
for op in utils.rse_supported_protocol_operations():
info['%s_protocol' % op] = 1 # 1 indicates the default protocol
query = None
terms = [models.RSEProtocols.rse_id == _rse.id]
if schemes:
if not type(schemes) is list:
schemes = [schemes]
terms.extend([models.RSEProtocols.scheme.in_(schemes)])
query = session.query(
models.RSEProtocols.hostname, models.RSEProtocols.scheme,
models.RSEProtocols.port, models.RSEProtocols.prefix,
models.RSEProtocols.impl, models.RSEProtocols.read_lan,
models.RSEProtocols.write_lan, models.RSEProtocols.delete_lan,
models.RSEProtocols.read_wan, models.RSEProtocols.write_wan,
models.RSEProtocols.delete_wan, models.RSEProtocols.third_party_copy,
models.RSEProtocols.extended_attributes).filter(*terms)
for row in query:
p = {
'hostname': row.hostname,
'scheme': row.scheme,
'port': row.port,
'prefix': row.prefix if row.prefix is not None else '',
'impl': row.impl,
'domains': {
'lan': {
'read': row.read_lan,
'write': row.write_lan,
'delete': row.delete_lan
},
'wan': {
'read': row.read_wan,
'write': row.write_wan,
'delete': row.delete_wan,
'third_party_copy': row.third_party_copy
}
},
'extended_attributes': row.extended_attributes
}
try:
p['extended_attributes'] = json.load(
StringIO(p['extended_attributes']))
except ValueError:
pass # If value is not a JSON string
info['protocols'].append(p)
return info
def add_protocol(rse, parameter, session=None):
"""
Add a protocol to an existing RSE. If entries with equal or less priority for
an operation exist, the existing one will be reorded (i.e. +1).
:param rse: the name of the new rse.
:param parameter: parameters of the new protocol entry.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEOperationNotSupported: If no scheme supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the scheme is to big or below zero.
:raises Duplicate: If scheme with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
# Insert new protocol entry
parameter['rse_id'] = rid
# Default values
parameter['port'] = parameter.get('port', 0)
parameter['hostname'] = parameter.get('hostname', 'localhost')
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in parameter.keys():
for s in parameter['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'The protocol domain \'%s\' is not defined in the schema.'
% s)
for op in parameter['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation \'%s\' not defined in schema.' % (op))
op_name = op if op == 'third_party_copy' else ''.join(
[op, '_', s]).lower()
if parameter['domains'][s][op] < 0:
raise exception.RSEProtocolPriorityError(
'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.'
% (parameter['domains'][s][op], op, s))
parameter[op_name] = parameter['domains'][s][op]
del parameter['domains']
if ('extended_attributes'
in parameter) and parameter['extended_attributes']:
try:
parameter['extended_attributes'] = json.dumps(
parameter['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if parameter['scheme'] == 'srm':
if ('extended_attributes'
not in parameter) or ('web_service_path'
not in parameter['extended_attributes']):
raise exception.InvalidObject(
'Missing values! For SRM, extended_attributes and web_service_path must be specified'
)
try:
new_protocol = models.RSEProtocols()
new_protocol.update(parameter)
new_protocol.save(session=session)
except (IntegrityError, FlushError, OperationalError) as error:
if ('UNIQUE constraint failed' in error.args[0]) or ('conflicts with persistent instance' in error.args[0]) \
or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \
or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0])\
or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]):
raise exception.Duplicate(
'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.'
% (parameter['scheme'], parameter['port'], rse,
parameter['hostname']))
elif 'may not be NULL' in error.args[0] \
or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \
or match('.*OperationalError.*cannot be null.*', error.args[0]):
raise exception.InvalidObject('Missing values!')
raise error
return new_protocol
def update_protocols(rse, scheme, data, hostname, port, session=None):
"""
Updates an existing protocol entry for an RSE. If necessary, priorities for read,
write, and delete operations of other protocol entires will be updated too.
:param rse: the name of the new rse.
:param scheme: Protocol identifer.
:param data: Dict with new values (keys must match column names in the database).
:param hostname: Hostname defined for the scheme, used if more than one scheme
is registered with the same identifier.
:param port: The port registered for the hostename, used if more than one scheme
is regsitered with the same identifier and hostname.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE.
:raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero.
:raises KeyNotFound: Invalid data for update provided.
:raises Duplicate: If protocol with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in data:
for s in data['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'The protocol domain \'%s\' is not defined in the schema.'
% s)
for op in data['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation \'%s\' not defined in schema.' % (op))
op_name = op
if op != 'third_party_copy':
op_name = ''.join([op, '_', s])
no = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= 0)).\
count()
if not 0 <= data['domains'][s][op] <= no:
raise exception.RSEProtocolPriorityError(
'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.'
% (data['domains'][s][op], op, s))
data[op_name] = data['domains'][s][op]
del data['domains']
if 'extended_attributes' in data:
try:
data['extended_attributes'] = json.dumps(
data['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
terms = [
models.RSEProtocols.rse_id == rid,
models.RSEProtocols.scheme == scheme,
models.RSEProtocols.hostname == hostname,
models.RSEProtocols.port == port
]
try:
up = session.query(models.RSEProtocols).filter(*terms).first()
if up is None:
msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % (
rse, scheme, hostname, port)
raise exception.RSEProtocolNotSupported(msg)
# Preparing gaps if priority is updated
for domain in utils.rse_supported_protocol_domains():
for op in utils.rse_supported_protocol_operations():
op_name = op
if op != 'third_party_copy':
op_name = ''.join([op, '_', domain])
if op_name in data:
prots = []
if (not getattr(up, op_name)) and data[
op_name]: # reactivate protocol e.g. from 0 to 1
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name])).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(up, op_name) and (
not data[op_name]
): # deactivate protocol e.g. from 1 to 0
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
elif getattr(
up, op_name
) > data[op_name]: # shift forward e.g. from 5 to 2
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name],
getattr(models.RSEProtocols, op_name) < getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(
up, op_name
) < data[op_name]: # shift backward e.g. from 1 to 3
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) <= data[op_name],
getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
for p in prots:
p.update({op_name: val})
val += 1
up.update(data, flush=True, session=session)
except (IntegrityError, OperationalError) as error:
if 'UNIQUE'.lower() in error.args[0].lower(
) or 'Duplicate' in error.args[
0]: # Covers SQLite, Oracle and MySQL error
raise exception.Duplicate(
'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.'
% (scheme, port, rse, hostname))
elif 'may not be NULL' in error.args[
0] or "cannot be null" in error.args[0]:
raise exception.InvalidObject('Missing values: %s' % error.args[0])
raise error
except DatabaseError as error:
if match(
'.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*',
error.args[0]):
raise exception.InvalidObject('Invalid values !')
raise error
def update_protocols(rse, scheme, data, hostname, port, session=None):
"""
Updates an existing protocol entry for an RSE. If necessary, priorities for read,
write, and delete operations of other protocol entires will be updated too.
:param rse: the name of the new rse.
:param scheme: Protocol identifer.
:param data: Dict with new values (keys must match column names in the database).
:param hostname: Hostname defined for the scheme, used if more than one scheme
is registered with the same identifier.
:param port: The port registered for the hostename, used if more than one scheme
is regsitered with the same identifier and hostname.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE.
:raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero.
:raises KeyNotFound: Invalid data for update provided.
:raises Duplicate: If protocol with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in data:
for s in data['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported('The protocol domain \'%s\' is not defined in the schema.' % s)
for op in data['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported('Operation \'%s\' not defined in schema.' % (op))
op_name = ''.join([op, '_', s])
no = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) > 0,
)).count()
if not(0 <= data['domains'][s][op] <= no):
raise exception.RSEProtocolPriorityError('The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (data['domains'][s][op], op, s))
data[op_name] = data['domains'][s][op]
del(data['domains'])
if 'extended_attributes' in data:
try:
data['extended_attributes'] = json.dumps(data['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
terms = [models.RSEProtocols.rse_id == rid,
models.RSEProtocols.scheme == scheme,
models.RSEProtocols.hostname == hostname,
models.RSEProtocols.port == port
]
try:
up = session.query(models.RSEProtocols).filter(*terms).first()
if up is None:
msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % (rse, scheme, hostname, port)
raise exception.RSEProtocolNotSupported(msg)
# Preparing gaps if priority is updated
for domain in utils.rse_supported_protocol_domains():
for op in utils.rse_supported_protocol_operations():
op_name = ''.join([op, '_', domain])
if op_name in data:
if (not getattr(up, op_name)) and data[op_name]: # reactivate protocol e.g. from 0 to 1
prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name]
)).order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(up, op_name) and (not data[op_name]): # deactivate protocol e.g. from 1 to 0
prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) > getattr(up, op_name)
)).order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
elif getattr(up, op_name) > data[op_name]: # shift forward e.g. from 5 to 2
prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name],
getattr(models.RSEProtocols, op_name) < getattr(up, op_name)
)).order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(up, op_name) < data[op_name]: # shift backward e.g. from 1 to 3
prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) <= data[op_name],
getattr(models.RSEProtocols, op_name) > getattr(up, op_name)
)).order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
for p in prots:
p.update({op_name: val})
val += 1
up.update(data, flush=True, session=session)
except (IntegrityError, OperationalError) as e:
if 'UNIQUE'.lower() in e.args[0].lower() or 'Duplicate' in e.args[0]: # Covers SQLite, Oracle and MySQL error
raise exception.Duplicate('Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (scheme, port, rse, hostname))
elif 'may not be NULL' in e.args[0] or "cannot be null" in e.args[0]:
raise exception.InvalidObject('Missing values: %s' % e.args[0])
raise e
except DatabaseError, e:
if match('.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*', e.args[0]):
raise exception.InvalidObject('Invalid values !')
raise e
def get_rse_protocols(rse, schemes=None, session=None):
"""
Returns protocol information. Parameter combinations are: (operation OR default) XOR scheme.
:param rse: The name of the rse.
:param schemes: a list of schemes to filter by.
:param session: The database session.
:returns: A dict with RSE information and supported protocols
:raises RSENotFound: If RSE is not found.
"""
_rse = get_rse(rse=rse, session=session)
if not _rse:
raise exception.RSENotFound('RSE \'%s\' not found')
read = True if _rse.availability & 4 else False
write = True if _rse.availability & 2 else False
delete = True if _rse.availability & 1 else False
info = {'id': _rse.id,
'rse': _rse.rse,
'availability_read': read,
'availability_write': write,
'availability_delete': delete,
'domain': utils.rse_supported_protocol_domains(),
'protocols': list(),
'deterministic': _rse.deterministic,
'rse_type': str(_rse.rse_type),
'credentials': None,
'volatile': _rse.volatile,
'staging_area': _rse.staging_area
}
for op in utils.rse_supported_protocol_operations():
info['%s_protocol' % op] = 1 # 1 indicates the default protocol
query = None
terms = [models.RSEProtocols.rse_id == _rse.id]
query = session.query(models.RSEProtocols.hostname,
models.RSEProtocols.scheme,
models.RSEProtocols.port,
models.RSEProtocols.prefix,
models.RSEProtocols.impl,
models.RSEProtocols.read_lan,
models.RSEProtocols.write_lan,
models.RSEProtocols.delete_lan,
models.RSEProtocols.read_wan,
models.RSEProtocols.write_wan,
models.RSEProtocols.delete_wan,
models.RSEProtocols.extended_attributes).filter(*terms)
for row in query:
p = {'hostname': row.hostname,
'scheme': row.scheme,
'port': row.port,
'prefix': row.prefix if row.prefix is not None else '',
'impl': row.impl,
'domains': {
'lan': {'read': row.read_lan,
'write': row.write_lan,
'delete': row.delete_lan
},
'wan': {'read': row.read_wan,
'write': row.write_wan,
'delete': row.delete_wan
}
},
'extended_attributes': row.extended_attributes
}
try:
p['extended_attributes'] = json.load(StringIO(p['extended_attributes']))
except ValueError:
pass # If value is not a JSON string
info['protocols'].append(p)
return info
def add_protocol(rse, parameter, session=None):
"""
Add a protocol to an existing RSE. If entries with equal or less priority for
an operation exist, the existing one will be reorded (i.e. +1).
:param rse: the name of the new rse.
:param parameter: parameters of the new protocol entry.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEOperationNotSupported: If no scheme supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the scheme is to big or below zero.
:raises Duplicate: If scheme with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
# Insert new protocol entry
parameter['rse_id'] = rid
# Default values
parameter['port'] = parameter.get('port', 0)
parameter['hostname'] = parameter.get('hostname', 'localhost')
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in parameter.keys():
for s in parameter['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported('The protocol domain \'%s\' is not defined in the schema.' % s)
for op in parameter['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported('Operation \'%s\' not defined in schema.' % (op))
op_name = ''.join([op, '_', s]).lower()
no = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) > 0,
)).count()
if parameter['domains'][s][op] > (no + 1):
parameter['domains'][s][op] = (no + 1)
if not(0 <= parameter['domains'][s][op] <= (no + 1)):
raise exception.RSEProtocolPriorityError('The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (parameter['domains'][s][op], op, s))
parameter[op_name] = parameter['domains'][s][op]
del(parameter['domains'])
if ('extended_attributes' in parameter) and parameter['extended_attributes']:
try:
parameter['extended_attributes'] = json.dumps(parameter['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if parameter['scheme'] == 'srm':
if ('space_token' not in parameter['extended_attributes']) or ('web_service_path' not in parameter['extended_attributes']):
raise exception.InvalidObject('Missing values! For SRM, extended_attributes and web_service_path must be specified')
try:
# Open gaps in protocols priorities for new protocol
for domain in utils.rse_supported_protocol_domains():
for op in utils.rse_supported_protocol_operations():
op_name = ''.join([op, '_', domain])
if (op_name in parameter) and parameter[op_name]:
prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= parameter[op_name],
)).order_by(getattr(models.RSEProtocols, op_name).asc())
val = parameter[op_name] + 1
for p in prots:
p.update({op_name: val})
val += 1
new_protocol = models.RSEProtocols()
new_protocol.update(parameter)
new_protocol.save(session=session)
except (IntegrityError, FlushError, OperationalError) as e:
if ('not unique' in e.args[0]) or ('conflicts with persistent instance' in e.args[0]) \
or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', e.args[0]):
raise exception.Duplicate('Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (parameter['scheme'], parameter['port'], rse, parameter['hostname']))
elif 'may not be NULL' in e.args[0] \
or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', e.args[0]) \
or match('.*OperationalError.*cannot be null.*', e.args[0]):
raise exception.InvalidObject('Missing values!')
raise e
return new_protocol
