def get_rse_name(rse_id, session=None, include_deleted=True):
"""
Get a RSE name or raise if it does not exist.
:param rse_id: the rse uuid from the database.
:param session: The database session in use.
:param include_deleted: Flag to toggle finding rse's marked as deleted.
:returns: The rse name.
:raises RSENotFound: If referred RSE was not found in the database.
"""
if include_deleted:
cache_key = 'rse-name_{}'.format(rse_id)
result = REGION.get(cache_key)
if result != NO_VALUE:
return result
try:
query = session.query(models.RSE.rse).filter_by(id=rse_id)
if not include_deleted:
query = query.filter_by(deleted=False)
result = query.one()[0]
except sqlalchemy.orm.exc.NoResultFound:
raise exception.RSENotFound('RSE with ID \'%s\' cannot be found' % rse_id)
if include_deleted:
REGION.set(cache_key, result)
return result
def get_rse(rse, rse_id=None, session=None):
"""
Get a RSE or raise if it does not exist.
:param rse: The rse name.
:param rse_id: The rse id. To be used if the rse parameter is none.
:param session: The database session in use.
:raises RSENotFound: If referred RSE was not found in the database.
"""
false_value = False # To make pep8 checker happy ...
try:
if rse:
tmp = session.query(models.RSE).\
filter(sqlalchemy.and_(models.RSE.deleted == false_value,
models.RSE.rse == rse))\
.one()
else:
tmp = session.query(models.RSE).\
filter(sqlalchemy.and_(models.RSE.deleted == false_value,
models.RSE.id == rse_id))\
.one()
tmp['type'] = tmp.rse_type
return tmp
except sqlalchemy.orm.exc.NoResultFound:
raise exception.RSENotFound('RSE \'%s\' cannot be found' % rse)
def del_rse(rse_id, session=None):
"""
Disable a rse with the given rse id.
:param rse_id: the rse id.
:param session: The database session in use.
"""
old_rse = None
try:
old_rse = session.query(models.RSE).filter_by(id=rse_id,
deleted=False).one()
if not rse_is_empty(rse_id=rse_id, session=session):
raise exception.RSEOperationNotSupported(
'RSE \'%s\' is not empty' %
get_rse_name(rse_id=rse_id, session=session))
except sqlalchemy.orm.exc.NoResultFound:
raise exception.RSENotFound('RSE with id \'%s\' cannot be found' %
rse_id)
rse = old_rse.rse
old_rse.delete(session=session)
try:
del_rse_attribute(rse_id=rse_id, key=rse, session=session)
except exception.RSEAttributeNotFound:
pass
def del_protocols(rse_id, scheme, hostname=None, port=None, session=None):
"""
Deletes an existing protocol entry for an RSE.
:param rse_id: the id of the new rse.
:param scheme: Protocol identifer.
:param hostname: Hostname defined for the scheme, used if more than one scheme
is registered with the same identifier.
:param port: The port registered for the hostename, used if more than one scheme
is regsitered with the same identifier and hostname.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEProtocolNotSupported: If no macthing scheme was found for the given RSE.
"""
try:
rse_name = get_rse_name(rse_id=rse_id,
session=session,
include_deleted=False)
except exception.RSENotFound:
raise exception.RSENotFound('RSE \'%s\' not found' % rse_id)
terms = [
models.RSEProtocols.rse_id == rse_id,
models.RSEProtocols.scheme == scheme
]
if hostname:
terms.append(models.RSEProtocols.hostname == hostname)
if port:
terms.append(models.RSEProtocols.port == port)
p = session.query(models.RSEProtocols).filter(*terms)
if not p.all():
msg = 'RSE \'%s\' does not support protocol \'%s\'' % (rse_name,
scheme)
msg += ' for hostname \'%s\'' % hostname if hostname else ''
msg += ' on port \'%s\'' % port if port else ''
raise exception.RSEProtocolNotSupported(msg)
for row in p:
row.delete(session=session)
# Filling gaps in protocol priorities
for domain in utils.rse_supported_protocol_domains():
for op in utils.rse_supported_protocol_operations():
op_name = ''.join([op, '_', domain])
if getattr(models.RSEProtocols, op_name, None):
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rse_id,
getattr(models.RSEProtocols, op_name) > 0)).\
order_by(getattr(models.RSEProtocols, op_name).asc())
i = 1
for p in prots:
p.update({op_name: i})
i += 1
def update_rse(rse_id, parameters, session=None):
"""
Update RSE properties like availability or name.
:param rse_id: the id of the new rse.
:param parameters: A dictionnary with property (name, read, write, delete as keys).
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
"""
try:
query = session.query(models.RSE).filter_by(id=rse_id).one()
except sqlalchemy.orm.exc.NoResultFound:
raise exception.RSENotFound('RSE \'%s\' cannot be found' % rse_id)
availability = 0
rse = query.rse
for column in query:
if column[0] == 'availability':
availability = column[1] or availability
param = {}
availability_mapping = {
'availability_read': 4,
'availability_write': 2,
'availability_delete': 1
}
for key in parameters:
if key == 'name':
param['rse'] = parameters['name']
elif key in [
'availability_read', 'availability_write',
'availability_delete'
]:
if parameters[key] is True:
availability = availability | availability_mapping[key]
else:
availability = availability & ~availability_mapping[key]
elif key in [
'latitude', 'longitude', 'time_zone', 'rse_type', 'volatile',
'deterministic', 'region_code', 'country_name', 'city',
'staging_area'
]:
param[key] = parameters[key]
param['availability'] = availability
query.update(param)
if 'name' in parameters:
add_rse_attribute(rse_id=rse_id,
key=parameters['name'],
value=1,
session=session)
query = session.query(models.RSEAttrAssociation).filter_by(
rse_id=rse_id).filter(models.RSEAttrAssociation.key == rse)
rse_attr = query.one()
rse_attr.delete(session=session)
def del_rse(rse, session=None):
"""
Disable a rse with the given rse name.
:param rse: the rse name.
:param session: The database session in use.
"""
try:
old_rse = session.query(models.RSE).filter_by(rse=rse).one()
except sqlalchemy.orm.exc.NoResultFound:
raise exception.RSENotFound('RSE \'%s\' cannot be found' % rse)
old_rse.delete(session=session)
del_rse_attribute(rse=rse, key=rse, session=session)
def get_rse_id(rse, session=None):
"""
Get a RSE ID or raise if it does not exist.
:param rse: the rse name.
:param session: The database session in use.
:returns: The rse id.
:raises RSENotFound: If referred RSE was not found in the database.
"""
try:
return session.query(models.RSE.id).filter_by(rse=rse).one()[0]
except sqlalchemy.orm.exc.NoResultFound:
raise exception.RSENotFound('RSE \'%s\' cannot be found' % rse)
def get_rse_protocols(rse, schemes=None, session=None):
"""
Returns protocol information. Parameter combinations are: (operation OR default) XOR scheme.
:param rse: The name of the rse.
:param schemes: a list of schemes to filter by.
:param session: The database session.
:returns: A dict with RSE information and supported protocols
:raises RSENotFound: If RSE is not found.
"""
_rse = get_rse(rse=rse, session=session)
if not _rse:
raise exception.RSENotFound('RSE \'%s\' not found')
lfn2pfn_algorithms = get_rse_attribute('lfn2pfn_algorithm',
rse_id=_rse.id,
session=session)
# Resolve LFN2PFN default algorithm as soon as possible. This way, we can send back the actual
# algorithm name in response to REST queries.
lfn2pfn_algorithm = get_lfn2pfn_algorithm_default()
if lfn2pfn_algorithms:
lfn2pfn_algorithm = lfn2pfn_algorithms[0]
# Copy verify_checksum from the attributes, later: assume True if not specified
verify_checksum = get_rse_attribute('verify_checksum',
rse_id=_rse.id,
session=session)
read = True if _rse.availability & 4 else False
write = True if _rse.availability & 2 else False
delete = True if _rse.availability & 1 else False
info = {
'id': _rse.id,
'rse': _rse.rse,
'availability_read': read,
'availability_write': write,
'availability_delete': delete,
'domain': utils.rse_supported_protocol_domains(),
'protocols': list(),
'deterministic': _rse.deterministic,
'lfn2pfn_algorithm': lfn2pfn_algorithm,
'rse_type': str(_rse.rse_type),
'credentials': None,
'volatile': _rse.volatile,
'verify_checksum': verify_checksum[0] if verify_checksum else True,
'staging_area': _rse.staging_area
}
for op in utils.rse_supported_protocol_operations():
info['%s_protocol' % op] = 1 # 1 indicates the default protocol
query = None
terms = [models.RSEProtocols.rse_id == _rse.id]
if schemes:
if not type(schemes) is list:
schemes = [schemes]
terms.extend([models.RSEProtocols.scheme.in_(schemes)])
query = session.query(
models.RSEProtocols.hostname, models.RSEProtocols.scheme,
models.RSEProtocols.port, models.RSEProtocols.prefix,
models.RSEProtocols.impl, models.RSEProtocols.read_lan,
models.RSEProtocols.write_lan, models.RSEProtocols.delete_lan,
models.RSEProtocols.read_wan, models.RSEProtocols.write_wan,
models.RSEProtocols.delete_wan, models.RSEProtocols.third_party_copy,
models.RSEProtocols.extended_attributes).filter(*terms)
for row in query:
p = {
'hostname': row.hostname,
'scheme': row.scheme,
'port': row.port,
'prefix': row.prefix if row.prefix is not None else '',
'impl': row.impl,
'domains': {
'lan': {
'read': row.read_lan,
'write': row.write_lan,
'delete': row.delete_lan
},
'wan': {
'read': row.read_wan,
'write': row.write_wan,
'delete': row.delete_wan,
'third_party_copy': row.third_party_copy
}
},
'extended_attributes': row.extended_attributes
}
try:
p['extended_attributes'] = json.load(
StringIO(p['extended_attributes']))
except ValueError:
pass # If value is not a JSON string
info['protocols'].append(p)
return info
def add_protocol(rse, parameter, session=None):
"""
Add a protocol to an existing RSE. If entries with equal or less priority for
an operation exist, the existing one will be reorded (i.e. +1).
:param rse: the name of the new rse.
:param parameter: parameters of the new protocol entry.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEOperationNotSupported: If no scheme supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the scheme is to big or below zero.
:raises Duplicate: If scheme with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
# Insert new protocol entry
parameter['rse_id'] = rid
# Default values
parameter['port'] = parameter.get('port', 0)
parameter['hostname'] = parameter.get('hostname', 'localhost')
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in parameter.keys():
for s in parameter['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'The protocol domain \'%s\' is not defined in the schema.'
% s)
for op in parameter['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation \'%s\' not defined in schema.' % (op))
op_name = op if op == 'third_party_copy' else ''.join(
[op, '_', s]).lower()
if parameter['domains'][s][op] < 0:
raise exception.RSEProtocolPriorityError(
'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.'
% (parameter['domains'][s][op], op, s))
parameter[op_name] = parameter['domains'][s][op]
del parameter['domains']
if ('extended_attributes'
in parameter) and parameter['extended_attributes']:
try:
parameter['extended_attributes'] = json.dumps(
parameter['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if parameter['scheme'] == 'srm':
if ('extended_attributes'
not in parameter) or ('web_service_path'
not in parameter['extended_attributes']):
raise exception.InvalidObject(
'Missing values! For SRM, extended_attributes and web_service_path must be specified'
)
try:
new_protocol = models.RSEProtocols()
new_protocol.update(parameter)
new_protocol.save(session=session)
except (IntegrityError, FlushError, OperationalError) as error:
if ('UNIQUE constraint failed' in error.args[0]) or ('conflicts with persistent instance' in error.args[0]) \
or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \
or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0])\
or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]):
raise exception.Duplicate(
'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.'
% (parameter['scheme'], parameter['port'], rse,
parameter['hostname']))
elif 'may not be NULL' in error.args[0] \
or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \
or match('.*OperationalError.*cannot be null.*', error.args[0]):
raise exception.InvalidObject('Missing values!')
raise error
return new_protocol
def update_protocols(rse, scheme, data, hostname, port, session=None):
"""
Updates an existing protocol entry for an RSE. If necessary, priorities for read,
write, and delete operations of other protocol entires will be updated too.
:param rse: the name of the new rse.
:param scheme: Protocol identifer.
:param data: Dict with new values (keys must match column names in the database).
:param hostname: Hostname defined for the scheme, used if more than one scheme
is registered with the same identifier.
:param port: The port registered for the hostename, used if more than one scheme
is regsitered with the same identifier and hostname.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE.
:raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero.
:raises KeyNotFound: Invalid data for update provided.
:raises Duplicate: If protocol with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in data:
for s in data['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'The protocol domain \'%s\' is not defined in the schema.'
% s)
for op in data['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation \'%s\' not defined in schema.' % (op))
op_name = op
if op != 'third_party_copy':
op_name = ''.join([op, '_', s])
no = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= 0)).\
count()
if not 0 <= data['domains'][s][op] <= no:
raise exception.RSEProtocolPriorityError(
'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.'
% (data['domains'][s][op], op, s))
data[op_name] = data['domains'][s][op]
del data['domains']
if 'extended_attributes' in data:
try:
data['extended_attributes'] = json.dumps(
data['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
terms = [
models.RSEProtocols.rse_id == rid,
models.RSEProtocols.scheme == scheme,
models.RSEProtocols.hostname == hostname,
models.RSEProtocols.port == port
]
try:
up = session.query(models.RSEProtocols).filter(*terms).first()
if up is None:
msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % (
rse, scheme, hostname, port)
raise exception.RSEProtocolNotSupported(msg)
# Preparing gaps if priority is updated
for domain in utils.rse_supported_protocol_domains():
for op in utils.rse_supported_protocol_operations():
op_name = op
if op != 'third_party_copy':
op_name = ''.join([op, '_', domain])
if op_name in data:
prots = []
if (not getattr(up, op_name)) and data[
op_name]: # reactivate protocol e.g. from 0 to 1
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name])).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(up, op_name) and (
not data[op_name]
): # deactivate protocol e.g. from 1 to 0
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
elif getattr(
up, op_name
) > data[op_name]: # shift forward e.g. from 5 to 2
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name],
getattr(models.RSEProtocols, op_name) < getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(
up, op_name
) < data[op_name]: # shift backward e.g. from 1 to 3
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) <= data[op_name],
getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
for p in prots:
p.update({op_name: val})
val += 1
up.update(data, flush=True, session=session)
except (IntegrityError, OperationalError) as error:
if 'UNIQUE'.lower() in error.args[0].lower(
) or 'Duplicate' in error.args[
0]: # Covers SQLite, Oracle and MySQL error
raise exception.Duplicate(
'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.'
% (scheme, port, rse, hostname))
elif 'may not be NULL' in error.args[
0] or "cannot be null" in error.args[0]:
raise exception.InvalidObject('Missing values: %s' % error.args[0])
raise error
except DatabaseError as error:
if match(
'.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*',
error.args[0]):
raise exception.InvalidObject('Invalid values !')
raise error
