def list_rses(filters={}, session=None):
"""
Returns a list of all RSEs.
:param filters: dictionary of attributes by which the results should be filtered.
:param session: The database session in use.
:returns: a list of dictionaries.
"""
rse_list = []
availability_mask1 = 0
availability_mask2 = 7
availability_mapping = {
'availability_read': 4,
'availability_write': 2,
'availability_delete': 1
}
false_value = False # To make pep8 checker happy ...
if filters:
if 'availability' in filters and ('availability_read' in filters
or 'availability_write' in filters
or 'availability_delete' in filters):
raise exception.InvalidObject(
'Cannot use availability and read, write, delete filter at the same time.'
)
query = session.query(models.RSE).\
join(models.RSEAttrAssociation, models.RSE.id == models.RSEAttrAssociation.rse_id).\
filter(models.RSE.deleted == false_value).group_by(models.RSE)
for (k, v) in filters.items():
if hasattr(models.RSE, k):
if k == 'rse_type':
query = query.filter(
getattr(models.RSE, k) == RSEType.from_sym(v))
else:
query = query.filter(getattr(models.RSE, k) == v)
elif k in [
'availability_read', 'availability_write',
'availability_delete'
]:
if v:
availability_mask1 = availability_mask1 | availability_mapping[
k]
else:
availability_mask2 = availability_mask2 & ~availability_mapping[
k]
else:
t = aliased(models.RSEAttrAssociation)
query = query.join(
t, t.rse_id == models.RSEAttrAssociation.rse_id)
query = query.filter(t.key == k)
# FIXME
# ATLAS RSE listing workaround (since booleans are capital 'True'/'False')
# remove elif branch after appropriate database fix has been applied
# see also db/types.py
if isinstance(v, bool):
query = query.filter(
or_(t.value == v, t.value == 'tmp_atlas_%s' % v,
t.value == 'tmp_atlas_%s' % 1 if v else 0))
else:
query = query.filter(
or_(t.value == v, t.value == 'tmp_atlas_%s' % v))
condition1, condition2 = [], []
for i in range(0, 8):
if i | availability_mask1 == i:
condition1.append(models.RSE.availability == i)
if i & availability_mask2 == i:
condition2.append(models.RSE.availability == i)
if 'availability' not in filters:
query = query.filter(
sqlalchemy.and_(sqlalchemy.or_(*condition1),
sqlalchemy.or_(*condition2)))
for row in query:
d = {}
for column in row.__table__.columns:
d[column.name] = getattr(row, column.name)
rse_list.append(d)
else:
query = session.query(models.RSE).filter_by(deleted=False).order_by(
models.RSE.rse)
for row in query:
dic = {}
for column in row.__table__.columns:
dic[column.name] = getattr(row, column.name)
rse_list.append(dic)
return rse_list
def add_protocol(rse, parameter, session=None):
"""
Add a protocol to an existing RSE. If entries with equal or less priority for
an operation exist, the existing one will be reorded (i.e. +1).
:param rse: the name of the new rse.
:param parameter: parameters of the new protocol entry.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEOperationNotSupported: If no scheme supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the scheme is to big or below zero.
:raises Duplicate: If scheme with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
# Insert new protocol entry
parameter['rse_id'] = rid
# Default values
parameter['port'] = parameter.get('port', 0)
parameter['hostname'] = parameter.get('hostname', 'localhost')
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in parameter.keys():
for s in parameter['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'The protocol domain \'%s\' is not defined in the schema.'
% s)
for op in parameter['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation \'%s\' not defined in schema.' % (op))
op_name = op if op == 'third_party_copy' else ''.join(
[op, '_', s]).lower()
if parameter['domains'][s][op] < 0:
raise exception.RSEProtocolPriorityError(
'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.'
% (parameter['domains'][s][op], op, s))
parameter[op_name] = parameter['domains'][s][op]
del parameter['domains']
if ('extended_attributes'
in parameter) and parameter['extended_attributes']:
try:
parameter['extended_attributes'] = json.dumps(
parameter['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if parameter['scheme'] == 'srm':
if ('extended_attributes'
not in parameter) or ('web_service_path'
not in parameter['extended_attributes']):
raise exception.InvalidObject(
'Missing values! For SRM, extended_attributes and web_service_path must be specified'
)
try:
new_protocol = models.RSEProtocols()
new_protocol.update(parameter)
new_protocol.save(session=session)
except (IntegrityError, FlushError, OperationalError) as error:
if ('UNIQUE constraint failed' in error.args[0]) or ('conflicts with persistent instance' in error.args[0]) \
or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \
or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0])\
or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]):
raise exception.Duplicate(
'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.'
% (parameter['scheme'], parameter['port'], rse,
parameter['hostname']))
elif 'may not be NULL' in error.args[0] \
or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \
or match('.*OperationalError.*cannot be null.*', error.args[0]):
raise exception.InvalidObject('Missing values!')
raise error
return new_protocol
def update_protocols(rse, scheme, data, hostname, port, session=None):
"""
Updates an existing protocol entry for an RSE. If necessary, priorities for read,
write, and delete operations of other protocol entires will be updated too.
:param rse: the name of the new rse.
:param scheme: Protocol identifer.
:param data: Dict with new values (keys must match column names in the database).
:param hostname: Hostname defined for the scheme, used if more than one scheme
is registered with the same identifier.
:param port: The port registered for the hostename, used if more than one scheme
is regsitered with the same identifier and hostname.
:param session: The database session in use.
:raises RSENotFound: If RSE is not found.
:raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE.
:raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE.
:raises RSEProtocolDomainNotSupported: If an undefined domain was provided.
:raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero.
:raises KeyNotFound: Invalid data for update provided.
:raises Duplicate: If protocol with identifier, hostname and port already exists
for the given RSE.
"""
rid = get_rse_id(rse=rse, session=session)
# Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan]
if 'domains' in data:
for s in data['domains']:
if s not in utils.rse_supported_protocol_domains():
raise exception.RSEProtocolDomainNotSupported(
'The protocol domain \'%s\' is not defined in the schema.'
% s)
for op in data['domains'][s]:
if op not in utils.rse_supported_protocol_operations():
raise exception.RSEOperationNotSupported(
'Operation \'%s\' not defined in schema.' % (op))
op_name = op
if op != 'third_party_copy':
op_name = ''.join([op, '_', s])
no = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= 0)).\
count()
if not 0 <= data['domains'][s][op] <= no:
raise exception.RSEProtocolPriorityError(
'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.'
% (data['domains'][s][op], op, s))
data[op_name] = data['domains'][s][op]
del data['domains']
if 'extended_attributes' in data:
try:
data['extended_attributes'] = json.dumps(
data['extended_attributes'], separators=(',', ':'))
except ValueError:
pass # String is not JSON
if not rid:
raise exception.RSENotFound('RSE \'%s\' not found')
terms = [
models.RSEProtocols.rse_id == rid,
models.RSEProtocols.scheme == scheme,
models.RSEProtocols.hostname == hostname,
models.RSEProtocols.port == port
]
try:
up = session.query(models.RSEProtocols).filter(*terms).first()
if up is None:
msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % (
rse, scheme, hostname, port)
raise exception.RSEProtocolNotSupported(msg)
# Preparing gaps if priority is updated
for domain in utils.rse_supported_protocol_domains():
for op in utils.rse_supported_protocol_operations():
op_name = op
if op != 'third_party_copy':
op_name = ''.join([op, '_', domain])
if op_name in data:
prots = []
if (not getattr(up, op_name)) and data[
op_name]: # reactivate protocol e.g. from 0 to 1
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name])).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(up, op_name) and (
not data[op_name]
): # deactivate protocol e.g. from 1 to 0
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
elif getattr(
up, op_name
) > data[op_name]: # shift forward e.g. from 5 to 2
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) >= data[op_name],
getattr(models.RSEProtocols, op_name) < getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = data[op_name] + 1
elif getattr(
up, op_name
) < data[op_name]: # shift backward e.g. from 1 to 3
prots = session.query(models.RSEProtocols).\
filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
getattr(models.RSEProtocols, op_name) <= data[op_name],
getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\
order_by(getattr(models.RSEProtocols, op_name).asc())
val = getattr(up, op_name)
for p in prots:
p.update({op_name: val})
val += 1
up.update(data, flush=True, session=session)
except (IntegrityError, OperationalError) as error:
if 'UNIQUE'.lower() in error.args[0].lower(
) or 'Duplicate' in error.args[
0]: # Covers SQLite, Oracle and MySQL error
raise exception.Duplicate(
'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.'
% (scheme, port, rse, hostname))
elif 'may not be NULL' in error.args[
0] or "cannot be null" in error.args[0]:
raise exception.InvalidObject('Missing values: %s' % error.args[0])
raise error
except DatabaseError as error:
if match(
'.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*',
error.args[0]):
raise exception.InvalidObject('Invalid values !')
raise error
