def list_rses(filters={}, session=None): """ Returns a list of all RSEs. :param filters: dictionary of attributes by which the results should be filtered. :param session: The database session in use. :returns: a list of dictionaries. """ rse_list = [] availability_mask1 = 0 availability_mask2 = 7 availability_mapping = { 'availability_read': 4, 'availability_write': 2, 'availability_delete': 1 } false_value = False # To make pep8 checker happy ... if filters: if 'availability' in filters and ('availability_read' in filters or 'availability_write' in filters or 'availability_delete' in filters): raise exception.InvalidObject( 'Cannot use availability and read, write, delete filter at the same time.' ) query = session.query(models.RSE).\ join(models.RSEAttrAssociation, models.RSE.id == models.RSEAttrAssociation.rse_id).\ filter(models.RSE.deleted == false_value).group_by(models.RSE) for (k, v) in filters.items(): if hasattr(models.RSE, k): if k == 'rse_type': query = query.filter( getattr(models.RSE, k) == RSEType.from_sym(v)) else: query = query.filter(getattr(models.RSE, k) == v) elif k in [ 'availability_read', 'availability_write', 'availability_delete' ]: if v: availability_mask1 = availability_mask1 | availability_mapping[ k] else: availability_mask2 = availability_mask2 & ~availability_mapping[ k] else: t = aliased(models.RSEAttrAssociation) query = query.join( t, t.rse_id == models.RSEAttrAssociation.rse_id) query = query.filter(t.key == k) # FIXME # ATLAS RSE listing workaround (since booleans are capital 'True'/'False') # remove elif branch after appropriate database fix has been applied # see also db/types.py if isinstance(v, bool): query = query.filter( or_(t.value == v, t.value == 'tmp_atlas_%s' % v, t.value == 'tmp_atlas_%s' % 1 if v else 0)) else: query = query.filter( or_(t.value == v, t.value == 'tmp_atlas_%s' % v)) condition1, condition2 = [], [] for i in range(0, 8): if i | availability_mask1 == i: condition1.append(models.RSE.availability == i) if i & availability_mask2 == i: condition2.append(models.RSE.availability == i) if 'availability' not in filters: query = query.filter( sqlalchemy.and_(sqlalchemy.or_(*condition1), sqlalchemy.or_(*condition2))) for row in query: d = {} for column in row.__table__.columns: d[column.name] = getattr(row, column.name) rse_list.append(d) else: query = session.query(models.RSE).filter_by(deleted=False).order_by( models.RSE.rse) for row in query: dic = {} for column in row.__table__.columns: dic[column.name] = getattr(row, column.name) rse_list.append(dic) return rse_list
def add_protocol(rse, parameter, session=None): """ Add a protocol to an existing RSE. If entries with equal or less priority for an operation exist, the existing one will be reorded (i.e. +1). :param rse: the name of the new rse. :param parameter: parameters of the new protocol entry. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEOperationNotSupported: If no scheme supported the requested operation for the given RSE. :raises RSEProtocolDomainNotSupported: If an undefined domain was provided. :raises RSEProtocolPriorityError: If the provided priority for the scheme is to big or below zero. :raises Duplicate: If scheme with identifier, hostname and port already exists for the given RSE. """ rid = get_rse_id(rse=rse, session=session) if not rid: raise exception.RSENotFound('RSE \'%s\' not found') # Insert new protocol entry parameter['rse_id'] = rid # Default values parameter['port'] = parameter.get('port', 0) parameter['hostname'] = parameter.get('hostname', 'localhost') # Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan] if 'domains' in parameter.keys(): for s in parameter['domains']: if s not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'The protocol domain \'%s\' is not defined in the schema.' % s) for op in parameter['domains'][s]: if op not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation \'%s\' not defined in schema.' % (op)) op_name = op if op == 'third_party_copy' else ''.join( [op, '_', s]).lower() if parameter['domains'][s][op] < 0: raise exception.RSEProtocolPriorityError( 'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (parameter['domains'][s][op], op, s)) parameter[op_name] = parameter['domains'][s][op] del parameter['domains'] if ('extended_attributes' in parameter) and parameter['extended_attributes']: try: parameter['extended_attributes'] = json.dumps( parameter['extended_attributes'], separators=(',', ':')) except ValueError: pass # String is not JSON if parameter['scheme'] == 'srm': if ('extended_attributes' not in parameter) or ('web_service_path' not in parameter['extended_attributes']): raise exception.InvalidObject( 'Missing values! For SRM, extended_attributes and web_service_path must be specified' ) try: new_protocol = models.RSEProtocols() new_protocol.update(parameter) new_protocol.save(session=session) except (IntegrityError, FlushError, OperationalError) as error: if ('UNIQUE constraint failed' in error.args[0]) or ('conflicts with persistent instance' in error.args[0]) \ or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \ or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \ or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0])\ or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]): raise exception.Duplicate( 'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (parameter['scheme'], parameter['port'], rse, parameter['hostname'])) elif 'may not be NULL' in error.args[0] \ or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \ or match('.*OperationalError.*cannot be null.*', error.args[0]): raise exception.InvalidObject('Missing values!') raise error return new_protocol
def update_protocols(rse, scheme, data, hostname, port, session=None): """ Updates an existing protocol entry for an RSE. If necessary, priorities for read, write, and delete operations of other protocol entires will be updated too. :param rse: the name of the new rse. :param scheme: Protocol identifer. :param data: Dict with new values (keys must match column names in the database). :param hostname: Hostname defined for the scheme, used if more than one scheme is registered with the same identifier. :param port: The port registered for the hostename, used if more than one scheme is regsitered with the same identifier and hostname. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE. :raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE. :raises RSEProtocolDomainNotSupported: If an undefined domain was provided. :raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero. :raises KeyNotFound: Invalid data for update provided. :raises Duplicate: If protocol with identifier, hostname and port already exists for the given RSE. """ rid = get_rse_id(rse=rse, session=session) # Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan] if 'domains' in data: for s in data['domains']: if s not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'The protocol domain \'%s\' is not defined in the schema.' % s) for op in data['domains'][s]: if op not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation \'%s\' not defined in schema.' % (op)) op_name = op if op != 'third_party_copy': op_name = ''.join([op, '_', s]) no = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= 0)).\ count() if not 0 <= data['domains'][s][op] <= no: raise exception.RSEProtocolPriorityError( 'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (data['domains'][s][op], op, s)) data[op_name] = data['domains'][s][op] del data['domains'] if 'extended_attributes' in data: try: data['extended_attributes'] = json.dumps( data['extended_attributes'], separators=(',', ':')) except ValueError: pass # String is not JSON if not rid: raise exception.RSENotFound('RSE \'%s\' not found') terms = [ models.RSEProtocols.rse_id == rid, models.RSEProtocols.scheme == scheme, models.RSEProtocols.hostname == hostname, models.RSEProtocols.port == port ] try: up = session.query(models.RSEProtocols).filter(*terms).first() if up is None: msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % ( rse, scheme, hostname, port) raise exception.RSEProtocolNotSupported(msg) # Preparing gaps if priority is updated for domain in utils.rse_supported_protocol_domains(): for op in utils.rse_supported_protocol_operations(): op_name = op if op != 'third_party_copy': op_name = ''.join([op, '_', domain]) if op_name in data: prots = [] if (not getattr(up, op_name)) and data[ op_name]: # reactivate protocol e.g. from 0 to 1 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name])).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr(up, op_name) and ( not data[op_name] ): # deactivate protocol e.g. from 1 to 0 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) elif getattr( up, op_name ) > data[op_name]: # shift forward e.g. from 5 to 2 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name], getattr(models.RSEProtocols, op_name) < getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr( up, op_name ) < data[op_name]: # shift backward e.g. from 1 to 3 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) <= data[op_name], getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) for p in prots: p.update({op_name: val}) val += 1 up.update(data, flush=True, session=session) except (IntegrityError, OperationalError) as error: if 'UNIQUE'.lower() in error.args[0].lower( ) or 'Duplicate' in error.args[ 0]: # Covers SQLite, Oracle and MySQL error raise exception.Duplicate( 'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (scheme, port, rse, hostname)) elif 'may not be NULL' in error.args[ 0] or "cannot be null" in error.args[0]: raise exception.InvalidObject('Missing values: %s' % error.args[0]) raise error except DatabaseError as error: if match( '.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*', error.args[0]): raise exception.InvalidObject('Invalid values !') raise error