def create_protocol(rse_settings, operation, scheme=None, domain='wan'): """ Instanciates the protocol defined for the given operation. :param rse_attr: RSE attributes :param operation: Intended operation for this protocol :param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation :param domain: Optional specification of the domain :returns: An instance of the requested protocol """ # Verify feasibility of Protocol operation = operation.lower() if operation not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation %s is not supported' % operation) if domain and domain not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'Domain %s not supported' % domain) protocol_attr = select_protocol(rse_settings, operation, scheme, domain) # Instantiate protocol comp = protocol_attr['impl'].split('.') mod = __import__('.'.join(comp[:-1])) for n in comp[1:]: try: mod = getattr(mod, n) except AttributeError: print('Protocol implementation not found') raise # TODO: provide proper rucio exception protocol = mod(protocol_attr, rse_settings) return protocol
def get_protocols_ordered(rse_settings, operation, scheme=None, domain='wan'): if operation not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation) if domain and domain not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain) candidates = _get_possible_protocols(rse_settings, operation, scheme, domain) candidates.sort(key=lambda k: k['domains'][domain][operation]) return candidates
def select_protocol(rse_settings, operation, scheme=None, domain='wan'): if operation not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation) if domain and domain not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain) candidates = _get_possible_protocols(rse_settings, operation, scheme, domain) # Shuffle candidates to load-balance over equal sources random.shuffle(candidates) return min(candidates, key=lambda k: k['domains'][domain][operation])
def del_protocols(rse_id, scheme, hostname=None, port=None, session=None): """ Deletes an existing protocol entry for an RSE. :param rse_id: the id of the new rse. :param scheme: Protocol identifer. :param hostname: Hostname defined for the scheme, used if more than one scheme is registered with the same identifier. :param port: The port registered for the hostename, used if more than one scheme is regsitered with the same identifier and hostname. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEProtocolNotSupported: If no macthing scheme was found for the given RSE. """ try: rse_name = get_rse_name(rse_id=rse_id, session=session, include_deleted=False) except exception.RSENotFound: raise exception.RSENotFound('RSE \'%s\' not found' % rse_id) terms = [ models.RSEProtocols.rse_id == rse_id, models.RSEProtocols.scheme == scheme ] if hostname: terms.append(models.RSEProtocols.hostname == hostname) if port: terms.append(models.RSEProtocols.port == port) p = session.query(models.RSEProtocols).filter(*terms) if not p.all(): msg = 'RSE \'%s\' does not support protocol \'%s\'' % (rse_name, scheme) msg += ' for hostname \'%s\'' % hostname if hostname else '' msg += ' on port \'%s\'' % port if port else '' raise exception.RSEProtocolNotSupported(msg) for row in p: row.delete(session=session) # Filling gaps in protocol priorities for domain in utils.rse_supported_protocol_domains(): for op in utils.rse_supported_protocol_operations(): op_name = ''.join([op, '_', domain]) if getattr(models.RSEProtocols, op_name, None): prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rse_id, getattr(models.RSEProtocols, op_name) > 0)).\ order_by(getattr(models.RSEProtocols, op_name).asc()) i = 1 for p in prots: p.update({op_name: i}) i += 1
def del_protocols(rse, scheme, hostname=None, port=None, session=None): """ Deletes an existing protocol entry for an RSE. :param rse: the name of the new rse. :param scheme: Protocol identifer. :param hostname: Hostname defined for the scheme, used if more than one scheme is registered with the same identifier. :param port: The port registered for the hostename, used if more than one scheme is regsitered with the same identifier and hostname. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEProtocolNotSupported: If no macthing scheme was found for the given RSE. """ rid = get_rse_id(rse=rse, session=session) if not rid: raise exception.RSENotFound('RSE \'%s\' not found') terms = [models.RSEProtocols.rse_id == rid, models.RSEProtocols.scheme == scheme] if hostname: terms.append(models.RSEProtocols.hostname == hostname) if port: terms.append(models.RSEProtocols.port == port) p = session.query(models.RSEProtocols).filter(*terms) if not p.all(): msg = 'RSE \'%s\' does not support protocol \'%s\'' % (rse, scheme) msg += ' for hostname \'%s\'' % hostname if hostname else '' msg += ' on port \'%s\'' % port if port else '' raise exception.RSEProtocolNotSupported(msg) for row in p: row.delete(session=session) # Filling gaps in protocol priorities for domain in utils.rse_supported_protocol_domains(): for op in utils.rse_supported_protocol_operations(): op_name = ''.join([op, '_', domain]) prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) > 0 )).order_by(getattr(models.RSEProtocols, op_name).asc()) i = 1 for p in prots: p.update({op_name: i}) i += 1
def create_protocol(rse_settings, operation, scheme=None): """ Instanciates the protocol defined for the given operation. :param rse_attr: RSE attributes :param operation: the intended operation for this protocol :param scheme: optional filter if no specific protocol is defined in rse_setting for the provided operation :returns: an instance of the requested protocol """ # Verify feasibility of Protocol operation = operation.lower() if operation not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation %s is not supported' % operation) rse_settings['domain'] = [rse_settings['domain']] if type( rse_settings['domain']) is not list else rse_settings['domain'] for domain in rse_settings['domain']: if domain.lower() not in utils.rse_supported_protocol_domains(): raise exception.RSEOperationNotSupported( 'Domain %s not supported' % rse_settings['domain']) if rse_settings['%s_protocol' % operation] == DEFAULT_PROTOCOL: protocol_attr = select_protocol(rse_settings, operation, scheme) else: protocol_attr = rse_settings['%s_protocol' % operation] for d in rse_settings['domain']: if protocol_attr['domains'][d][operation] == 0: raise exception.RSEOperationNotSupported( 'Operation %s for domain %s not supported by %s' % (operation, rse_settings['domain'], protocol_attr['scheme'])) # Instanciate protocol comp = protocol_attr['impl'].split('.') mod = __import__('.'.join(comp[:-1])) for n in comp[1:]: try: mod = getattr(mod, n) except AttributeError: print 'Protocol implementation not found' raise # TODO: provide proper rucio exception protocol = mod(protocol_attr, rse_settings) return protocol
def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_token=None, protocol_attr=None, logger=logging.log): """ Instanciates the protocol defined for the given operation. :param rse_settings: RSE attributes :param operation: Intended operation for this protocol :param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation :param domain: Optional specification of the domain :param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication :param protocol_attr: Optionally passing the full protocol availability information to correctly select WAN/LAN :param logger: Optional decorated logger that can be passed from the calling daemons or servers. :returns: An instance of the requested protocol """ # Verify feasibility of Protocol operation = operation.lower() if operation not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation) if domain and domain not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain) if not protocol_attr: protocol_attr = select_protocol(rse_settings, operation, scheme, domain) else: candidates = _get_possible_protocols(rse_settings, operation, scheme, domain) if protocol_attr not in candidates: raise exception.RSEProtocolNotSupported('Protocol %s operation %s on domain %s not supported' % (protocol_attr, operation, domain)) # Instantiate protocol comp = protocol_attr['impl'].split('.') prefix = '.'.join(comp[-2:]) + ': ' logger = formatted_logger(logger, prefix + "%s") mod = __import__('.'.join(comp[:-1])) for n in comp[1:]: try: mod = getattr(mod, n) except AttributeError as e: logger(logging.DEBUG, 'Protocol implementations not supported.') raise exception.RucioException(str(e)) # TODO: provide proper rucio exception protocol_attr['auth_token'] = auth_token protocol = mod(protocol_attr, rse_settings, logger=logger) return protocol
def create_protocol(rse_settings, operation, scheme=None): """ Instanciates the protocol defined for the given operation. :param rse_attr: RSE attributes :param operation: the intended operation for this protocol :param scheme: optional filter if no specific protocol is defined in rse_setting for the provided operation :returns: an instance of the requested protocol """ # Verify feasibility of Protocol operation = operation.lower() if operation not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation) rse_settings['domain'] = [rse_settings['domain']] if type(rse_settings['domain']) is not list else rse_settings['domain'] for domain in rse_settings['domain']: if domain.lower() not in utils.rse_supported_protocol_domains(): raise exception.RSEOperationNotSupported('Domain %s not supported' % rse_settings['domain']) if rse_settings['%s_protocol' % operation] == DEFAULT_PROTOCOL: protocol_attr = select_protocol(rse_settings, operation, scheme) else: protocol_attr = rse_settings['%s_protocol' % operation] for d in rse_settings['domain']: if protocol_attr['domains'][d][operation] == 0: raise exception.RSEOperationNotSupported('Operation %s for domain %s not supported by %s' % (operation, rse_settings['domain'], protocol_attr['scheme'])) # Instanciate protocol comp = protocol_attr['impl'].split('.') mod = __import__('.'.join(comp[:-1])) for n in comp[1:]: try: mod = getattr(mod, n) except AttributeError: print 'Protocol implementation not found' raise # TODO: provide proper rucio exception protocol = mod(protocol_attr, rse_settings) return protocol
def get_rse_protocols(rse, schemes=None, session=None): """ Returns protocol information. Parameter combinations are: (operation OR default) XOR scheme. :param rse: The name of the rse. :param schemes: a list of schemes to filter by. :param session: The database session. :returns: A dict with RSE information and supported protocols :raises RSENotFound: If RSE is not found. """ _rse = get_rse(rse=rse, session=session) if not _rse: raise exception.RSENotFound('RSE \'%s\' not found') lfn2pfn_algorithms = get_rse_attribute('lfn2pfn_algorithm', rse_id=_rse.id, session=session) # Resolve LFN2PFN default algorithm as soon as possible. This way, we can send back the actual # algorithm name in response to REST queries. lfn2pfn_algorithm = get_lfn2pfn_algorithm_default() if lfn2pfn_algorithms: lfn2pfn_algorithm = lfn2pfn_algorithms[0] # Copy verify_checksum from the attributes, later: assume True if not specified verify_checksum = get_rse_attribute('verify_checksum', rse_id=_rse.id, session=session) read = True if _rse.availability & 4 else False write = True if _rse.availability & 2 else False delete = True if _rse.availability & 1 else False info = { 'id': _rse.id, 'rse': _rse.rse, 'availability_read': read, 'availability_write': write, 'availability_delete': delete, 'domain': utils.rse_supported_protocol_domains(), 'protocols': list(), 'deterministic': _rse.deterministic, 'lfn2pfn_algorithm': lfn2pfn_algorithm, 'rse_type': str(_rse.rse_type), 'credentials': None, 'volatile': _rse.volatile, 'verify_checksum': verify_checksum[0] if verify_checksum else True, 'staging_area': _rse.staging_area } for op in utils.rse_supported_protocol_operations(): info['%s_protocol' % op] = 1 # 1 indicates the default protocol query = None terms = [models.RSEProtocols.rse_id == _rse.id] if schemes: if not type(schemes) is list: schemes = [schemes] terms.extend([models.RSEProtocols.scheme.in_(schemes)]) query = session.query( models.RSEProtocols.hostname, models.RSEProtocols.scheme, models.RSEProtocols.port, models.RSEProtocols.prefix, models.RSEProtocols.impl, models.RSEProtocols.read_lan, models.RSEProtocols.write_lan, models.RSEProtocols.delete_lan, models.RSEProtocols.read_wan, models.RSEProtocols.write_wan, models.RSEProtocols.delete_wan, models.RSEProtocols.third_party_copy, models.RSEProtocols.extended_attributes).filter(*terms) for row in query: p = { 'hostname': row.hostname, 'scheme': row.scheme, 'port': row.port, 'prefix': row.prefix if row.prefix is not None else '', 'impl': row.impl, 'domains': { 'lan': { 'read': row.read_lan, 'write': row.write_lan, 'delete': row.delete_lan }, 'wan': { 'read': row.read_wan, 'write': row.write_wan, 'delete': row.delete_wan, 'third_party_copy': row.third_party_copy } }, 'extended_attributes': row.extended_attributes } try: p['extended_attributes'] = json.load( StringIO(p['extended_attributes'])) except ValueError: pass # If value is not a JSON string info['protocols'].append(p) return info
def add_protocol(rse, parameter, session=None): """ Add a protocol to an existing RSE. If entries with equal or less priority for an operation exist, the existing one will be reorded (i.e. +1). :param rse: the name of the new rse. :param parameter: parameters of the new protocol entry. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEOperationNotSupported: If no scheme supported the requested operation for the given RSE. :raises RSEProtocolDomainNotSupported: If an undefined domain was provided. :raises RSEProtocolPriorityError: If the provided priority for the scheme is to big or below zero. :raises Duplicate: If scheme with identifier, hostname and port already exists for the given RSE. """ rid = get_rse_id(rse=rse, session=session) if not rid: raise exception.RSENotFound('RSE \'%s\' not found') # Insert new protocol entry parameter['rse_id'] = rid # Default values parameter['port'] = parameter.get('port', 0) parameter['hostname'] = parameter.get('hostname', 'localhost') # Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan] if 'domains' in parameter.keys(): for s in parameter['domains']: if s not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'The protocol domain \'%s\' is not defined in the schema.' % s) for op in parameter['domains'][s]: if op not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation \'%s\' not defined in schema.' % (op)) op_name = op if op == 'third_party_copy' else ''.join( [op, '_', s]).lower() if parameter['domains'][s][op] < 0: raise exception.RSEProtocolPriorityError( 'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (parameter['domains'][s][op], op, s)) parameter[op_name] = parameter['domains'][s][op] del parameter['domains'] if ('extended_attributes' in parameter) and parameter['extended_attributes']: try: parameter['extended_attributes'] = json.dumps( parameter['extended_attributes'], separators=(',', ':')) except ValueError: pass # String is not JSON if parameter['scheme'] == 'srm': if ('extended_attributes' not in parameter) or ('web_service_path' not in parameter['extended_attributes']): raise exception.InvalidObject( 'Missing values! For SRM, extended_attributes and web_service_path must be specified' ) try: new_protocol = models.RSEProtocols() new_protocol.update(parameter) new_protocol.save(session=session) except (IntegrityError, FlushError, OperationalError) as error: if ('UNIQUE constraint failed' in error.args[0]) or ('conflicts with persistent instance' in error.args[0]) \ or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \ or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \ or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0])\ or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]): raise exception.Duplicate( 'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (parameter['scheme'], parameter['port'], rse, parameter['hostname'])) elif 'may not be NULL' in error.args[0] \ or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \ or match('.*OperationalError.*cannot be null.*', error.args[0]): raise exception.InvalidObject('Missing values!') raise error return new_protocol
def update_protocols(rse, scheme, data, hostname, port, session=None): """ Updates an existing protocol entry for an RSE. If necessary, priorities for read, write, and delete operations of other protocol entires will be updated too. :param rse: the name of the new rse. :param scheme: Protocol identifer. :param data: Dict with new values (keys must match column names in the database). :param hostname: Hostname defined for the scheme, used if more than one scheme is registered with the same identifier. :param port: The port registered for the hostename, used if more than one scheme is regsitered with the same identifier and hostname. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE. :raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE. :raises RSEProtocolDomainNotSupported: If an undefined domain was provided. :raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero. :raises KeyNotFound: Invalid data for update provided. :raises Duplicate: If protocol with identifier, hostname and port already exists for the given RSE. """ rid = get_rse_id(rse=rse, session=session) # Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan] if 'domains' in data: for s in data['domains']: if s not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'The protocol domain \'%s\' is not defined in the schema.' % s) for op in data['domains'][s]: if op not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation \'%s\' not defined in schema.' % (op)) op_name = op if op != 'third_party_copy': op_name = ''.join([op, '_', s]) no = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= 0)).\ count() if not 0 <= data['domains'][s][op] <= no: raise exception.RSEProtocolPriorityError( 'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (data['domains'][s][op], op, s)) data[op_name] = data['domains'][s][op] del data['domains'] if 'extended_attributes' in data: try: data['extended_attributes'] = json.dumps( data['extended_attributes'], separators=(',', ':')) except ValueError: pass # String is not JSON if not rid: raise exception.RSENotFound('RSE \'%s\' not found') terms = [ models.RSEProtocols.rse_id == rid, models.RSEProtocols.scheme == scheme, models.RSEProtocols.hostname == hostname, models.RSEProtocols.port == port ] try: up = session.query(models.RSEProtocols).filter(*terms).first() if up is None: msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % ( rse, scheme, hostname, port) raise exception.RSEProtocolNotSupported(msg) # Preparing gaps if priority is updated for domain in utils.rse_supported_protocol_domains(): for op in utils.rse_supported_protocol_operations(): op_name = op if op != 'third_party_copy': op_name = ''.join([op, '_', domain]) if op_name in data: prots = [] if (not getattr(up, op_name)) and data[ op_name]: # reactivate protocol e.g. from 0 to 1 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name])).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr(up, op_name) and ( not data[op_name] ): # deactivate protocol e.g. from 1 to 0 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) elif getattr( up, op_name ) > data[op_name]: # shift forward e.g. from 5 to 2 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name], getattr(models.RSEProtocols, op_name) < getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr( up, op_name ) < data[op_name]: # shift backward e.g. from 1 to 3 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) <= data[op_name], getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) for p in prots: p.update({op_name: val}) val += 1 up.update(data, flush=True, session=session) except (IntegrityError, OperationalError) as error: if 'UNIQUE'.lower() in error.args[0].lower( ) or 'Duplicate' in error.args[ 0]: # Covers SQLite, Oracle and MySQL error raise exception.Duplicate( 'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (scheme, port, rse, hostname)) elif 'may not be NULL' in error.args[ 0] or "cannot be null" in error.args[0]: raise exception.InvalidObject('Missing values: %s' % error.args[0]) raise error except DatabaseError as error: if match( '.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*', error.args[0]): raise exception.InvalidObject('Invalid values !') raise error
def update_protocols(rse, scheme, data, hostname, port, session=None): """ Updates an existing protocol entry for an RSE. If necessary, priorities for read, write, and delete operations of other protocol entires will be updated too. :param rse: the name of the new rse. :param scheme: Protocol identifer. :param data: Dict with new values (keys must match column names in the database). :param hostname: Hostname defined for the scheme, used if more than one scheme is registered with the same identifier. :param port: The port registered for the hostename, used if more than one scheme is regsitered with the same identifier and hostname. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE. :raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE. :raises RSEProtocolDomainNotSupported: If an undefined domain was provided. :raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero. :raises KeyNotFound: Invalid data for update provided. :raises Duplicate: If protocol with identifier, hostname and port already exists for the given RSE. """ rid = get_rse_id(rse=rse, session=session) # Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan] if 'domains' in data: for s in data['domains']: if s not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported('The protocol domain \'%s\' is not defined in the schema.' % s) for op in data['domains'][s]: if op not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported('Operation \'%s\' not defined in schema.' % (op)) op_name = ''.join([op, '_', s]) no = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) > 0, )).count() if not(0 <= data['domains'][s][op] <= no): raise exception.RSEProtocolPriorityError('The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (data['domains'][s][op], op, s)) data[op_name] = data['domains'][s][op] del(data['domains']) if 'extended_attributes' in data: try: data['extended_attributes'] = json.dumps(data['extended_attributes'], separators=(',', ':')) except ValueError: pass # String is not JSON if not rid: raise exception.RSENotFound('RSE \'%s\' not found') terms = [models.RSEProtocols.rse_id == rid, models.RSEProtocols.scheme == scheme, models.RSEProtocols.hostname == hostname, models.RSEProtocols.port == port ] try: up = session.query(models.RSEProtocols).filter(*terms).first() if up is None: msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % (rse, scheme, hostname, port) raise exception.RSEProtocolNotSupported(msg) # Preparing gaps if priority is updated for domain in utils.rse_supported_protocol_domains(): for op in utils.rse_supported_protocol_operations(): op_name = ''.join([op, '_', domain]) if op_name in data: if (not getattr(up, op_name)) and data[op_name]: # reactivate protocol e.g. from 0 to 1 prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name] )).order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr(up, op_name) and (not data[op_name]): # deactivate protocol e.g. from 1 to 0 prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) > getattr(up, op_name) )).order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) elif getattr(up, op_name) > data[op_name]: # shift forward e.g. from 5 to 2 prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name], getattr(models.RSEProtocols, op_name) < getattr(up, op_name) )).order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr(up, op_name) < data[op_name]: # shift backward e.g. from 1 to 3 prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) <= data[op_name], getattr(models.RSEProtocols, op_name) > getattr(up, op_name) )).order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) for p in prots: p.update({op_name: val}) val += 1 up.update(data, flush=True, session=session) except (IntegrityError, OperationalError) as e: if 'UNIQUE'.lower() in e.args[0].lower() or 'Duplicate' in e.args[0]: # Covers SQLite, Oracle and MySQL error raise exception.Duplicate('Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (scheme, port, rse, hostname)) elif 'may not be NULL' in e.args[0] or "cannot be null" in e.args[0]: raise exception.InvalidObject('Missing values: %s' % e.args[0]) raise e except DatabaseError, e: if match('.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*', e.args[0]): raise exception.InvalidObject('Invalid values !') raise e
def get_rse_protocols(rse, schemes=None, session=None): """ Returns protocol information. Parameter combinations are: (operation OR default) XOR scheme. :param rse: The name of the rse. :param schemes: a list of schemes to filter by. :param session: The database session. :returns: A dict with RSE information and supported protocols :raises RSENotFound: If RSE is not found. """ _rse = get_rse(rse=rse, session=session) if not _rse: raise exception.RSENotFound('RSE \'%s\' not found') read = True if _rse.availability & 4 else False write = True if _rse.availability & 2 else False delete = True if _rse.availability & 1 else False info = {'id': _rse.id, 'rse': _rse.rse, 'availability_read': read, 'availability_write': write, 'availability_delete': delete, 'domain': utils.rse_supported_protocol_domains(), 'protocols': list(), 'deterministic': _rse.deterministic, 'rse_type': str(_rse.rse_type), 'credentials': None, 'volatile': _rse.volatile, 'staging_area': _rse.staging_area } for op in utils.rse_supported_protocol_operations(): info['%s_protocol' % op] = 1 # 1 indicates the default protocol query = None terms = [models.RSEProtocols.rse_id == _rse.id] query = session.query(models.RSEProtocols.hostname, models.RSEProtocols.scheme, models.RSEProtocols.port, models.RSEProtocols.prefix, models.RSEProtocols.impl, models.RSEProtocols.read_lan, models.RSEProtocols.write_lan, models.RSEProtocols.delete_lan, models.RSEProtocols.read_wan, models.RSEProtocols.write_wan, models.RSEProtocols.delete_wan, models.RSEProtocols.extended_attributes).filter(*terms) for row in query: p = {'hostname': row.hostname, 'scheme': row.scheme, 'port': row.port, 'prefix': row.prefix if row.prefix is not None else '', 'impl': row.impl, 'domains': { 'lan': {'read': row.read_lan, 'write': row.write_lan, 'delete': row.delete_lan }, 'wan': {'read': row.read_wan, 'write': row.write_wan, 'delete': row.delete_wan } }, 'extended_attributes': row.extended_attributes } try: p['extended_attributes'] = json.load(StringIO(p['extended_attributes'])) except ValueError: pass # If value is not a JSON string info['protocols'].append(p) return info
def add_protocol(rse, parameter, session=None): """ Add a protocol to an existing RSE. If entries with equal or less priority for an operation exist, the existing one will be reorded (i.e. +1). :param rse: the name of the new rse. :param parameter: parameters of the new protocol entry. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEOperationNotSupported: If no scheme supported the requested operation for the given RSE. :raises RSEProtocolDomainNotSupported: If an undefined domain was provided. :raises RSEProtocolPriorityError: If the provided priority for the scheme is to big or below zero. :raises Duplicate: If scheme with identifier, hostname and port already exists for the given RSE. """ rid = get_rse_id(rse=rse, session=session) if not rid: raise exception.RSENotFound('RSE \'%s\' not found') # Insert new protocol entry parameter['rse_id'] = rid # Default values parameter['port'] = parameter.get('port', 0) parameter['hostname'] = parameter.get('hostname', 'localhost') # Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan] if 'domains' in parameter.keys(): for s in parameter['domains']: if s not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported('The protocol domain \'%s\' is not defined in the schema.' % s) for op in parameter['domains'][s]: if op not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported('Operation \'%s\' not defined in schema.' % (op)) op_name = ''.join([op, '_', s]).lower() no = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) > 0, )).count() if parameter['domains'][s][op] > (no + 1): parameter['domains'][s][op] = (no + 1) if not(0 <= parameter['domains'][s][op] <= (no + 1)): raise exception.RSEProtocolPriorityError('The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (parameter['domains'][s][op], op, s)) parameter[op_name] = parameter['domains'][s][op] del(parameter['domains']) if ('extended_attributes' in parameter) and parameter['extended_attributes']: try: parameter['extended_attributes'] = json.dumps(parameter['extended_attributes'], separators=(',', ':')) except ValueError: pass # String is not JSON if parameter['scheme'] == 'srm': if ('space_token' not in parameter['extended_attributes']) or ('web_service_path' not in parameter['extended_attributes']): raise exception.InvalidObject('Missing values! For SRM, extended_attributes and web_service_path must be specified') try: # Open gaps in protocols priorities for new protocol for domain in utils.rse_supported_protocol_domains(): for op in utils.rse_supported_protocol_operations(): op_name = ''.join([op, '_', domain]) if (op_name in parameter) and parameter[op_name]: prots = session.query(models.RSEProtocols).filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= parameter[op_name], )).order_by(getattr(models.RSEProtocols, op_name).asc()) val = parameter[op_name] + 1 for p in prots: p.update({op_name: val}) val += 1 new_protocol = models.RSEProtocols() new_protocol.update(parameter) new_protocol.save(session=session) except (IntegrityError, FlushError, OperationalError) as e: if ('not unique' in e.args[0]) or ('conflicts with persistent instance' in e.args[0]) \ or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', e.args[0]): raise exception.Duplicate('Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (parameter['scheme'], parameter['port'], rse, parameter['hostname'])) elif 'may not be NULL' in e.args[0] \ or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', e.args[0]) \ or match('.*OperationalError.*cannot be null.*', e.args[0]): raise exception.InvalidObject('Missing values!') raise e return new_protocol