def _get_possible_protocols(rse_settings, operation, scheme=None, domain=None, impl=None): """ Filter the list of available protocols or provided by the supported ones. :param rse_settings: The rse settings. :param operation: The operation (write, read). :param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation. :param domain: Optional domain (lan/wan), if not specified, both will be returned :returns: The list of possible protocols. """ operation = operation.lower() candidates = rse_settings['protocols'] # convert scheme to list, if given as string if scheme and not isinstance(scheme, list): scheme = scheme.split(',') tbr = [] for protocol in candidates: # Check if impl given and filter if so if impl and protocol['impl'] != impl: tbr.append(protocol) continue # Check if scheme given and filter if so if scheme and protocol['scheme'] not in scheme: tbr.append(protocol) continue filtered = True if not domain: for d in list(protocol['domains'].keys()): if protocol['domains'][d][operation] != 0: filtered = False else: if protocol['domains'].get(domain, { operation: 0 }).get(operation) != 0: filtered = False if filtered: tbr.append(protocol) if len(candidates) <= len(tbr): raise exception.RSEProtocolNotSupported( 'No protocol for provided settings' ' found : %s.' % str(rse_settings)) return [c for c in candidates if c not in tbr]
def del_protocols(rse_id, scheme, hostname=None, port=None, session=None): """ Deletes an existing protocol entry for an RSE. :param rse_id: the id of the new rse. :param scheme: Protocol identifer. :param hostname: Hostname defined for the scheme, used if more than one scheme is registered with the same identifier. :param port: The port registered for the hostename, used if more than one scheme is regsitered with the same identifier and hostname. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEProtocolNotSupported: If no macthing scheme was found for the given RSE. """ try: rse_name = get_rse_name(rse_id=rse_id, session=session, include_deleted=False) except exception.RSENotFound: raise exception.RSENotFound('RSE \'%s\' not found' % rse_id) terms = [ models.RSEProtocols.rse_id == rse_id, models.RSEProtocols.scheme == scheme ] if hostname: terms.append(models.RSEProtocols.hostname == hostname) if port: terms.append(models.RSEProtocols.port == port) p = session.query(models.RSEProtocols).filter(*terms) if not p.all(): msg = 'RSE \'%s\' does not support protocol \'%s\'' % (rse_name, scheme) msg += ' for hostname \'%s\'' % hostname if hostname else '' msg += ' on port \'%s\'' % port if port else '' raise exception.RSEProtocolNotSupported(msg) for row in p: row.delete(session=session) # Filling gaps in protocol priorities for domain in utils.rse_supported_protocol_domains(): for op in utils.rse_supported_protocol_operations(): op_name = ''.join([op, '_', domain]) if getattr(models.RSEProtocols, op_name, None): prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rse_id, getattr(models.RSEProtocols, op_name) > 0)).\ order_by(getattr(models.RSEProtocols, op_name).asc()) i = 1 for p in prots: p.update({op_name: i}) i += 1
def _get_possible_protocols(rse_settings, operation, scheme=None): """ Filter the list of available protocols or provided by the supported ones. :param rse_settings: The rse settings. :param operation: The operation (write, read). :param scheme: optional filter if no specific protocol is defined in rse_setting for the provided operation. :returns: The list of possible protocols. """ operation = operation.lower() candidates = rse_settings['protocols'] if type(rse_settings['domain']) is not list: raise exception.RSEProtocolDomainNotSupported( 'Domain setting must be list.') for d in rse_settings['domain']: if d not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'Domain %s is not supported' ' by Rucio.' % d) # convert scheme to list, if given as string if scheme and not isinstance(scheme, list): scheme = scheme.split(',') tbr = [] for protocol in candidates: # Check if scheme given and filter if so if scheme and protocol['scheme'] not in scheme: tbr.append(protocol) continue filtered = True for d in rse_settings['domain']: if protocol['domains'][d][operation] != 0: filtered = False if filtered: tbr.append(protocol) if len(candidates) <= len(tbr): raise exception.RSEProtocolNotSupported( 'No protocol for provided settings' ' found : %s.' % str(rse_settings)) return [c for c in candidates if c not in tbr]
def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_token=None, protocol_attr=None, logger=logging.log): """ Instanciates the protocol defined for the given operation. :param rse_settings: RSE attributes :param operation: Intended operation for this protocol :param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation :param domain: Optional specification of the domain :param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication :param protocol_attr: Optionally passing the full protocol availability information to correctly select WAN/LAN :param logger: Optional decorated logger that can be passed from the calling daemons or servers. :returns: An instance of the requested protocol """ # Verify feasibility of Protocol operation = operation.lower() if operation not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation) if domain and domain not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain) if not protocol_attr: protocol_attr = select_protocol(rse_settings, operation, scheme, domain) else: candidates = _get_possible_protocols(rse_settings, operation, scheme, domain) if protocol_attr not in candidates: raise exception.RSEProtocolNotSupported('Protocol %s operation %s on domain %s not supported' % (protocol_attr, operation, domain)) # Instantiate protocol comp = protocol_attr['impl'].split('.') prefix = '.'.join(comp[-2:]) + ': ' logger = formatted_logger(logger, prefix + "%s") mod = __import__('.'.join(comp[:-1])) for n in comp[1:]: try: mod = getattr(mod, n) except AttributeError as e: logger(logging.DEBUG, 'Protocol implementations not supported.') raise exception.RucioException(str(e)) # TODO: provide proper rucio exception protocol_attr['auth_token'] = auth_token protocol = mod(protocol_attr, rse_settings, logger=logger) return protocol
def select_protocol(rse_settings, operation, scheme=None): operation = operation.lower() candidates = copy.copy(rse_settings['protocols']) if type(rse_settings['domain']) is not list: raise exception.RSEProtocolDomainNotSupported('Domain setting must be list.') for d in rse_settings['domain']: if d not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported('Domain %s is not supported by Rucio.' % d) tbr = list() for protocol in candidates: # Check if scheme given and filter if so if scheme: if not isinstance(scheme, list): scheme = scheme.split(',') if protocol['scheme'] not in scheme: tbr.append(protocol) continue for d in rse_settings['domain']: if protocol['domains'][d][operation] == 0: tbr.append(protocol) break for r in tbr: candidates.remove(r) if not len(candidates): raise exception.RSEProtocolNotSupported('No protocol for provided settings found : %s.' % str(rse_settings)) # Select the one with the highest priority candidates = sorted(candidates, key=lambda k: k['scheme']) best_choice = candidates[0] candidates.remove(best_choice) domain = rse_settings['domain'][0] for p in candidates: if p['domains'][domain][operation] < best_choice['domains'][domain][operation]: best_choice = p return best_choice
def find_matching_scheme(rse_settings_dest, rse_settings_src, operation_src, operation_dest, domain='wan', scheme=None): """ Find the best matching scheme between two RSEs :param rse_settings_dest: RSE settings for the destination RSE. :param rse_settings_src: RSE settings for the src RSE. :param operation_src: Source Operation such as read, write. :param operation_dest: Dest Operation such as read, write. :param domain: Domain such as lan, wan. :param scheme: List of supported schemes. :returns: Tuple of matching schemes (dest_scheme, src_scheme). """ operation_src = operation_src.lower() operation_dest = operation_dest.lower() src_candidates = copy.copy(rse_settings_src['protocols']) dest_candidates = copy.copy(rse_settings_dest['protocols']) # Clean up src_candidates tbr = list() for protocol in src_candidates: # Check if scheme given and filter if so if scheme: if not isinstance(scheme, list): scheme = scheme.split(',') if protocol['scheme'] not in scheme: tbr.append(protocol) continue if protocol['domains'].get(domain, {}).get(operation_src, 1) == 0: tbr.append(protocol) for r in tbr: src_candidates.remove(r) # Clean up dest_candidates tbr = list() for protocol in dest_candidates: # Check if scheme given and filter if so if scheme: if not isinstance(scheme, list): scheme = scheme.split(',') if protocol['scheme'] not in scheme: tbr.append(protocol) continue if protocol['domains'].get(domain, {}).get(operation_dest, 1) == 0: tbr.append(protocol) for r in tbr: dest_candidates.remove(r) if not len(src_candidates) or not len(dest_candidates): raise exception.RSEProtocolNotSupported( 'No protocol for provided settings found : %s.' % str(rse_settings_dest)) # Shuffle the candidates to load-balance across equal weights. random.shuffle(dest_candidates) random.shuffle(src_candidates) # Select the one with the highest priority dest_candidates = sorted( dest_candidates, key=lambda k: k['domains'][domain][operation_dest]) src_candidates = sorted(src_candidates, key=lambda k: k['domains'][domain][operation_src]) for dest_protocol in dest_candidates: for src_protocol in src_candidates: if __check_compatible_scheme(dest_protocol['scheme'], src_protocol['scheme']): return (dest_protocol['scheme'], src_protocol['scheme']) raise exception.RSEProtocolNotSupported( 'No protocol for provided settings found : %s.' % str(rse_settings_dest))
def update_protocols(rse, scheme, data, hostname, port, session=None): """ Updates an existing protocol entry for an RSE. If necessary, priorities for read, write, and delete operations of other protocol entires will be updated too. :param rse: the name of the new rse. :param scheme: Protocol identifer. :param data: Dict with new values (keys must match column names in the database). :param hostname: Hostname defined for the scheme, used if more than one scheme is registered with the same identifier. :param port: The port registered for the hostename, used if more than one scheme is regsitered with the same identifier and hostname. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE. :raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE. :raises RSEProtocolDomainNotSupported: If an undefined domain was provided. :raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero. :raises KeyNotFound: Invalid data for update provided. :raises Duplicate: If protocol with identifier, hostname and port already exists for the given RSE. """ rid = get_rse_id(rse=rse, session=session) # Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan] if 'domains' in data: for s in data['domains']: if s not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'The protocol domain \'%s\' is not defined in the schema.' % s) for op in data['domains'][s]: if op not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation \'%s\' not defined in schema.' % (op)) op_name = op if op != 'third_party_copy': op_name = ''.join([op, '_', s]) no = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= 0)).\ count() if not 0 <= data['domains'][s][op] <= no: raise exception.RSEProtocolPriorityError( 'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (data['domains'][s][op], op, s)) data[op_name] = data['domains'][s][op] del data['domains'] if 'extended_attributes' in data: try: data['extended_attributes'] = json.dumps( data['extended_attributes'], separators=(',', ':')) except ValueError: pass # String is not JSON if not rid: raise exception.RSENotFound('RSE \'%s\' not found') terms = [ models.RSEProtocols.rse_id == rid, models.RSEProtocols.scheme == scheme, models.RSEProtocols.hostname == hostname, models.RSEProtocols.port == port ] try: up = session.query(models.RSEProtocols).filter(*terms).first() if up is None: msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % ( rse, scheme, hostname, port) raise exception.RSEProtocolNotSupported(msg) # Preparing gaps if priority is updated for domain in utils.rse_supported_protocol_domains(): for op in utils.rse_supported_protocol_operations(): op_name = op if op != 'third_party_copy': op_name = ''.join([op, '_', domain]) if op_name in data: prots = [] if (not getattr(up, op_name)) and data[ op_name]: # reactivate protocol e.g. from 0 to 1 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name])).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr(up, op_name) and ( not data[op_name] ): # deactivate protocol e.g. from 1 to 0 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) elif getattr( up, op_name ) > data[op_name]: # shift forward e.g. from 5 to 2 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name], getattr(models.RSEProtocols, op_name) < getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr( up, op_name ) < data[op_name]: # shift backward e.g. from 1 to 3 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) <= data[op_name], getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) for p in prots: p.update({op_name: val}) val += 1 up.update(data, flush=True, session=session) except (IntegrityError, OperationalError) as error: if 'UNIQUE'.lower() in error.args[0].lower( ) or 'Duplicate' in error.args[ 0]: # Covers SQLite, Oracle and MySQL error raise exception.Duplicate( 'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (scheme, port, rse, hostname)) elif 'may not be NULL' in error.args[ 0] or "cannot be null" in error.args[0]: raise exception.InvalidObject('Missing values: %s' % error.args[0]) raise error except DatabaseError as error: if match( '.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*', error.args[0]): raise exception.InvalidObject('Invalid values !') raise error