def create_protocol(rse_settings, operation, scheme=None, domain='wan'): """ Instanciates the protocol defined for the given operation. :param rse_attr: RSE attributes :param operation: Intended operation for this protocol :param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation :param domain: Optional specification of the domain :returns: An instance of the requested protocol """ # Verify feasibility of Protocol operation = operation.lower() if operation not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation %s is not supported' % operation) if domain and domain not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'Domain %s not supported' % domain) protocol_attr = select_protocol(rse_settings, operation, scheme, domain) # Instantiate protocol comp = protocol_attr['impl'].split('.') mod = __import__('.'.join(comp[:-1])) for n in comp[1:]: try: mod = getattr(mod, n) except AttributeError: print('Protocol implementation not found') raise # TODO: provide proper rucio exception protocol = mod(protocol_attr, rse_settings) return protocol
def _get_possible_protocols(rse_settings, operation, scheme=None): """ Filter the list of available protocols or provided by the supported ones. :param rse_settings: The rse settings. :param operation: The operation (write, read). :param scheme: optional filter if no specific protocol is defined in rse_setting for the provided operation. :returns: The list of possible protocols. """ operation = operation.lower() candidates = rse_settings['protocols'] if type(rse_settings['domain']) is not list: raise exception.RSEProtocolDomainNotSupported( 'Domain setting must be list.') for d in rse_settings['domain']: if d not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'Domain %s is not supported' ' by Rucio.' % d) # convert scheme to list, if given as string if scheme and not isinstance(scheme, list): scheme = scheme.split(',') tbr = [] for protocol in candidates: # Check if scheme given and filter if so if scheme and protocol['scheme'] not in scheme: tbr.append(protocol) continue filtered = True for d in rse_settings['domain']: if protocol['domains'][d][operation] != 0: filtered = False if filtered: tbr.append(protocol) if len(candidates) <= len(tbr): raise exception.RSEProtocolNotSupported( 'No protocol for provided settings' ' found : %s.' % str(rse_settings)) return [c for c in candidates if c not in tbr]
def get_protocols_ordered(rse_settings, operation, scheme=None, domain='wan'): if operation not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation) if domain and domain not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain) candidates = _get_possible_protocols(rse_settings, operation, scheme, domain) candidates.sort(key=lambda k: k['domains'][domain][operation]) return candidates
def select_protocol(rse_settings, operation, scheme=None, domain='wan'): if operation not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation) if domain and domain not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain) candidates = _get_possible_protocols(rse_settings, operation, scheme, domain) # Shuffle candidates to load-balance over equal sources random.shuffle(candidates) return min(candidates, key=lambda k: k['domains'][domain][operation])
def select_protocol(rse_settings, operation, scheme=None): operation = operation.lower() candidates = copy.copy(rse_settings['protocols']) if type(rse_settings['domain']) is not list: raise exception.RSEProtocolDomainNotSupported('Domain setting must be list.') for d in rse_settings['domain']: if d not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported('Domain %s is not supported by Rucio.' % d) tbr = list() for protocol in candidates: # Check if scheme given and filter if so if scheme: if not isinstance(scheme, list): scheme = scheme.split(',') if protocol['scheme'] not in scheme: tbr.append(protocol) continue for d in rse_settings['domain']: if protocol['domains'][d][operation] == 0: tbr.append(protocol) break for r in tbr: candidates.remove(r) if not len(candidates): raise exception.RSEProtocolNotSupported('No protocol for provided settings found : %s.' % str(rse_settings)) # Select the one with the highest priority candidates = sorted(candidates, key=lambda k: k['scheme']) best_choice = candidates[0] candidates.remove(best_choice) domain = rse_settings['domain'][0] for p in candidates: if p['domains'][domain][operation] < best_choice['domains'][domain][operation]: best_choice = p return best_choice
def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_token=None, protocol_attr=None, logger=logging.log): """ Instanciates the protocol defined for the given operation. :param rse_settings: RSE attributes :param operation: Intended operation for this protocol :param scheme: Optional filter if no specific protocol is defined in rse_setting for the provided operation :param domain: Optional specification of the domain :param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication :param protocol_attr: Optionally passing the full protocol availability information to correctly select WAN/LAN :param logger: Optional decorated logger that can be passed from the calling daemons or servers. :returns: An instance of the requested protocol """ # Verify feasibility of Protocol operation = operation.lower() if operation not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation) if domain and domain not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain) if not protocol_attr: protocol_attr = select_protocol(rse_settings, operation, scheme, domain) else: candidates = _get_possible_protocols(rse_settings, operation, scheme, domain) if protocol_attr not in candidates: raise exception.RSEProtocolNotSupported('Protocol %s operation %s on domain %s not supported' % (protocol_attr, operation, domain)) # Instantiate protocol comp = protocol_attr['impl'].split('.') prefix = '.'.join(comp[-2:]) + ': ' logger = formatted_logger(logger, prefix + "%s") mod = __import__('.'.join(comp[:-1])) for n in comp[1:]: try: mod = getattr(mod, n) except AttributeError as e: logger(logging.DEBUG, 'Protocol implementations not supported.') raise exception.RucioException(str(e)) # TODO: provide proper rucio exception protocol_attr['auth_token'] = auth_token protocol = mod(protocol_attr, rse_settings, logger=logger) return protocol
def add_protocol(rse, parameter, session=None): """ Add a protocol to an existing RSE. If entries with equal or less priority for an operation exist, the existing one will be reorded (i.e. +1). :param rse: the name of the new rse. :param parameter: parameters of the new protocol entry. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEOperationNotSupported: If no scheme supported the requested operation for the given RSE. :raises RSEProtocolDomainNotSupported: If an undefined domain was provided. :raises RSEProtocolPriorityError: If the provided priority for the scheme is to big or below zero. :raises Duplicate: If scheme with identifier, hostname and port already exists for the given RSE. """ rid = get_rse_id(rse=rse, session=session) if not rid: raise exception.RSENotFound('RSE \'%s\' not found') # Insert new protocol entry parameter['rse_id'] = rid # Default values parameter['port'] = parameter.get('port', 0) parameter['hostname'] = parameter.get('hostname', 'localhost') # Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan] if 'domains' in parameter.keys(): for s in parameter['domains']: if s not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'The protocol domain \'%s\' is not defined in the schema.' % s) for op in parameter['domains'][s]: if op not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation \'%s\' not defined in schema.' % (op)) op_name = op if op == 'third_party_copy' else ''.join( [op, '_', s]).lower() if parameter['domains'][s][op] < 0: raise exception.RSEProtocolPriorityError( 'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (parameter['domains'][s][op], op, s)) parameter[op_name] = parameter['domains'][s][op] del parameter['domains'] if ('extended_attributes' in parameter) and parameter['extended_attributes']: try: parameter['extended_attributes'] = json.dumps( parameter['extended_attributes'], separators=(',', ':')) except ValueError: pass # String is not JSON if parameter['scheme'] == 'srm': if ('extended_attributes' not in parameter) or ('web_service_path' not in parameter['extended_attributes']): raise exception.InvalidObject( 'Missing values! For SRM, extended_attributes and web_service_path must be specified' ) try: new_protocol = models.RSEProtocols() new_protocol.update(parameter) new_protocol.save(session=session) except (IntegrityError, FlushError, OperationalError) as error: if ('UNIQUE constraint failed' in error.args[0]) or ('conflicts with persistent instance' in error.args[0]) \ or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \ or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \ or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0])\ or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]): raise exception.Duplicate( 'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (parameter['scheme'], parameter['port'], rse, parameter['hostname'])) elif 'may not be NULL' in error.args[0] \ or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \ or match('.*OperationalError.*cannot be null.*', error.args[0]): raise exception.InvalidObject('Missing values!') raise error return new_protocol
def update_protocols(rse, scheme, data, hostname, port, session=None): """ Updates an existing protocol entry for an RSE. If necessary, priorities for read, write, and delete operations of other protocol entires will be updated too. :param rse: the name of the new rse. :param scheme: Protocol identifer. :param data: Dict with new values (keys must match column names in the database). :param hostname: Hostname defined for the scheme, used if more than one scheme is registered with the same identifier. :param port: The port registered for the hostename, used if more than one scheme is regsitered with the same identifier and hostname. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE. :raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE. :raises RSEProtocolDomainNotSupported: If an undefined domain was provided. :raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero. :raises KeyNotFound: Invalid data for update provided. :raises Duplicate: If protocol with identifier, hostname and port already exists for the given RSE. """ rid = get_rse_id(rse=rse, session=session) # Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan] if 'domains' in data: for s in data['domains']: if s not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'The protocol domain \'%s\' is not defined in the schema.' % s) for op in data['domains'][s]: if op not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation \'%s\' not defined in schema.' % (op)) op_name = op if op != 'third_party_copy': op_name = ''.join([op, '_', s]) no = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= 0)).\ count() if not 0 <= data['domains'][s][op] <= no: raise exception.RSEProtocolPriorityError( 'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (data['domains'][s][op], op, s)) data[op_name] = data['domains'][s][op] del data['domains'] if 'extended_attributes' in data: try: data['extended_attributes'] = json.dumps( data['extended_attributes'], separators=(',', ':')) except ValueError: pass # String is not JSON if not rid: raise exception.RSENotFound('RSE \'%s\' not found') terms = [ models.RSEProtocols.rse_id == rid, models.RSEProtocols.scheme == scheme, models.RSEProtocols.hostname == hostname, models.RSEProtocols.port == port ] try: up = session.query(models.RSEProtocols).filter(*terms).first() if up is None: msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % ( rse, scheme, hostname, port) raise exception.RSEProtocolNotSupported(msg) # Preparing gaps if priority is updated for domain in utils.rse_supported_protocol_domains(): for op in utils.rse_supported_protocol_operations(): op_name = op if op != 'third_party_copy': op_name = ''.join([op, '_', domain]) if op_name in data: prots = [] if (not getattr(up, op_name)) and data[ op_name]: # reactivate protocol e.g. from 0 to 1 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name])).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr(up, op_name) and ( not data[op_name] ): # deactivate protocol e.g. from 1 to 0 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) elif getattr( up, op_name ) > data[op_name]: # shift forward e.g. from 5 to 2 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name], getattr(models.RSEProtocols, op_name) < getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr( up, op_name ) < data[op_name]: # shift backward e.g. from 1 to 3 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) <= data[op_name], getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) for p in prots: p.update({op_name: val}) val += 1 up.update(data, flush=True, session=session) except (IntegrityError, OperationalError) as error: if 'UNIQUE'.lower() in error.args[0].lower( ) or 'Duplicate' in error.args[ 0]: # Covers SQLite, Oracle and MySQL error raise exception.Duplicate( 'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (scheme, port, rse, hostname)) elif 'may not be NULL' in error.args[ 0] or "cannot be null" in error.args[0]: raise exception.InvalidObject('Missing values: %s' % error.args[0]) raise error except DatabaseError as error: if match( '.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*', error.args[0]): raise exception.InvalidObject('Invalid values !') raise error
def find_matching_scheme(rse_settings_dest, rse_settings_src, operation_src, operation_dest, domain=None, scheme=None): """ Find the best matching scheme between two RSEs :param rse_settings_dest: RSE settings for the destination RSE. :param rse_settings_src: RSE settings for the src RSE. :param operation_src: Source Operation such as read, write. :param operation_dest: Dest Operation such as read, write. :param domain: Domain such as lan, wan. :param scheme: List of supported schemes. :returns: Tuple of matching schemes (dest_scheme, src_scheme). """ operation_src = operation_src.lower() operation_dest = operation_dest.lower() src_candidates = copy.copy(rse_settings_src['protocols']) dest_candidates = copy.copy(rse_settings_dest['protocols']) if not domain: if type(rse_settings_dest['domain']) is not list: raise exception.RSEProtocolDomainNotSupported( 'Domain setting must be list.') domain = rse_settings_dest['domain'][0] # Clean up src_candidates tbr = list() for protocol in src_candidates: # Check if scheme given and filter if so if scheme: if not isinstance(scheme, list): scheme = scheme.split(',') if protocol['scheme'] not in scheme: tbr.append(protocol) continue if protocol['domains'].get(domain, {}).get(operation_src, 1) == 0: tbr.append(protocol) for r in tbr: src_candidates.remove(r) # Clean up dest_candidates tbr = list() for protocol in dest_candidates: # Check if scheme given and filter if so if scheme: if not isinstance(scheme, list): scheme = scheme.split(',') if protocol['scheme'] not in scheme: tbr.append(protocol) continue if protocol['domains'].get(domain, {}).get(operation_dest, 1) == 0: tbr.append(protocol) for r in tbr: dest_candidates.remove(r) if not len(src_candidates) or not len(dest_candidates): raise exception.RSEProtocolNotSupported( 'No protocol for provided settings found : %s.' % str(rse_settings_dest)) # Select the one with the highest priority dest_candidates = sorted( dest_candidates, key=lambda k: k['domains'][domain][operation_dest]) src_candidates = sorted(src_candidates, key=lambda k: k['domains'][domain][operation_src]) for dest_protocol in dest_candidates: for src_protocol in src_candidates: if __check_compatible_scheme(dest_protocol['scheme'], src_protocol['scheme']): return (dest_protocol['scheme'], src_protocol['scheme']) raise exception.RSEProtocolNotSupported( 'No protocol for provided settings found : %s.' % str(rse_settings_dest))