def add_account_identity(identity, type, account, email, default=False, password=None, session=None): """ Adds a membership association between identity and account. :param identity: The identity key name. For example x509 DN, or a username. :param type: The type of the authentication (x509, gss, userpass, ssh, saml, oidc). :param account: The account name. :param email: The Email address associated with the identity. :param default: If True, the account should be used by default with the provided identity. :param password: Password if type is userpass. :param session: The database session in use. """ if not account_exists(account, session=session): raise exception.AccountNotFound('Account \'%s\' does not exist.' % account) id = session.query(models.Identity).filter_by(identity=identity, identity_type=type).first() if id is None: add_identity(identity=identity, type=type, email=email, password=password, session=session) id = session.query(models.Identity).filter_by(identity=identity, identity_type=type).first() iaa = models.IdentityAccountAssociation(identity=id.identity, identity_type=id.identity_type, account=account) try: iaa.save(session=session) except IntegrityError: raise exception.Duplicate('Identity pair \'%s\',\'%s\' already exists!' % (identity, type))
def add_account(account, type, email, session=None): """ Add an account with the given account name and type. :param account: the name of the new account. :param type: the type of the new account. :param email: The Email address associated with the account. :param session: the database session in use. """ vo = account.vo if not vo_exists(vo=vo, session=session): raise exception.VONotFound('VO {} not found'.format(vo)) # Reserve the name 'super_root' for multi_vo admins if account.external == 'super_root': if not (vo == 'def' and config_get_bool( 'common', 'multi_vo', raise_exception=False, default=False)): raise exception.UnsupportedAccountName( 'The name "%s" cannot be used.' % account.external) new_account = models.Account(account=account, account_type=type, email=email, status=AccountStatus.ACTIVE) try: new_account.save(session=session) except IntegrityError: raise exception.Duplicate('Account ID \'%s\' already exists!' % account) # Create the account counters for this account rucio.core.account_counter.create_counters_for_new_account(account=account, session=session)
def add_identity(identity, type, email, password=None, session=None): """ Creates a user identity. :param identity: The identity key name. For example x509 DN, or a username. :param type: The type of the authentication (x509, gss, userpass, ssh, saml, oidc) :param email: The Email address associated with the identity. :param password: If type==userpass, this sets the password. :param session: The database session in use. """ if type == IdentityType.USERPASS and password is None: raise exception.IdentityError('You must provide a password!') new_id = models.Identity() new_id.update({'identity': identity, 'identity_type': type, 'email': email}) if type == IdentityType.USERPASS and password is not None: salt = os.urandom(255) # make sure the salt has the length of the hash if six.PY3: decoded_salt = b64encode(salt).decode() salted_password = ('%s%s' % (decoded_salt, password)).encode() else: salted_password = '******' % (salt, str(password)) password = hashlib.sha256(salted_password).hexdigest() # hash it new_id.update({'salt': salt, 'password': password, 'email': email}) try: new_id.save(session=session) except IntegrityError as e: if match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', e.args[0]): raise exception.Duplicate('Identity pair \'%s\',\'%s\' already exists!' % (identity, type)) raise exception.DatabaseException(str(e))
def add_distance(source, destination, issuer, vo='def', ranking=None, distance=None, geoip_distance=None, active=None, submitted=None, finished=None, failed=None, transfer_speed=None): """ Add a src-dest distance. :param source: The source. :param destination: The destination. :param issuer: The issuer account. :param vo: The VO to act on. :param ranking: Ranking as an integer. :param distance: Distance as an integer. :param geoip_distance: GEOIP Distance as an integer. :param active: Active FTS transfers as an integer. :param submitted: Submitted FTS transfers as an integer. :param finished: Finished FTS transfers as an integer. :param failed: Failed FTS transfers as an integer. :param transfer_speed: FTS transfer speed as an integer. """ kwargs = {'source': source, 'destination': destination} if not permission.has_permission(issuer=issuer, vo=vo, action='add_distance', kwargs=kwargs): raise exception.AccessDenied('Account %s can not add RSE distances' % (issuer)) try: return distance_module.add_distance(src_rse_id=rse_module.get_rse_id(source, vo=vo), dest_rse_id=rse_module.get_rse_id(destination, vo=vo), ranking=ranking, agis_distance=distance, geoip_distance=geoip_distance, active=active, submitted=submitted, finished=finished, failed=failed, transfer_speed=transfer_speed) except exception.Duplicate: # use source and destination RSE names raise exception.Duplicate('Distance from %s to %s already exists!' % (source, destination))
def add_account_attribute(account, key, value, session=None): """ Add an attribute for the given account name. :param key: the key for the new attribute. :param value: the value for the new attribute. :param account: the account to add the attribute to. :param session: The database session in use. """ query = session.query(models.Account).filter_by( account=account, status=AccountStatus.ACTIVE) try: query.one() except exc.NoResultFound: raise exception.AccountNotFound( "Account ID '{0}' does not exist".format(account)) new_attr = models.AccountAttrAssociation(account=account, key=key, value=value) try: new_attr.save(session=session) except IntegrityError as error: if match('.*IntegrityError.*ORA-00001: unique constraint.*ACCOUNT_ATTR_MAP_PK.*violated.*', error.args[0]) \ or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \ or match('.*IntegrityError.*UNIQUE constraint failed: account_attr_map.account, account_attr_map.key.*', error.args[0]) \ or match('.*IntegrityError.*columns? account.*key.*not unique.*', error.args[0]) \ or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \ or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]): raise exception.Duplicate( 'Key {0} already exist for account {1}!'.format(key, account)) except Exception: raise exception.RucioException(str(format_exc()))
def add_distance(src_rse_id, dest_rse_id, ranking=None, agis_distance=None, geoip_distance=None, active=None, submitted=None, finished=None, failed=None, transfer_speed=None, session=None): """ Add a src-dest distance. :param src_rse_id: The source RSE ID. :param dest_rse_id: The destination RSE ID. :param ranking: Ranking as an integer. :param agis_distance: AGIS Distance as an integer. :param geoip_distance: GEOIP Distance as an integer. :param active: Active FTS transfers as an integer. :param submitted: Submitted FTS transfers as an integer. :param finished: Finished FTS transfers as an integer. :param failed: Failed FTS transfers as an integer. :param transfer_speed: FTS transfer speed as an integer. :param session: The database session to use. """ try: new_distance = Distance(src_rse_id=src_rse_id, dest_rse_id=dest_rse_id, ranking=ranking, agis_distance=agis_distance, geoip_distance=geoip_distance, active=active, submitted=submitted, finished=finished, failed=failed, transfer_speed=transfer_speed) new_distance.save(session=session) except IntegrityError: raise exception.Duplicate('Distance from %s to %s already exists!' % (src_rse_id, dest_rse_id)) except DatabaseError, e: raise exception.RucioException(e.args)
def add_account_identity(identity, type_, account, email, default=False, password=None, session=None): """ Adds a membership association between identity and account. :param identity: The identity key name. For example x509 DN, or a username. :param type_: The type of the authentication (x509, gss, userpass, ssh, saml, oidc). :param account: The account name. :param email: The Email address associated with the identity. :param default: If True, the account should be used by default with the provided identity. :param password: Password if type is userpass. :param session: The database session in use. """ if not account_exists(account, session=session): raise exception.AccountNotFound('Account \'%s\' does not exist.' % account) id_ = session.query(models.Identity).filter_by(identity=identity, identity_type=type_).first() if id_ is None: add_identity(identity=identity, type_=type_, email=email, password=password, session=session) id_ = session.query(models.Identity).filter_by(identity=identity, identity_type=type_).first() iaa = models.IdentityAccountAssociation(identity=id_.identity, identity_type=id_.identity_type, account=account, is_default=default) try: iaa.save(session=session) except IntegrityError as error: if match('.*IntegrityError.*ORA-00001: unique constraint.*violated.*', error.args[0]) \ or match('.*IntegrityError.*UNIQUE constraint failed.*', error.args[0]) \ or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \ or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \ or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]) \ or match('.*IntegrityError.*columns? .*not unique.*', error.args[0]): raise exception.Duplicate('Identity pair \'%s\',\'%s\' already exists!' % (identity, type_))
def add_rse(rse, deterministic=True, volatile=False, city=None, region_code=None, country_name=None, continent=None, time_zone=None, ISP=None, staging_area=False, session=None): """ Add a rse with the given location name. :param rse: the name of the new rse. :param deterministic: Boolean to know if the pfn is generated deterministically. :param volatile: Boolean for RSE cache. :param city: City for the RSE. :param region_code: The region code for the RSE. :param country_name: The country. :param continent: The continent. :param time_zone: Timezone. :param ISP: Internet service provider. :param staging_area: Staging area. :param session: The database session in use. """ new_rse = models.RSE(rse=rse, deterministic=deterministic, volatile=volatile, city=city, region_code=region_code, country_name=country_name, continent=continent, time_zone=time_zone, staging_area=staging_area, ISP=ISP, availability=7) try: new_rse.save(session=session) except IntegrityError: raise exception.Duplicate('RSE \'%(rse)s\' already exists!' % locals()) except DatabaseError as error: raise exception.RucioException(error.args) # Add rse name as a RSE-Tag add_rse_attribute(rse=rse, key=rse, value=True, session=session) # Add counter to monitor the space usage add_counter(rse_id=new_rse.id, session=session) # Add account counter rucio.core.account_counter.create_counters_for_new_rse(rse_id=new_rse.id, session=session) return new_rse.id
def add_vo(vo, description, email, session=None): """ Add a VO and setup a new root user. New root user will have account name 'root' and a userpass identity with username: '******' and password: '******' :param vo: 3-letter unique tag for a VO. :param descrition: Descriptive string for the VO (e.g. Full name). :param email: Contact email for the VO. :param session: The db session in use. """ if not config_get_bool( 'common', 'multi_vo', raise_exception=False, default=False): raise exception.UnsupportedOperation( 'VO operations cannot be performed in single VO mode.') if len(vo) != 3: raise exception.RucioException('Invalid VO tag, must be 3 chars.') new_vo = models.VO(vo=vo, description=description, email=email) try: new_vo.save(session=session) except IntegrityError: raise exception.Duplicate('VO {} already exists!'.format(vo)) except DatabaseError as error: raise exception.RucioException(error.args) from rucio.core.account import add_account, list_identities from rucio.core.identity import add_account_identity new_root = InternalAccount('root', vo=vo) add_account(account=new_root, type_=AccountType['SERVICE'], email=email, session=session) add_account_identity(identity='root@{}'.format(vo), type_=IdentityType['USERPASS'], account=new_root, email=email, default=False, password='******', session=session) for ident in list_identities(account=InternalAccount('super_root', vo='def'), session=session): add_account_identity(identity=ident['identity'], type_=ident['type'], account=new_root, email='', session=session)
def add_account(account, type, email, session=None): """ Add an account with the given account name and type. :param account: the name of the new account. :param type: the type of the new account. :param email: The Email address associated with the account. :param session: the database session in use. """ new_account = models.Account(account=account, account_type=type, email=email, status=AccountStatus.ACTIVE) try: new_account.save(session=session) except IntegrityError: raise exception.Duplicate('Account ID \'%s\' already exists!' % account) # Create the account counters for this account rucio.core.account_counter.create_counters_for_new_account(account=account, session=session)
def add_vo(vo, description, password, email, session=None): """ Add a VO and setup a new root user. New root user will have account name 'root' and a userpass identity with username: '******' and password from the rootpass parameter :param vo: 3-letter unique tag for a VO. :param descrition: Descriptive string for the VO (e.g. Full name). :param email: Contact email for the VO. :param password: The password to set for the root user of the new VO :param session: The db session in use. """ if len(vo) != 3: raise exception.RucioException('Invalid VO tag, must be 3 chars.') new_vo = models.VO(vo=vo, description=description, email=email) try: new_vo.save(session=session) except IntegrityError: raise exception.Duplicate('VO {} already exists!'.format(vo)) except DatabaseError as error: raise exception.RucioException(error.args) from rucio.core.account import add_account, list_identities from rucio.core.identity import add_account_identity new_root = InternalAccount('root', vo=vo) add_account(account=new_root, type=AccountType.from_sym('SERVICE'), email=email, session=session) add_account_identity(identity='root@{}'.format(vo), type=IdentityType.from_sym('userpass'), account=new_root, email=email, default=False, password=password, session=session) for ident in list_identities(account=InternalAccount('super_root', vo='def'), session=session): add_account_identity(identity=ident['identity'], type=ident['type'], account=new_root, email='', session=session)
def add_rse_attribute(rse_id, key, value, session=None): """ Adds a RSE attribute. :param rse_id: the rse id. :param key: the key name. :param value: the value name. :param issuer: The issuer account. :param session: The database session in use. :returns: True is successful """ try: new_rse_attr = models.RSEAttrAssociation(rse_id=rse_id, key=key, value=value) new_rse_attr = session.merge(new_rse_attr) new_rse_attr.save(session=session) except IntegrityError: rse = get_rse_name(rse_id=rse_id, session=session) raise exception.Duplicate("RSE attribute '%(key)s-%(value)s\' for RSE '%(rse)s' already exists!" % locals()) return True
def add_protocol(rse, parameter, session=None): """ Add a protocol to an existing RSE. If entries with equal or less priority for an operation exist, the existing one will be reorded (i.e. +1). :param rse: the name of the new rse. :param parameter: parameters of the new protocol entry. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEOperationNotSupported: If no scheme supported the requested operation for the given RSE. :raises RSEProtocolDomainNotSupported: If an undefined domain was provided. :raises RSEProtocolPriorityError: If the provided priority for the scheme is to big or below zero. :raises Duplicate: If scheme with identifier, hostname and port already exists for the given RSE. """ rid = get_rse_id(rse=rse, session=session) if not rid: raise exception.RSENotFound('RSE \'%s\' not found') # Insert new protocol entry parameter['rse_id'] = rid # Default values parameter['port'] = parameter.get('port', 0) parameter['hostname'] = parameter.get('hostname', 'localhost') # Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan] if 'domains' in parameter.keys(): for s in parameter['domains']: if s not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'The protocol domain \'%s\' is not defined in the schema.' % s) for op in parameter['domains'][s]: if op not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation \'%s\' not defined in schema.' % (op)) op_name = op if op == 'third_party_copy' else ''.join( [op, '_', s]).lower() if parameter['domains'][s][op] < 0: raise exception.RSEProtocolPriorityError( 'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (parameter['domains'][s][op], op, s)) parameter[op_name] = parameter['domains'][s][op] del parameter['domains'] if ('extended_attributes' in parameter) and parameter['extended_attributes']: try: parameter['extended_attributes'] = json.dumps( parameter['extended_attributes'], separators=(',', ':')) except ValueError: pass # String is not JSON if parameter['scheme'] == 'srm': if ('extended_attributes' not in parameter) or ('web_service_path' not in parameter['extended_attributes']): raise exception.InvalidObject( 'Missing values! For SRM, extended_attributes and web_service_path must be specified' ) try: new_protocol = models.RSEProtocols() new_protocol.update(parameter) new_protocol.save(session=session) except (IntegrityError, FlushError, OperationalError) as error: if ('UNIQUE constraint failed' in error.args[0]) or ('conflicts with persistent instance' in error.args[0]) \ or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \ or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \ or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0])\ or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]): raise exception.Duplicate( 'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (parameter['scheme'], parameter['port'], rse, parameter['hostname'])) elif 'may not be NULL' in error.args[0] \ or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \ or match('.*OperationalError.*cannot be null.*', error.args[0]): raise exception.InvalidObject('Missing values!') raise error return new_protocol
def update_protocols(rse, scheme, data, hostname, port, session=None): """ Updates an existing protocol entry for an RSE. If necessary, priorities for read, write, and delete operations of other protocol entires will be updated too. :param rse: the name of the new rse. :param scheme: Protocol identifer. :param data: Dict with new values (keys must match column names in the database). :param hostname: Hostname defined for the scheme, used if more than one scheme is registered with the same identifier. :param port: The port registered for the hostename, used if more than one scheme is regsitered with the same identifier and hostname. :param session: The database session in use. :raises RSENotFound: If RSE is not found. :raises RSEProtocolNotSupported: If no macthing protocol was found for the given RSE. :raises RSEOperationNotSupported: If no protocol supported the requested operation for the given RSE. :raises RSEProtocolDomainNotSupported: If an undefined domain was provided. :raises RSEProtocolPriorityError: If the provided priority for the protocol is to big or below zero. :raises KeyNotFound: Invalid data for update provided. :raises Duplicate: If protocol with identifier, hostname and port already exists for the given RSE. """ rid = get_rse_id(rse=rse, session=session) # Transform nested domains to match DB schema e.g. [domains][lan][read] => [read_lan] if 'domains' in data: for s in data['domains']: if s not in utils.rse_supported_protocol_domains(): raise exception.RSEProtocolDomainNotSupported( 'The protocol domain \'%s\' is not defined in the schema.' % s) for op in data['domains'][s]: if op not in utils.rse_supported_protocol_operations(): raise exception.RSEOperationNotSupported( 'Operation \'%s\' not defined in schema.' % (op)) op_name = op if op != 'third_party_copy': op_name = ''.join([op, '_', s]) no = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= 0)).\ count() if not 0 <= data['domains'][s][op] <= no: raise exception.RSEProtocolPriorityError( 'The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (data['domains'][s][op], op, s)) data[op_name] = data['domains'][s][op] del data['domains'] if 'extended_attributes' in data: try: data['extended_attributes'] = json.dumps( data['extended_attributes'], separators=(',', ':')) except ValueError: pass # String is not JSON if not rid: raise exception.RSENotFound('RSE \'%s\' not found') terms = [ models.RSEProtocols.rse_id == rid, models.RSEProtocols.scheme == scheme, models.RSEProtocols.hostname == hostname, models.RSEProtocols.port == port ] try: up = session.query(models.RSEProtocols).filter(*terms).first() if up is None: msg = 'RSE \'%s\' does not support protocol \'%s\' for hostname \'%s\' on port \'%s\'' % ( rse, scheme, hostname, port) raise exception.RSEProtocolNotSupported(msg) # Preparing gaps if priority is updated for domain in utils.rse_supported_protocol_domains(): for op in utils.rse_supported_protocol_operations(): op_name = op if op != 'third_party_copy': op_name = ''.join([op, '_', domain]) if op_name in data: prots = [] if (not getattr(up, op_name)) and data[ op_name]: # reactivate protocol e.g. from 0 to 1 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name])).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr(up, op_name) and ( not data[op_name] ): # deactivate protocol e.g. from 1 to 0 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) elif getattr( up, op_name ) > data[op_name]: # shift forward e.g. from 5 to 2 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) >= data[op_name], getattr(models.RSEProtocols, op_name) < getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = data[op_name] + 1 elif getattr( up, op_name ) < data[op_name]: # shift backward e.g. from 1 to 3 prots = session.query(models.RSEProtocols).\ filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid, getattr(models.RSEProtocols, op_name) <= data[op_name], getattr(models.RSEProtocols, op_name) > getattr(up, op_name))).\ order_by(getattr(models.RSEProtocols, op_name).asc()) val = getattr(up, op_name) for p in prots: p.update({op_name: val}) val += 1 up.update(data, flush=True, session=session) except (IntegrityError, OperationalError) as error: if 'UNIQUE'.lower() in error.args[0].lower( ) or 'Duplicate' in error.args[ 0]: # Covers SQLite, Oracle and MySQL error raise exception.Duplicate( 'Protocol \'%s\' on port %s already registered for \'%s\' with hostname \'%s\'.' % (scheme, port, rse, hostname)) elif 'may not be NULL' in error.args[ 0] or "cannot be null" in error.args[0]: raise exception.InvalidObject('Missing values: %s' % error.args[0]) raise error except DatabaseError as error: if match( '.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*', error.args[0]): raise exception.InvalidObject('Invalid values !') raise error
def add_rse(rse, deterministic=True, volatile=False, city=None, region_code=None, country_name=None, continent=None, time_zone=None, ISP=None, staging_area=False, rse_type=RSEType.DISK, longitude=None, latitude=None, ASN=None, availability=7, session=None, supported_chksums=None): """ Add a rse with the given location name. :param rse: the name of the new rse. :param deterministic: Boolean to know if the pfn is generated deterministically. :param volatile: Boolean for RSE cache. :param city: City for the RSE. :param region_code: The region code for the RSE. :param country_name: The country. :param continent: The continent. :param time_zone: Timezone. :param ISP: Internet service provider. :param staging_area: Staging area. :param rse_type: RSE type. :param latitude: Latitude coordinate of RSE. :param longitude: Longitude coordinate of RSE. :param ASN: Access service network. :param availability: Availability. :param session: The database session in use. :param supported_chksums: The checksums supported by the RSE. """ if isinstance(rse_type, str) or isinstance(rse_type, unicode): rse_type = RSEType.from_string(str(rse_type)) new_rse = models.RSE(rse=rse, deterministic=deterministic, volatile=volatile, city=city, region_code=region_code, country_name=country_name, continent=continent, time_zone=time_zone, staging_area=staging_area, ISP=ISP, availability=availability, rse_type=rse_type, longitude=longitude, latitude=latitude, ASN=ASN) try: new_rse.save(session=session) except IntegrityError: raise exception.Duplicate('RSE \'%(rse)s\' already exists!' % locals()) except DatabaseError as error: raise exception.RucioException(error.args) # Add rse name as a RSE-Tag add_rse_attribute(rse=rse, key=rse, value=True, session=session) # Add supported checksums in rse_attr_map if specified if supported_chksums: for chksum in supported_chksums: add_rse_checksum(rse=rse, chksum_name=chksum, session=session) # Add counter to monitor the space usage add_counter(rse_id=new_rse.id, session=session) # Add account counter rucio.core.account_counter.create_counters_for_new_rse(rse_id=new_rse.id, session=session) return new_rse.id