def validate_cms_did(obj): """ Special checking for DIDs Most of the checking is done with JSON schema, but this check makes sure user LFNs are in the correct /store/user/rucio/USERNAME namespace """ if not obj: return lfn = obj['name'] did_type = obj['type'] scope = obj['scope'] if scope.startswith('user.') and did_type == 'FILE': _, user = scope.split('.', 1) if not lfn.startswith('/store/user/rucio/%s/' % user): raise InvalidObject( "Problem with LFN %(lfn)s : Not allowed for user %(user)s" % locals()) if lfn.startswith( '/store/user') and not lfn.startswith('/store/user/rucio/'): raise InvalidObject( "Problem with LFN %(lfn)s : Legacy user files are not managed with Rucio" ) if lfn.startswith('/store/user/rucio') and not scope.startswith('user.'): raise InvalidObject( "Problem with LFN %(lfn)s : Only user scopes allowed in /store/user/rucio" ) if scope == 'logs' and did_type == 'FILE': if not lfn.startswith('/store/logs/'): raise InvalidObject( "Problem with LFN %(lfn)s : Logs must start with /store/logs" % locals())
def add_subscription(name, account, filter_, replication_rules, comments, lifetime, retroactive, dry_run, priority=None, issuer=None, vo='def'): """ Adds a new subscription which will be verified against every new added file and dataset :param account: Account identifier :type account: String :param name: Name of the subscription :type: String :param filter_: Dictionary of attributes by which the input data should be filtered **Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}`` :type filter_: Dict :param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression :type replication_rules: Dict :param comments: Comments for the subscription :type comments: String :param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime :type lifetime: Integer or False :param retroactive: Flag to know if the subscription should be applied on previous data :type retroactive: Boolean :param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set) :type dry_run: Boolean :param priority: The priority of the subscription :type priority: Integer :param issuer: The account issuing this operation. :type issuer: String :param vo: The VO to act on. :type vo: String :returns: subscription_id :rtype: String """ if not has_permission(issuer=issuer, vo=vo, action='add_subscription', kwargs={'account': account}): raise AccessDenied('Account %s can not add subscription' % (issuer)) try: if filter_: if not isinstance(filter_, dict): raise TypeError('filter should be a dict') validate_schema(name='subscription_filter', obj=filter_, vo=vo) if replication_rules: if not isinstance(replication_rules, list): raise TypeError('replication_rules should be a list') else: for rule in replication_rules: validate_schema(name='activity', obj=rule.get('activity', 'default'), vo=vo) else: raise InvalidObject('You must specify a rule') except ValueError as error: raise TypeError(error) account = InternalAccount(account, vo=vo) keys = ['scope', 'account'] types = [InternalScope, InternalAccount] for _key, _type in zip(keys, types): if _key in filter_: if isinstance(filter_[_key], list): filter_[_key] = [_type(val, vo=vo).internal for val in filter_[_key]] else: filter_[_key] = _type(filter_[_key], vo=vo).internal return subscription.add_subscription(name=name, account=account, filter_=dumps(filter_), replication_rules=dumps(replication_rules), comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run, priority=priority)
def add_message(event_type, payload, session=None): """ Add a message to be submitted asynchronously to a message broker. In the case of nolimit, a placeholder string is written to the NOT NULL payload column. :param event_type: The type of the event as a string, e.g., NEW_DID. :param payload: The message payload. Will be persisted as JSON. :param session: The database session to use. """ services_list = REGION.get('services_list') if services_list == NO_VALUE: try: services_list = get('hermes', 'services_list') except ConfigNotFound: services_list = None REGION.set('services_list', services_list) try: payload = json.dumps(payload, cls=APIEncoder) except TypeError as e: raise InvalidObject('Invalid JSON for payload: %(e)s' % locals()) if len(payload) > 4000: new_message = Message(event_type=event_type, payload='nolimit', payload_nolimit=payload, services=services_list) else: new_message = Message(event_type=event_type, payload=payload, services=services_list) new_message.save(session=session, flush=False)
def validate_schema(obj): """ Validate object against json schema :param obj: The object to validate. :raises: InvalidObject """ obj = json.loads(obj) try: if obj and 'eventType' in obj: event_type = SCHEMAS.get(obj['eventType'].lower()) if not event_type: validation_error = ValidationError( message= f"Trace schema for eventType {obj['eventType']} is not currently supported." ) validation_error.cause = "SCHEMA_NOT_FOUND" raise validation_error validate(obj, SCHEMAS.get(obj['eventType'].lower()), format_checker=FORMAT_CHECKER) except ValidationError as error: if error.cause == "SCHEMA_NOT_FOUND": LOGGER.error(error) else: raise InvalidObject(error)
def add_subscription(name, account, filter, replication_rules, comments, lifetime, retroactive, dry_run, priority=None, issuer=None): """ Adds a new subscription which will be verified against every new added file and dataset :param account: Account identifier :type account: String :param name: Name of the subscription :type: String :param filter: Dictionary of attributes by which the input data should be filtered **Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}`` :type filter: Dict :param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression :type replication_rules: Dict :param comments: Comments for the subscription :type comments: String :param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime :type lifetime: Integer or False :param retroactive: Flag to know if the subscription should be applied on previous data :type retroactive: Boolean :param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set) :type dry_run: Boolean :param priority: The priority of the subscription :type priority: Integer :param issuer: The account issuing this operation. :type comments: String :returns: subscription_id :rtype: String """ if not has_permission(issuer=issuer, action='add_subscription', kwargs={'account': account}): raise AccessDenied('Account %s can not add subscription' % (issuer)) try: if filter: if type(filter) != dict: raise TypeError('filter should be a dict') validate_schema(name='subscription_filter', obj=filter) if replication_rules: if type(replication_rules) != list: raise TypeError('replication_rules should be a list') else: for rule in replication_rules: validate_schema(name='activity', obj=rule.get('activity', 'default')) else: raise InvalidObject('You must specify a rule') except ValueError, error: raise TypeError(error)
def validate_schema(name, obj): """ Validate object against json schema :param name: The json schema name. :param obj: The object to validate. """ try: if obj: validate(obj, SCHEMAS.get(name, {})) except ValidationError as error: # NOQA, pylint: disable=W0612 raise InvalidObject("Problem validating %(name)s : %(error)s" % locals())
def add_message(event_type, payload, session=None): """ Add a message to be submitted asynchronously to a message broker. :param event_type: The type of the event as a string, e.g., NEW_DID. :param payload: The message payload. Will be persisted as JSON. :param session: The database session to use. """ try: new_message = Message(event_type=event_type, payload=json.dumps(payload)) except TypeError, e: raise InvalidObject('Invalid JSON for payload: %(e)s' % locals())
def validate_name(scope, name, did_type, session=None): """ Validate a name according to a naming convention. :param scope: the name for the scope. :param name: the name. :param did_type: the type of did. :param session: The database session in use. :returns: a dictionary with metadata. """ if scope.external.startswith('user'): return {'project': 'user'} elif scope.external.startswith('group'): return {'project': 'group'} # Check if naming convention can be found in cache region regexp = REGION.get(scope.internal) if regexp is NO_VALUE: # no cached entry found regexp = get_naming_convention(scope=scope, convention_type=KeyType.DATASET, session=session) regexp and REGION.set(scope.internal, regexp) if not regexp: return # Validate with regexp groups = match(regexp, str(name)) if groups: meta = groups.groupdict() # Hack to get task_id from version if 'version' in meta and meta['version']: matched = match(r'(?P<version>\w+)_tid(?P<task_id>\d+)_\w+$', meta['version']) if matched: meta['version'] = matched.groupdict()['version'] meta['task_id'] = int(matched.groupdict()['task_id']) if 'run_number' in meta and meta['run_number']: meta['run_number'] = int(meta['run_number']) return meta print( "Provided name %(name)s doesn't match the naming convention %(regexp)s" % locals()) raise InvalidObject( "Provided name %(name)s doesn't match the naming convention %(regexp)s" % locals())
def validate_schema(obj): """ Validate object against json schema :param obj: The object to validate. :raises: ValidationError """ obj = json.loads(obj) try: if obj and 'eventType' in obj: validate(obj, SCHEMAS.get(obj['eventType'].lower())) except ValidationError as error: raise InvalidObject(error)
def validate_cms_did(obj): """ Special checking for DIDs Most of the checking is done with JSON schema, but this check makes sure user LFNs are in the correct /store/user/rucio/USERNAME namespace """ if not obj: return lfn = obj['name'] did_type = obj['type'] scope = obj['scope'] if scope.startswith('user.') and did_type == 'FILE': _, user = scope.split('.', 1) if not lfn.startswith('/store/user/rucio/%s/' % user): raise InvalidObject( "Problem with LFN %(lfn)s : Not allowed for user %(user)s" % locals())
def validate_schema(name, obj): """ Validate object against json schema :param name: The json schema name. :param obj: The object to validate. """ try: if obj: validate(obj, SCHEMAS.get(name, {})) except ValidationError as error: # NOQA, pylint: disable=W0612 raise InvalidObject("Problem validating %(name)s : %(error)s" % locals()) # Apply some extra constraints to CMS DIDs if name.lower() in ['did']: validate_cms_did(obj) elif name.lower() in ['dids']: for did in obj: validate_cms_did(did)
def validate_meta(meta, did_type, session=None): """ Validates metadata for a did. :param meta: the dictionary of metadata. :param meta: the type of the did, e.g, DATASET, CONTAINER, FILE. :param session: The database session in use. :returns: True """ # For now only validate the datatype for datasets key = 'datatype' if did_type == DIDType.DATASET and key in meta: try: session.query(models.DIDKeyValueAssociation.value).\ filter_by(key=key).\ filter_by(value=meta[key]).\ one() except NoResultFound: print("The value '%s' for the key '%s' is not valid" % (meta[key], key)) raise InvalidObject("The value '%s' for the key '%s' is not valid" % (meta[key], key))
def add_message(event_type, payload, session=None): """ Add a message to be submitted asynchronously to a message broker. In the case of nolimit, a placeholder string is written to the NOT NULL payload column. :param event_type: The type of the event as a string, e.g., NEW_DID. :param payload: The message payload. Will be persisted as JSON. :param session: The database session to use. """ try: payload = json.dumps(payload) except TypeError as e: raise InvalidObject('Invalid JSON for payload: %(e)s' % locals()) if len(payload) > 4000: new_message = Message(event_type=event_type, payload='nolimit', payload_nolimit=payload) else: new_message = Message(event_type=event_type, payload=payload) new_message.save(session=session, flush=False)