def validate_cms_did(obj):
"""
Special checking for DIDs
Most of the checking is done with JSON schema, but this check
makes sure user LFNs are in the correct /store/user/rucio/USERNAME namespace
"""
if not obj:
return
lfn = obj['name']
did_type = obj['type']
scope = obj['scope']
if scope.startswith('user.') and did_type == 'FILE':
_, user = scope.split('.', 1)
if not lfn.startswith('/store/user/rucio/%s/' % user):
raise InvalidObject(
"Problem with LFN %(lfn)s : Not allowed for user %(user)s" %
locals())
if lfn.startswith(
'/store/user') and not lfn.startswith('/store/user/rucio/'):
raise InvalidObject(
"Problem with LFN %(lfn)s : Legacy user files are not managed with Rucio"
)
if lfn.startswith('/store/user/rucio') and not scope.startswith('user.'):
raise InvalidObject(
"Problem with LFN %(lfn)s : Only user scopes allowed in /store/user/rucio"
)
if scope == 'logs' and did_type == 'FILE':
if not lfn.startswith('/store/logs/'):
raise InvalidObject(
"Problem with LFN %(lfn)s : Logs must start with /store/logs" %
locals())
def add_subscription(name, account, filter_, replication_rules, comments, lifetime, retroactive, dry_run, priority=None, issuer=None, vo='def'):
"""
Adds a new subscription which will be verified against every new added file and dataset
:param account: Account identifier
:type account: String
:param name: Name of the subscription
:type: String
:param filter_: Dictionary of attributes by which the input data should be filtered
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
:type filter_: Dict
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
:type replication_rules: Dict
:param comments: Comments for the subscription
:type comments: String
:param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime
:type lifetime: Integer or False
:param retroactive: Flag to know if the subscription should be applied on previous data
:type retroactive: Boolean
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
:type dry_run: Boolean
:param priority: The priority of the subscription
:type priority: Integer
:param issuer: The account issuing this operation.
:type issuer: String
:param vo: The VO to act on.
:type vo: String
:returns: subscription_id
:rtype: String
"""
if not has_permission(issuer=issuer, vo=vo, action='add_subscription', kwargs={'account': account}):
raise AccessDenied('Account %s can not add subscription' % (issuer))
try:
if filter_:
if not isinstance(filter_, dict):
raise TypeError('filter should be a dict')
validate_schema(name='subscription_filter', obj=filter_, vo=vo)
if replication_rules:
if not isinstance(replication_rules, list):
raise TypeError('replication_rules should be a list')
else:
for rule in replication_rules:
validate_schema(name='activity', obj=rule.get('activity', 'default'), vo=vo)
else:
raise InvalidObject('You must specify a rule')
except ValueError as error:
raise TypeError(error)
account = InternalAccount(account, vo=vo)
keys = ['scope', 'account']
types = [InternalScope, InternalAccount]
for _key, _type in zip(keys, types):
if _key in filter_:
if isinstance(filter_[_key], list):
filter_[_key] = [_type(val, vo=vo).internal for val in filter_[_key]]
else:
filter_[_key] = _type(filter_[_key], vo=vo).internal
return subscription.add_subscription(name=name, account=account, filter_=dumps(filter_), replication_rules=dumps(replication_rules), comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run, priority=priority)
def add_message(event_type, payload, session=None):
"""
Add a message to be submitted asynchronously to a message broker.
In the case of nolimit, a placeholder string is written to the NOT NULL payload column.
:param event_type: The type of the event as a string, e.g., NEW_DID.
:param payload: The message payload. Will be persisted as JSON.
:param session: The database session to use.
"""
services_list = REGION.get('services_list')
if services_list == NO_VALUE:
try:
services_list = get('hermes', 'services_list')
except ConfigNotFound:
services_list = None
REGION.set('services_list', services_list)
try:
payload = json.dumps(payload, cls=APIEncoder)
except TypeError as e:
raise InvalidObject('Invalid JSON for payload: %(e)s' % locals())
if len(payload) > 4000:
new_message = Message(event_type=event_type,
payload='nolimit',
payload_nolimit=payload,
services=services_list)
else:
new_message = Message(event_type=event_type,
payload=payload,
services=services_list)
new_message.save(session=session, flush=False)
def validate_schema(obj):
"""
Validate object against json schema
:param obj: The object to validate.
:raises: InvalidObject
"""
obj = json.loads(obj)
try:
if obj and 'eventType' in obj:
event_type = SCHEMAS.get(obj['eventType'].lower())
if not event_type:
validation_error = ValidationError(
message=
f"Trace schema for eventType {obj['eventType']} is not currently supported."
)
validation_error.cause = "SCHEMA_NOT_FOUND"
raise validation_error
validate(obj,
SCHEMAS.get(obj['eventType'].lower()),
format_checker=FORMAT_CHECKER)
except ValidationError as error:
if error.cause == "SCHEMA_NOT_FOUND":
LOGGER.error(error)
else:
raise InvalidObject(error)
def add_subscription(name,
account,
filter,
replication_rules,
comments,
lifetime,
retroactive,
dry_run,
priority=None,
issuer=None):
"""
Adds a new subscription which will be verified against every new added file and dataset
:param account: Account identifier
:type account: String
:param name: Name of the subscription
:type: String
:param filter: Dictionary of attributes by which the input data should be filtered
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
:type filter: Dict
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
:type replication_rules: Dict
:param comments: Comments for the subscription
:type comments: String
:param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime
:type lifetime: Integer or False
:param retroactive: Flag to know if the subscription should be applied on previous data
:type retroactive: Boolean
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
:type dry_run: Boolean
:param priority: The priority of the subscription
:type priority: Integer
:param issuer: The account issuing this operation.
:type comments: String
:returns: subscription_id
:rtype: String
"""
if not has_permission(issuer=issuer,
action='add_subscription',
kwargs={'account': account}):
raise AccessDenied('Account %s can not add subscription' % (issuer))
try:
if filter:
if type(filter) != dict:
raise TypeError('filter should be a dict')
validate_schema(name='subscription_filter', obj=filter)
if replication_rules:
if type(replication_rules) != list:
raise TypeError('replication_rules should be a list')
else:
for rule in replication_rules:
validate_schema(name='activity',
obj=rule.get('activity', 'default'))
else:
raise InvalidObject('You must specify a rule')
except ValueError, error:
raise TypeError(error)
def validate_schema(name, obj):
"""
Validate object against json schema
:param name: The json schema name.
:param obj: The object to validate.
"""
try:
if obj:
validate(obj, SCHEMAS.get(name, {}))
except ValidationError as error: # NOQA, pylint: disable=W0612
raise InvalidObject("Problem validating %(name)s : %(error)s" % locals())
def add_message(event_type, payload, session=None):
"""
Add a message to be submitted asynchronously to a message broker.
:param event_type: The type of the event as a string, e.g., NEW_DID.
:param payload: The message payload. Will be persisted as JSON.
:param session: The database session to use.
"""
try:
new_message = Message(event_type=event_type,
payload=json.dumps(payload))
except TypeError, e:
raise InvalidObject('Invalid JSON for payload: %(e)s' % locals())
def validate_name(scope, name, did_type, session=None):
"""
Validate a name according to a naming convention.
:param scope: the name for the scope.
:param name: the name.
:param did_type: the type of did.
:param session: The database session in use.
:returns: a dictionary with metadata.
"""
if scope.external.startswith('user'):
return {'project': 'user'}
elif scope.external.startswith('group'):
return {'project': 'group'}
# Check if naming convention can be found in cache region
regexp = REGION.get(scope.internal)
if regexp is NO_VALUE: # no cached entry found
regexp = get_naming_convention(scope=scope,
convention_type=KeyType.DATASET,
session=session)
regexp and REGION.set(scope.internal, regexp)
if not regexp:
return
# Validate with regexp
groups = match(regexp, str(name))
if groups:
meta = groups.groupdict()
# Hack to get task_id from version
if 'version' in meta and meta['version']:
matched = match(r'(?P<version>\w+)_tid(?P<task_id>\d+)_\w+$',
meta['version'])
if matched:
meta['version'] = matched.groupdict()['version']
meta['task_id'] = int(matched.groupdict()['task_id'])
if 'run_number' in meta and meta['run_number']:
meta['run_number'] = int(meta['run_number'])
return meta
print(
"Provided name %(name)s doesn't match the naming convention %(regexp)s"
% locals())
raise InvalidObject(
"Provided name %(name)s doesn't match the naming convention %(regexp)s"
% locals())
def validate_schema(obj):
"""
Validate object against json schema
:param obj: The object to validate.
:raises: ValidationError
"""
obj = json.loads(obj)
try:
if obj and 'eventType' in obj:
validate(obj, SCHEMAS.get(obj['eventType'].lower()))
except ValidationError as error:
raise InvalidObject(error)
def validate_cms_did(obj):
"""
Special checking for DIDs
Most of the checking is done with JSON schema, but this check
makes sure user LFNs are in the correct /store/user/rucio/USERNAME namespace
"""
if not obj:
return
lfn = obj['name']
did_type = obj['type']
scope = obj['scope']
if scope.startswith('user.') and did_type == 'FILE':
_, user = scope.split('.', 1)
if not lfn.startswith('/store/user/rucio/%s/' % user):
raise InvalidObject(
"Problem with LFN %(lfn)s : Not allowed for user %(user)s" %
locals())
def validate_schema(name, obj):
"""
Validate object against json schema
:param name: The json schema name.
:param obj: The object to validate.
"""
try:
if obj:
validate(obj, SCHEMAS.get(name, {}))
except ValidationError as error: # NOQA, pylint: disable=W0612
raise InvalidObject("Problem validating %(name)s : %(error)s" % locals())
# Apply some extra constraints to CMS DIDs
if name.lower() in ['did']:
validate_cms_did(obj)
elif name.lower() in ['dids']:
for did in obj:
validate_cms_did(did)
def validate_meta(meta, did_type, session=None):
"""
Validates metadata for a did.
:param meta: the dictionary of metadata.
:param meta: the type of the did, e.g, DATASET, CONTAINER, FILE.
:param session: The database session in use.
:returns: True
"""
# For now only validate the datatype for datasets
key = 'datatype'
if did_type == DIDType.DATASET and key in meta:
try:
session.query(models.DIDKeyValueAssociation.value).\
filter_by(key=key).\
filter_by(value=meta[key]).\
one()
except NoResultFound:
print("The value '%s' for the key '%s' is not valid" % (meta[key], key))
raise InvalidObject("The value '%s' for the key '%s' is not valid" % (meta[key], key))
def add_message(event_type, payload, session=None):
"""
Add a message to be submitted asynchronously to a message broker.
In the case of nolimit, a placeholder string is written to the NOT NULL payload column.
:param event_type: The type of the event as a string, e.g., NEW_DID.
:param payload: The message payload. Will be persisted as JSON.
:param session: The database session to use.
"""
try:
payload = json.dumps(payload)
except TypeError as e:
raise InvalidObject('Invalid JSON for payload: %(e)s' % locals())
if len(payload) > 4000:
new_message = Message(event_type=event_type, payload='nolimit', payload_nolimit=payload)
else:
new_message = Message(event_type=event_type, payload=payload)
new_message.save(session=session, flush=False)
