def add_files(lfns, issuer, ignore_availability):
"""
Bulk add files :
- Create the file and replica.
- If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites.
- Create all the ascendants of the dataset if they do not exist
:param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
:param issuer: The issuer account.
:param ignore_availability: A boolean to ignore blocked sites.
"""
scopes = list_scopes()
dids = []
rses = {}
for lfn in lfns:
scope, name = extract_scope(lfn['lfn'], scopes)
dids.append({'scope': scope, 'name': name})
rse = lfn['rse']
if rse not in rses:
rse_id = get_rse_id(rse=rse)
rses[rse] = rse_id
lfn['rse_id'] = rses[rse]
# Check if the issuer can add dids and use skip_availabitlity
for rse in rses:
rse_id = rses[rse]
kwargs = {'rse': rse, 'rse_id': rse_id}
if not has_permission(
issuer=issuer, action='add_replicas', kwargs=kwargs):
raise AccessDenied('Account %s can not add file replicas on %s' %
(issuer, rse))
if not has_permission(issuer=issuer,
action='skip_availability_check',
kwargs=kwargs):
ignore_availability = False
# Check if the issuer can add the files
kwargs = {'issuer': issuer, 'dids': dids}
if not has_permission(issuer=issuer, action='add_dids', kwargs=kwargs):
raise AccessDenied('Account %s can not bulk add data identifier' %
(issuer))
dirac.add_files(lfns=lfns,
account=issuer,
ignore_availability=ignore_availability,
session=None)
def move_replication_rule(rule_id, rse_expression, issuer, vo='def'):
"""
Move a replication rule to another RSE and, once done, delete the original one.
:param rule_id: Rule to be moved.
:param rse_expression: RSE expression of the new rule.
:param session: The DB Session.
:param vo: The VO to act on.
:raises: RuleNotFound, RuleReplaceFailed
"""
kwargs = {'rule_id': rule_id, 'rse_expression': rse_expression}
if not has_permission(issuer=issuer, vo=vo, action='access_rule_vo', kwargs=kwargs):
raise AccessDenied('Account %s can not access rules at other VOs.' % (issuer))
if not has_permission(issuer=issuer, vo=vo, action='move_rule', kwargs=kwargs):
raise AccessDenied('Account %s can not move this replication rule.' % (issuer))
return rule.move_rule(rule_id=rule_id, rse_expression=rse_expression)
def add_subscription(name,
account,
filter,
replication_rules,
comments,
lifetime,
retroactive,
dry_run,
priority=None,
issuer=None):
"""
Adds a new subscription which will be verified against every new added file and dataset
:param account: Account identifier
:type account: String
:param name: Name of the subscription
:type: String
:param filter: Dictionary of attributes by which the input data should be filtered
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
:type filter: Dict
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
:type replication_rules: Dict
:param comments: Comments for the subscription
:type comments: String
:param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime
:type lifetime: Integer or False
:param retroactive: Flag to know if the subscription should be applied on previous data
:type retroactive: Boolean
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
:type dry_run: Boolean
:param priority: The priority of the subscription
:type priority: Integer
:param issuer: The account issuing this operation.
:type comments: String
:returns: subscription_id
:rtype: String
"""
if not has_permission(issuer=issuer,
action='add_subscription',
kwargs={'account': account}):
raise AccessDenied('Account %s can not add subscription' % (issuer))
try:
if filter:
if type(filter) != dict:
raise TypeError('filter should be a dict')
validate_schema(name='subscription_filter', obj=filter)
if replication_rules:
if type(replication_rules) != list:
raise TypeError('replication_rules should be a list')
else:
for rule in replication_rules:
validate_schema(name='activity',
obj=rule.get('activity', 'default'))
else:
raise InvalidObject('You must specify a rule')
except ValueError, error:
raise TypeError(error)
def reduce_replication_rule(rule_id, copies, exclude_expression, issuer, vo='def'):
"""
Reduce the number of copies for a rule by atomically replacing the rule.
:param rule_id: Rule to be reduced.
:param copies: Number of copies of the new rule.
:param exclude_expression: RSE Expression of RSEs to exclude.
:param issuer: The issuing account of this operation
:param vo: The VO to act on.
:raises: RuleReplaceFailed, RuleNotFound
"""
kwargs = {'rule_id': rule_id, 'copies': copies, 'exclude_expression': exclude_expression}
if not has_permission(issuer=issuer, vo=vo, action='access_rule_vo', kwargs=kwargs):
raise AccessDenied('Account %s can not access rules at other VOs.' % (issuer))
if not has_permission(issuer=issuer, vo=vo, action='reduce_rule', kwargs=kwargs):
raise AccessDenied('Account %s can not reduce this replication rule.' % (issuer))
return rule.reduce_rule(rule_id=rule_id, copies=copies, exclude_expression=exclude_expression)
def update_subscription(name, account, metadata=None, issuer=None, vo='def'):
"""
Updates a subscription
:param name: Name of the subscription
:type: String
:param account: Account identifier
:type account: String
:param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed
:type metadata: Dict
:param issuer: The account issuing this operation.
:type issuer: String
:param vo: The VO to act on.
:type vo: String
:raises: SubscriptionNotFound if subscription is not found
"""
if not has_permission(issuer=issuer,
vo=vo,
action='update_subscription',
kwargs={'account': account}):
raise AccessDenied('Account %s can not update subscription' % (issuer))
try:
if not isinstance(metadata, dict):
raise TypeError('metadata should be a dict')
if 'filter' in metadata and metadata['filter']:
if not isinstance(metadata['filter'], dict):
raise TypeError('filter should be a dict')
validate_schema(name='subscription_filter', obj=metadata['filter'])
if 'replication_rules' in metadata and metadata['replication_rules']:
if not isinstance(metadata['replication_rules'], list):
raise TypeError('replication_rules should be a list')
else:
for rule in metadata['replication_rules']:
validate_schema(name='activity',
obj=rule.get('activity', 'default'))
except ValueError as error:
raise TypeError(error)
account = InternalAccount(account, vo=vo)
if 'filter' in metadata and metadata['filter'] is not None:
filter = metadata['filter']
keys = ['scope', 'account']
types = [InternalScope, InternalAccount]
for _key, _type in zip(keys, types):
if _key in filter and filter[_key] is not None:
if isinstance(filter[_key], list):
filter[_key] = [
_type(val, vo=vo).internal for val in filter[_key]
]
else:
filter[_key] = _type(filter[_key], vo=vo).internal
return subscription.update_subscription(name=name,
account=account,
metadata=metadata)
def delete_replication_rule(rule_id, purge_replicas, issuer, vo='def'):
"""
Deletes a replication rule and all associated locks.
:param rule_id: The id of the rule to be deleted
:param purge_replicas: Purge the replicas immediately
:param issuer: The issuing account of this operation
:param vo: The VO to act on.
:raises: RuleNotFound, AccessDenied
"""
kwargs = {'rule_id': rule_id, 'purge_replicas': purge_replicas}
if is_multi_vo() and not has_permission(
issuer=issuer, vo=vo, action='access_rule_vo', kwargs=kwargs):
raise AccessDenied('Account %s can not access rules at other VOs.' %
(issuer))
if not has_permission(
issuer=issuer, vo=vo, action='del_rule', kwargs=kwargs):
raise AccessDenied('Account %s can not remove this replication rule.' %
(issuer))
rule.delete_rule(rule_id=rule_id, purge_replicas=purge_replicas, soft=True)
def list_replication_rule_history(rule_id, issuer, vo='def'):
"""
Lists replication rule history..
:param rule_id: The rule_id to list.
:param issuer: The issuing account of this operation.
:param vo: The VO of the issuer.
"""
kwargs = {'rule_id': rule_id}
if not has_permission(issuer=issuer, vo=vo, action='access_rule_vo', kwargs=kwargs):
raise AccessDenied('Account %s can not access rules at other VOs.' % (issuer))
return rule.list_rule_history(rule_id)
def delete_replication_rule(rule_id, purge_replicas, issuer):
"""
Deletes a replication rule and all associated locks.
:param rule_id: The id of the rule to be deleted
:param issuer: The issuing account of this operation
:raises: RuleNotFound, AccessDenied
"""
kwargs = {'rule_id': rule_id, 'purge_replicas': purge_replicas}
if not has_permission(issuer=issuer, action='del_rule', kwargs=kwargs):
raise AccessDenied('Account %s can not remove this replication rule.' % (issuer))
rule.delete_rule(rule_id=rule_id, purge_replicas=purge_replicas, soft=True)
def add_value(key, value, issuer):
"""
Add a new value to a key.
:param key: the name for the key.
:param value: the value.
"""
kwargs = {'key': key, 'value': value}
if not has_permission(issuer=issuer, action='add_value', kwargs=kwargs):
raise AccessDenied('Account %s can not add value %s to key %s' %
(issuer, value, key))
return meta.add_value(key=key, value=value)
def move_replication_rule(rule_id,
rse_expression,
override,
issuer,
vo='def',
session=None):
"""
Move a replication rule to another RSE and, once done, delete the original one.
:param rule_id: Rule to be moved.
:param rse_expression: RSE expression of the new rule.
:param override: Configurations to update for the new rule.
:param session: The DB Session.
:param vo: The VO to act on.
:raises: RuleNotFound, RuleReplaceFailed, InvalidRSEExpression, AccessDenied
"""
kwargs = {
'rule_id': rule_id,
'rse_expression': rse_expression,
'override': override,
}
if is_multi_vo(
session=session) and not has_permission(issuer=issuer,
vo=vo,
action='access_rule_vo',
kwargs=kwargs,
session=session):
raise AccessDenied('Account %s can not access rules at other VOs.' %
(issuer))
if not has_permission(issuer=issuer,
vo=vo,
action='move_rule',
kwargs=kwargs,
session=session):
raise AccessDenied('Account %s can not move this replication rule.' %
(issuer))
return rule.move_rule(**kwargs, session=session)
def get_replication_rule(rule_id, issuer, estimate_ttc=None, vo='def'):
"""
Get replication rule by it's id.
:param rule_id: The rule_id to get.
:param issuer: The issuing account of this operation.
:param vo: The VO of the issuer.
"""
kwargs = {'rule_id': rule_id}
if not has_permission(issuer=issuer, vo=vo, action='access_rule_vo', kwargs=kwargs):
raise AccessDenied('Account %s can not access rules at other VOs.' % (issuer))
result = rule.get_rule(rule_id, estimate_ttc)
return api_update_return_dict(result)
def get_subscription_by_id(subscription_id, vo='def'):
"""
Get a specific subscription by id.
:param subscription_id: The subscription_id to select.
:param vo: The VO of the user issuing command.
:raises: SubscriptionNotFound if no Subscription can be found.
"""
sub = subscription.get_subscription_by_id(subscription_id)
if sub['account'].vo != vo:
raise AccessDenied('Unable to get subscription')
sub['account'] = sub['account'].external
return sub
def update_replication_rule(rule_id, options, issuer):
"""
Update lock state of a replication rule.
:param rule_id: The rule_id to lock.
:param options: Options dictionary.
:param issuer: The issuing account of this operation
:raises: RuleNotFound if no Rule can be found.
"""
kwargs = {'rule_id': rule_id, 'options': options}
if 'approve' in options:
if not has_permission(issuer=issuer, action='approve_rule', kwargs=kwargs):
raise AccessDenied('Account %s can not approve/deny this replication rule.' % (issuer))
issuer = InternalAccount(issuer)
if options['approve']:
rule.approve_rule(rule_id=rule_id, approver=issuer)
else:
rule.deny_rule(rule_id=rule_id, approver=issuer, reason=options.get('comment', None))
else:
if not has_permission(issuer=issuer, action='update_rule', kwargs=kwargs):
raise AccessDenied('Account %s can not update this replication rule.' % (issuer))
rule.update_rule(rule_id=rule_id, options=options)
def examine_replication_rule(rule_id, issuer, vo='def'):
"""
Examine a replication rule.
:param rule_id: The rule_id to get.
:param issuer: The issuing account of this operation.
:param vo: The VO of the issuer.
"""
kwargs = {'rule_id': rule_id}
if not has_permission(issuer=issuer, vo=vo, action='access_rule_vo', kwargs=kwargs):
raise AccessDenied('Account %s can not access rules at other VOs.' % (issuer))
result = rule.examine_rule(rule_id)
result = api_update_return_dict(result)
if 'transfers' in result:
result['transfers'] = [api_update_return_dict(t) for t in result['transfers']]
return result
def add_value(key, value, issuer, vo='def', session=None):
"""
Add a new value to a key.
:param key: the name for the key.
:param value: the value.
:param vo: the vo to act on.
:param session: The database session in use.
"""
kwargs = {'key': key, 'value': value}
if not has_permission(issuer=issuer,
vo=vo,
action='add_value',
kwargs=kwargs,
session=session):
raise AccessDenied('Account %s can not add value %s to key %s' %
(issuer, value, key))
return meta.add_value(key=key, value=value, session=session)
def list_replication_rule_history(rule_id, issuer, vo='def', session=None):
"""
Lists replication rule history..
:param rule_id: The rule_id to list.
:param issuer: The issuing account of this operation.
:param vo: The VO of the issuer.
:param session: The database session in use.
"""
kwargs = {'rule_id': rule_id}
if is_multi_vo(
session=session) and not has_permission(issuer=issuer,
vo=vo,
action='access_rule_vo',
kwargs=kwargs,
session=session):
raise AccessDenied('Account %s can not access rules at other VOs.' %
(issuer))
return rule.list_rule_history(rule_id, session=session)
def get_replication_rule(rule_id, issuer, vo='def', session=None):
"""
Get replication rule by it's id.
:param rule_id: The rule_id to get.
:param issuer: The issuing account of this operation.
:param vo: The VO of the issuer.
:param session: The database session in use.
"""
kwargs = {'rule_id': rule_id}
if is_multi_vo(
session=session) and not has_permission(issuer=issuer,
vo=vo,
action='access_rule_vo',
kwargs=kwargs,
session=session):
raise AccessDenied('Account %s can not access rules at other VOs.' %
(issuer))
result = rule.get_rule(rule_id, session=session)
return api_update_return_dict(result, session=session)
def add_key(key, key_type, issuer, value_type=None, value_regexp=None):
"""
Add a new allowed key.
:param key: the name for the new key.
:param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
:param issuer: The issuer account.
:param value_type: the type of the value, if defined.
:param value_regexp: the regular expression that values should match, if defined.
"""
kwargs = {
'key': key,
'key_type': key_type,
'value_type': value_type,
'value_regexp': value_regexp
}
if not has_permission(issuer=issuer, action='add_key', kwargs=kwargs):
raise AccessDenied('Account %s can not add key' % (issuer))
return meta.add_key(key=key,
key_type=KeyType.from_sym(key_type),
value_type=value_type,
value_regexp=value_regexp)
def update_subscription(name, account, metadata=None, issuer=None):
"""
Updates a subscription
:param name: Name of the subscription
:type: String
:param account: Account identifier
:type account: String
:param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed
:type metadata: Dict
:raises: SubscriptionNotFound if subscription is not found
"""
if not has_permission(issuer=issuer,
action='update_subscription',
kwargs={'account': account}):
raise AccessDenied('Account %s can not update subscription' % (issuer))
try:
if not isinstance(metadata, dict):
raise TypeError('metadata should be a dict')
if 'filter' in metadata and metadata['filter']:
if not isinstance(metadata['filter'], dict):
raise TypeError('filter should be a dict')
validate_schema(name='subscription_filter', obj=metadata['filter'])
if 'replication_rules' in metadata and metadata['replication_rules']:
if not isinstance(metadata['replication_rules'], list):
raise TypeError('replication_rules should be a list')
else:
for rule in metadata['replication_rules']:
validate_schema(name='activity',
obj=rule.get('activity', 'default'))
except ValueError as error:
raise TypeError(error)
account = InternalAccount(account)
return subscription.update_subscription(name=name,
account=account,
metadata=metadata)
def get_subscription_by_id(subscription_id, vo='def'):
"""
Get a specific subscription by id.
:param subscription_id: The subscription_id to select.
:param vo: The VO of the user issuing command.
:raises: SubscriptionNotFound if no Subscription can be found.
"""
sub = subscription.get_subscription_by_id(subscription_id)
if sub['account'].vo != vo:
raise AccessDenied('Unable to get subscription')
sub['account'] = sub['account'].external
if 'filter' in sub:
fil = loads(sub['filter'])
if 'account' in fil:
fil['account'] = [InternalAccount(acc, fromExternal=False).external for acc in fil['account']]
if 'scope' in fil:
fil['scope'] = [InternalScope(sco, fromExternal=False).external for sco in fil['scope']]
sub['filter'] = dumps(fil)
return sub
def add_replication_rule(dids,
copies,
rse_expression,
weight,
lifetime,
grouping,
account,
locked,
subscription_id,
source_replica_expression,
activity,
notify,
purge_replicas,
ignore_availability,
comment,
ask_approval,
asynchronous,
priority,
split_container,
meta,
issuer,
vo='def'):
"""
Adds a replication rule.
:param dids: The data identifier set.
:param copies: The number of replicas.
:param rse_expression: Boolean string expression to give the list of RSEs.
:param weight: If the weighting option of the replication rule is used, the choice of RSEs takes their weight into account.
:param lifetime: The lifetime of the replication rules (in seconds).
:param grouping: ALL - All files will be replicated to the same RSE.
DATASET - All files in the same dataset will be replicated to the same RSE.
NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all.
:param account: The account owning the rule.
:param locked: If the rule is locked, it cannot be deleted.
:param subscription_id: The subscription_id, if the rule is created by a subscription.
:param source_replica_expression: Only use replicas from this RSE as sources.
:param activity: Activity to be passed on to the conveyor.
:param notify: Notification setting of the rule.
:purge purge_replicas: The purge setting to delete replicas immediately after rule deletion.
:param ignore_availability: Option to ignore the availability of RSEs.
:param comment: Comment about the rule.
:param ask_approval: Ask for approval of this rule.
:param asynchronous: Create rule asynchronously by judge-injector.
:param priority: Priority of the transfers.
:param split_container: Should a container rule be split into individual dataset rules.
:param meta: WFMS metadata as a dictionary.
:param issuer: The issuing account of this operation.
:param vo: The VO to act on.
:returns: List of created replication rules.
"""
if account is None:
account = issuer
if activity is None:
activity = 'User Subscriptions'
kwargs = {
'dids': dids,
'copies': copies,
'rse_expression': rse_expression,
'weight': weight,
'lifetime': lifetime,
'grouping': grouping,
'account': account,
'locked': locked,
'subscription_id': subscription_id,
'source_replica_expression': source_replica_expression,
'notify': notify,
'activity': activity,
'purge_replicas': purge_replicas,
'ignore_availability': ignore_availability,
'comment': comment,
'ask_approval': ask_approval,
'asynchronous': asynchronous,
'priority': priority,
'split_container': split_container,
'meta': meta
}
validate_schema(name='rule', obj=kwargs)
if not has_permission(
issuer=issuer, vo=vo, action='add_rule', kwargs=kwargs):
raise AccessDenied('Account %s can not add replication rule' %
(issuer))
account = InternalAccount(account, vo=vo)
for d in dids:
d['scope'] = InternalScope(d['scope'], vo=vo)
if vo != 'def':
if rse_expression is not None:
rse_expression = 'vo={}&({})'.format(vo, rse_expression)
else:
rse_expression = 'vo={}'.format(vo)
if source_replica_expression is not None:
source_replica_expression = 'vo={}&({})'.format(
vo, source_replica_expression)
else:
source_replica_expression = 'vo={}'.format(vo)
return rule.add_rule(account=account,
dids=dids,
copies=copies,
rse_expression=rse_expression,
grouping=grouping,
weight=weight,
lifetime=lifetime,
locked=locked,
subscription_id=subscription_id,
source_replica_expression=source_replica_expression,
activity=activity,
notify=notify,
purge_replicas=purge_replicas,
ignore_availability=ignore_availability,
comment=comment,
ask_approval=ask_approval,
asynchronous=asynchronous,
priority=priority,
split_container=split_container,
meta=meta)
def add_subscription(name,
account,
filter_,
replication_rules,
comments,
lifetime,
retroactive,
dry_run,
priority=None,
issuer=None,
vo='def',
session=None):
"""
Adds a new subscription which will be verified against every new added file and dataset
:param account: Account identifier
:type account: String
:param name: Name of the subscription
:type: String
:param filter_: Dictionary of attributes by which the input data should be filtered
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
:type filter_: Dict
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
:type replication_rules: Dict
:param comments: Comments for the subscription
:type comments: String
:param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime
:type lifetime: Integer or False
:param retroactive: Flag to know if the subscription should be applied on previous data
:type retroactive: Boolean
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
:type dry_run: Boolean
:param priority: The priority of the subscription
:type priority: Integer
:param issuer: The account issuing this operation.
:type issuer: String
:param vo: The VO to act on.
:type vo: String
:param session: The database session in use.
:returns: subscription_id
:rtype: String
"""
if not has_permission(issuer=issuer,
vo=vo,
action='add_subscription',
kwargs={'account': account},
session=session):
raise AccessDenied('Account %s can not add subscription' % (issuer))
try:
if filter_:
if not isinstance(filter_, dict):
raise TypeError('filter should be a dict')
validate_schema(name='subscription_filter', obj=filter_, vo=vo)
if replication_rules:
if not isinstance(replication_rules, list):
raise TypeError('replication_rules should be a list')
else:
for rule in replication_rules:
validate_schema(name='activity',
obj=rule.get('activity', 'default'),
vo=vo)
else:
raise InvalidObject('You must specify a rule')
except ValueError as error:
raise TypeError(error)
account = InternalAccount(account, vo=vo)
keys = ['scope', 'account']
types = [InternalScope, InternalAccount]
for _key, _type in zip(keys, types):
if _key in filter_:
if isinstance(filter_[_key], list):
filter_[_key] = [
_type(val, vo=vo).internal for val in filter_[_key]
]
else:
filter_[_key] = _type(filter_[_key], vo=vo).internal
return subscription.add_subscription(
name=name,
account=account,
filter_=dumps(filter_),
replication_rules=dumps(replication_rules),
comments=comments,
lifetime=lifetime,
retroactive=retroactive,
dry_run=dry_run,
priority=priority,
session=session)
