def delete_replicas(rse, files, issuer, ignore_availability=False):
"""
Bulk delete file replicas.
:param rse: The RSE name.
:param files: The list of files.
:param issuer: The issuer account.
:param ignore_availability: Ignore the RSE blacklisting.
:returns: True is successful, False otherwise
"""
validate_schema(name='r_dids', obj=files)
rse_id = get_rse_id(rse=rse)
kwargs = {'rse': rse, 'rse_id': rse_id}
if not permission.has_permission(
issuer=issuer, action='delete_replicas', kwargs=kwargs):
raise exception.AccessDenied(
'Account %s can not delete file replicas on %s' % (issuer, rse))
if not permission.has_permission(
issuer=issuer, action='skip_availability_check', kwargs=kwargs):
ignore_availability = False
for f in files:
f['scope'] = InternalScope(f['scope'])
replica.delete_replicas(rse_id=rse_id,
files=files,
ignore_availability=ignore_availability)
def add_vo(new_vo,
issuer,
description=None,
email=None,
vo='def',
session=None):
'''
Add a new VO.
:param new_vo: The name/tag of the VO to add (3 characters).
:param description: A description of the VO. e.g the full name or a brief description
:param email: A contact for the VO.
:param issuer: The user issuing the command.
:param vo: The vo of the user issuing the command.
:param session: The database session in use.
'''
new_vo = vo_core.map_vo(new_vo)
validate_schema('vo', new_vo, vo=vo)
kwargs = {}
if not has_permission(
issuer=issuer, action='add_vo', kwargs=kwargs, vo=vo,
session=session):
raise exception.AccessDenied(
'Account {} cannot add a VO'.format(issuer))
vo_core.add_vo(vo=new_vo,
description=description,
email=email,
session=session)
def add_replicas(rse, files, issuer, ignore_availability=False):
"""
Bulk add file replicas.
:param rse: The RSE name.
:param files: The list of files.
:param issuer: The issuer account.
:param ignore_availability: Ignore the RSE blacklisting.
:returns: True is successful, False otherwise
"""
validate_schema(name='dids', obj=files)
kwargs = {'rse': rse}
if not permission.has_permission(
issuer=issuer, action='add_replicas', kwargs=kwargs):
raise exception.AccessDenied(
'Account %s can not add file replicas on %s' % (issuer, rse))
if not permission.has_permission(
issuer=issuer, action='skip_availability_check', kwargs=kwargs):
ignore_availability = False
replica.add_replicas(rse=rse,
files=files,
account=issuer,
ignore_availability=ignore_availability)
def update_replicas_states(rse, files, issuer):
"""
Update File replica information and state.
:param rse: The RSE name.
:param files: The list of files.
:param issuer: The issuer account.
"""
for v_file in files:
v_file.update(
{"type":
"FILE"}) # Make sure DIDs are identified as files for checking
validate_schema(name='dids', obj=files)
rse_id = get_rse_id(rse=rse)
kwargs = {'rse': rse, 'rse_id': rse_id}
if not permission.has_permission(
issuer=issuer, action='update_replicas_states', kwargs=kwargs):
raise exception.AccessDenied(
'Account %s can not update file replicas state on %s' %
(issuer, rse))
replicas = []
for file in files:
rep = file
rep['rse_id'] = rse_id
rep['scope'] = InternalScope(rep['scope'])
replicas.append(rep)
replica.update_replicas_states(replicas=replicas)
def add_subscription(name, account, filter_, replication_rules, comments, lifetime, retroactive, dry_run, priority=None, issuer=None, vo='def'):
"""
Adds a new subscription which will be verified against every new added file and dataset
:param account: Account identifier
:type account: String
:param name: Name of the subscription
:type: String
:param filter_: Dictionary of attributes by which the input data should be filtered
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
:type filter_: Dict
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
:type replication_rules: Dict
:param comments: Comments for the subscription
:type comments: String
:param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime
:type lifetime: Integer or False
:param retroactive: Flag to know if the subscription should be applied on previous data
:type retroactive: Boolean
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
:type dry_run: Boolean
:param priority: The priority of the subscription
:type priority: Integer
:param issuer: The account issuing this operation.
:type issuer: String
:param vo: The VO to act on.
:type vo: String
:returns: subscription_id
:rtype: String
"""
if not has_permission(issuer=issuer, vo=vo, action='add_subscription', kwargs={'account': account}):
raise AccessDenied('Account %s can not add subscription' % (issuer))
try:
if filter_:
if not isinstance(filter_, dict):
raise TypeError('filter should be a dict')
validate_schema(name='subscription_filter', obj=filter_, vo=vo)
if replication_rules:
if not isinstance(replication_rules, list):
raise TypeError('replication_rules should be a list')
else:
for rule in replication_rules:
validate_schema(name='activity', obj=rule.get('activity', 'default'), vo=vo)
else:
raise InvalidObject('You must specify a rule')
except ValueError as error:
raise TypeError(error)
account = InternalAccount(account, vo=vo)
keys = ['scope', 'account']
types = [InternalScope, InternalAccount]
for _key, _type in zip(keys, types):
if _key in filter_:
if isinstance(filter_[_key], list):
filter_[_key] = [_type(val, vo=vo).internal for val in filter_[_key]]
else:
filter_[_key] = _type(filter_[_key], vo=vo).internal
return subscription.add_subscription(name=name, account=account, filter_=dumps(filter_), replication_rules=dumps(replication_rules), comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run, priority=priority)
def message_handle(self, msg):
record_counter('daemons.cache.consumer.message_handle.message')
try:
if isinstance(msg, dict) and 'operation' in msg.keys():
if msg['operation'] == 'add_replicas':
validate_schema(name='cache_add_replicas', obj=msg)
if 'rse' in msg.keys() and 'files' in msg.keys():
logging.debug('[%s] %s %s %s' % (self.__broker, msg['operation'], msg['rse'], msg['files']))
try:
if not self.__rse_volatile.get_volatile(msg['rse']):
logging.error("%s volatile is not True, Rucio Cache should not update it." % (msg['rse']))
else:
cache_add_replicas(rse=msg['rse'], files=msg['files'], account=self.__account, lifetime=msg['lifetime'])
except Exception, e:
logging.error('[%s] %s %s %s %s with error details: %s' % (self.__broker, msg['operation'], msg['rse'], msg['files'], str(e), str(format_exc())))
if msg['operation'] == 'delete_replicas':
validate_schema(name='cache_delete_replicas', obj=msg)
if 'rse' in msg.keys() and 'files' in msg.keys():
logging.debug('[%s] %s %s %s' % (self.__broker, msg['operation'], msg['rse'], msg['files']))
try:
if not self.__rse_volatile.get_volatile(msg['rse']):
logging.error("%s volatile is not True, Rucio Cache should not update it." % (msg['rse']))
else:
cache_delete_replicas(rse=msg['rse'], files=msg['files'], account=self.__account)
except Exception, e:
logging.error('[%s] %s %s %s %s with error details: %s' % (self.__broker, msg['operation'], msg['rse'], msg['files'], str(e), str(format_exc())))
def list_dids(scope,
filters,
type='collection',
ignore_case=False,
limit=None,
offset=None,
long=False,
recursive=False):
"""
List dids in a scope.
:param scope: The scope name.
:param pattern: The wildcard pattern.
:param type: The type of the did: all(container, dataset, file), collection(dataset or container), dataset, container
:param ignore_case: Ignore case distinctions.
:param limit: The maximum number of DIDs returned.
:param offset: Offset number.
:param long: Long format option to display more information for each DID.
:param recursive: Recursively list DIDs content.
"""
validate_schema(name='did_filters', obj=filters)
return did.list_dids(scope=scope,
filters=filters,
type=type,
ignore_case=ignore_case,
limit=limit,
offset=offset,
long=long,
recursive=recursive)
def list_replicas(dids, schemes=None, unavailable=False, request_id=None,
ignore_availability=True, all_states=False, rse_expression=None,
client_location=None, domain=None, lifetime=None, issuer=None):
"""
List file replicas for a list of data identifiers.
:param dids: The list of data identifiers (DIDs).
:param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...)
:param unavailable: Also include unavailable replicas in the list.
:param request_id: ID associated with the request for debugging.
:param all_states: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
:param rse_expression: The RSE expression to restrict replicas on a set of RSEs.
:param client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'}
:param domain: The network domain for the call, either None, 'wan' or 'lan'. Compatibility fallback: None falls back to 'wan'.
:param lifetime: If supported, in seconds, restrict the lifetime of the replica PFN.
:param issuer: The issuer account.
"""
validate_schema(name='r_dids', obj=dids)
# Allow selected authenticated users to retrieve signed URLs.
# Unauthenticated users, or permission-less users will get the raw URL without the signature.
sign_urls = False
if permission.has_permission(issuer=issuer, action='get_signed_urls', kwargs={}):
sign_urls = True
return replica.list_replicas(dids=dids, schemes=schemes, unavailable=unavailable,
request_id=request_id,
ignore_availability=ignore_availability,
all_states=all_states, rse_expression=rse_expression,
client_location=client_location, domain=domain,
sign_urls=sign_urls, lifetime=lifetime)
def add_account(account, type_, email, issuer, vo='def', session=None):
"""
Creates an account with the provided account name, contact information, etc.
:param account: The account name.
:param type_: The account type
:param email: The Email address associated with the account.
:param issuer: The issuer account_core.
:param vo: The VO to act on.
:param session: The database session in use.
"""
validate_schema(name='account', obj=account, vo=vo)
kwargs = {'account': account, 'type': type_}
if not rucio.api.permission.has_permission(issuer=issuer,
vo=vo,
action='add_account',
kwargs=kwargs,
session=session):
raise rucio.common.exception.AccessDenied(
'Account %s can not add account' % (issuer))
account = InternalAccount(account, vo=vo)
account_core.add_account(account,
AccountType[type_.upper()],
email,
session=session)
def add_rse(rse,
issuer,
vo='def',
deterministic=True,
volatile=False,
city=None,
region_code=None,
country_name=None,
continent=None,
time_zone=None,
ISP=None,
staging_area=False,
rse_type=None,
latitude=None,
longitude=None,
ASN=None,
availability=None):
"""
Creates a new Rucio Storage Element(RSE).
:param rse: The RSE name.
:param issuer: The issuer account.
:param vo: The VO to act on.
:param deterministic: Boolean to know if the pfn is generated deterministically.
:param volatile: Boolean for RSE cache.
:param city: City for the RSE.
:param region_code: The region code for the RSE.
:param country_name: The country.
:param continent: The continent.
:param time_zone: Timezone.
:param staging_area: staging area.
:param ISP: Internet service provider.
:param rse_type: RSE type.
:param latitude: Latitude coordinate of RSE.
:param longitude: Longitude coordinate of RSE.
:param ASN: Access service network.
:param availability: Availability.
"""
validate_schema(name='rse', obj=rse)
kwargs = {'rse': rse}
if not permission.has_permission(
issuer=issuer, vo=vo, action='add_rse', kwargs=kwargs):
raise exception.AccessDenied('Account %s can not add RSE' % (issuer))
return rse_module.add_rse(rse,
vo=vo,
deterministic=deterministic,
volatile=volatile,
city=city,
region_code=region_code,
country_name=country_name,
staging_area=staging_area,
continent=continent,
time_zone=time_zone,
ISP=ISP,
rse_type=rse_type,
latitude=latitude,
longitude=longitude,
ASN=ASN,
availability=availability)
def list_replicas(dids,
schemes=None,
unavailable=False,
request_id=None,
ignore_availability=True,
all_states=False,
rse_expression=None):
"""
List file replicas for a list of data identifiers.
:param dids: The list of data identifiers (DIDs).
:param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...)
:param unavailable: Also include unavailable replicas in the list.
:param request_id: ID associated with the request for debugging.
:param all_states: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
:param rse_expression: The RSE expression to restrict replicas on a set of RSEs.
"""
validate_schema(name='r_dids', obj=dids)
return replica.list_replicas(dids=dids,
schemes=schemes,
unavailable=unavailable,
request_id=request_id,
ignore_availability=ignore_availability,
all_states=all_states,
rse_expression=rse_expression)
def add_account_attribute(key, value, account, issuer, vo='def', session=None):
"""
Add an attribute to an account.
:param key: attribute key.
:param value: attribute value.
:param account: The account name.
:param issuer: The issuer account.
:param vo: The VO to act on.
:param session: The database session in use.
"""
validate_schema(name='account_attribute', obj=key, vo=vo)
validate_schema(name='account_attribute', obj=value, vo=vo)
kwargs = {'account': account, 'key': key, 'value': value}
if not rucio.api.permission.has_permission(issuer=issuer,
vo=vo,
action='add_attribute',
kwargs=kwargs,
session=session):
raise rucio.common.exception.AccessDenied(
'Account %s can not add attributes' % (issuer))
account = InternalAccount(account, vo=vo)
account_core.add_account_attribute(account, key, value, session=session)
def attach_dids_to_dids(attachments, issuer, ignore_duplicate=False):
"""
Append content to dids.
:param attachments: The contents.
:param issuer: The issuer account.
:param ignore_duplicate: If True, ignore duplicate entries.
"""
validate_schema(name='attachments', obj=attachments)
for a in attachments:
if 'rse' in a:
rse_id = None
if a['rse'] is not None:
rse_id = get_rse_id(rse=a['rse'])
a['rse_id'] = rse_id
if not rucio.api.permission.has_permission(
issuer=issuer,
action='attach_dids_to_dids',
kwargs={'attachments': attachments}):
raise rucio.common.exception.AccessDenied(
'Account %s can not add data identifiers' % (issuer))
issuer = InternalAccount(issuer)
for attachment in attachments:
attachment['scope'] = InternalScope(attachment['scope'])
for d in attachment['dids']:
d['scope'] = InternalScope(d['scope'])
return did.attach_dids_to_dids(attachments=attachments,
account=issuer,
ignore_duplicate=ignore_duplicate)
def update_subscription(name, account, metadata=None, issuer=None, vo='def'):
"""
Updates a subscription
:param name: Name of the subscription
:type: String
:param account: Account identifier
:type account: String
:param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed
:type metadata: Dict
:param issuer: The account issuing this operation.
:type issuer: String
:param vo: The VO to act on.
:type vo: String
:raises: SubscriptionNotFound if subscription is not found
"""
if not has_permission(issuer=issuer,
vo=vo,
action='update_subscription',
kwargs={'account': account}):
raise AccessDenied('Account %s can not update subscription' % (issuer))
try:
if not isinstance(metadata, dict):
raise TypeError('metadata should be a dict')
if 'filter' in metadata and metadata['filter']:
if not isinstance(metadata['filter'], dict):
raise TypeError('filter should be a dict')
validate_schema(name='subscription_filter',
obj=metadata['filter'],
vo=vo)
if 'replication_rules' in metadata and metadata['replication_rules']:
if not isinstance(metadata['replication_rules'], list):
raise TypeError('replication_rules should be a list')
else:
for rule in metadata['replication_rules']:
validate_schema(name='activity',
obj=rule.get('activity', 'default'),
vo=vo)
except ValueError as error:
raise TypeError(error)
account = InternalAccount(account, vo=vo)
if 'filter' in metadata and metadata['filter'] is not None:
filter = metadata['filter']
keys = ['scope', 'account']
types = [InternalScope, InternalAccount]
for _key, _type in zip(keys, types):
if _key in filter and filter[_key] is not None:
if isinstance(filter[_key], list):
filter[_key] = [
_type(val, vo=vo).internal for val in filter[_key]
]
else:
filter[_key] = _type(filter[_key], vo=vo).internal
return subscription.update_subscription(name=name,
account=account,
metadata=metadata)
def list_replicas(dids,
schemes=None,
unavailable=False,
request_id=None,
ignore_availability=True,
all_states=False,
rse_expression=None,
client_location=None,
domain=None):
"""
List file replicas for a list of data identifiers.
:param dids: The list of data identifiers (DIDs).
:param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...)
:param unavailable: Also include unavailable replicas in the list.
:param request_id: ID associated with the request for debugging.
:param all_states: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
:param rse_expression: The RSE expression to restrict replicas on a set of RSEs.
:param client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'}
:param domain: The network domain for the call, either None, 'wan' or 'lan'. Compatibility fallback: None falls back to 'wan'.
"""
validate_schema(name='r_dids', obj=dids)
return replica.list_replicas(dids=dids,
schemes=schemes,
unavailable=unavailable,
request_id=request_id,
ignore_availability=ignore_availability,
all_states=all_states,
rse_expression=rse_expression,
client_location=client_location,
domain=domain)
def update_subscription(name, account, metadata=None, issuer=None):
"""
Updates a subscription
:param name: Name of the subscription
:type: String
:param account: Account identifier
:type account: String
:param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed
:type metadata: Dict
:raises: SubscriptionNotFound if subscription is not found
"""
if not has_permission(issuer=issuer, action='update_subscription', kwargs={'account': account}):
raise AccessDenied('Account %s can not update subscription' % (issuer))
try:
if not isinstance(metadata, dict):
raise TypeError('metadata should be a dict')
if 'filter' in metadata and metadata['filter']:
if not isinstance(metadata['filter'], dict):
raise TypeError('filter should be a dict')
validate_schema(name='subscription_filter', obj=metadata['filter'])
if 'replication_rules' in metadata and metadata['replication_rules']:
if not isinstance(metadata['replication_rules'], list):
raise TypeError('replication_rules should be a list')
else:
for rule in metadata['replication_rules']:
validate_schema(name='activity', obj=rule.get('activity', 'default'))
except ValueError as error:
raise TypeError(error)
return subscription.update_subscription(name=name, account=account, metadata=metadata)
def list_dataset_replicas_bulk(dids, vo='def'):
"""
:param dids: The list of did dictionaries with scope and name.
:param vo: The VO to act on.
:returns: A list of dict dataset replicas
"""
validate_schema(name='r_dids', obj=dids, vo=vo)
names_by_scope = dict()
for d in dids:
if d['scope'] in names_by_scope:
names_by_scope[d['scope']].append(d['name'])
else:
names_by_scope[d['scope']] = [
d['name'],
]
names_by_intscope = dict()
for scope in names_by_scope:
internal_scope = InternalScope(scope, vo=vo)
names_by_intscope[internal_scope] = names_by_scope[scope]
replicas = replica.list_dataset_replicas_bulk(names_by_intscope)
for r in replicas:
yield api_update_return_dict(r)
def attach_dids(scope, name, attachment, issuer):
"""
Append content to data did.
:param attachment: The attachment.
:param issuer: The issuer account.
"""
validate_schema(name='attachment', obj=attachment)
rse_id = None
if 'rse' in attachment:
rse_id = get_rse_id(rse=attachment.get('rse'))
attachment['rse_id'] = rse_id
kwargs = {'scope': scope, 'name': name, 'attachment': attachment}
if not rucio.api.permission.has_permission(
issuer=issuer, action='attach_dids', kwargs=kwargs):
raise rucio.common.exception.AccessDenied(
'Account %s can not add data identifiers to %s:%s' %
(issuer, scope, name))
if rse_id is not None:
dids = did.attach_dids(scope=scope,
name=name,
dids=attachment['dids'],
account=attachment.get('account', issuer),
rse_id=rse_id)
else:
dids = did.attach_dids(scope=scope,
name=name,
dids=attachment['dids'],
account=attachment.get('account', issuer))
return dids
def add_replicas(rse, files, issuer, ignore_availability=False, vo='def'):
"""
Bulk add file replicas.
:param rse: The RSE name.
:param files: The list of files.
:param issuer: The issuer account.
:param ignore_availability: Ignore blocked RSEs.
:param vo: The VO to act on.
:returns: True is successful, False otherwise
"""
for v_file in files:
v_file.update({"type": "FILE"}) # Make sure DIDs are identified as files for checking
validate_schema(name='dids', obj=files, vo=vo)
rse_id = get_rse_id(rse=rse, vo=vo)
kwargs = {'rse': rse, 'rse_id': rse_id}
if not permission.has_permission(issuer=issuer, vo=vo, action='add_replicas', kwargs=kwargs):
raise exception.AccessDenied('Account %s can not add file replicas on %s' % (issuer, rse))
if not permission.has_permission(issuer=issuer, vo=vo, action='skip_availability_check', kwargs=kwargs):
ignore_availability = False
issuer = InternalAccount(issuer, vo=vo)
for f in files:
f['scope'] = InternalScope(f['scope'], vo=vo)
if 'account' in f:
f['account'] = InternalAccount(f['account'], vo=vo)
replica.add_replicas(rse_id=rse_id, files=files, account=issuer, ignore_availability=ignore_availability)
def update_replicas_states(rse, files, issuer, vo='def'):
"""
Update File replica information and state.
:param rse: The RSE name.
:param files: The list of files.
:param issuer: The issuer account.
:param vo: The VO to act on.
"""
validate_schema(name='dids', obj=files)
rse_id = get_rse_id(rse=rse, vo=vo)
kwargs = {'rse': rse, 'rse_id': rse_id}
if not permission.has_permission(
issuer=issuer, vo=vo, action='update_replicas_states',
kwargs=kwargs):
raise exception.AccessDenied(
'Account %s can not update file replicas state on %s' %
(issuer, rse))
replicas = []
for file in files:
rep = file
rep['rse_id'] = rse_id
rep['scope'] = InternalScope(rep['scope'], vo=vo)
replicas.append(rep)
replica.update_replicas_states(replicas=replicas)
def list_dids_extended(scope, filters, did_type='collection', ignore_case=False, limit=None, offset=None, long=False, recursive=False, vo='def'):
"""
List dids in a scope.
:param scope: The scope name.
:param pattern: The wildcard pattern.
:param did_type: The type of the did: all(container, dataset, file), collection(dataset or container), dataset, container
:param ignore_case: Ignore case distinctions.
:param limit: The maximum number of DIDs returned.
:param offset: Offset number.
:param long: Long format option to display more information for each DID.
:param recursive: Recursively list DIDs content.
"""
validate_schema(name='did_filters', obj=filters, vo=vo)
scope = InternalScope(scope, vo=vo)
if 'account' in filters:
filters['account'] = InternalAccount(filters['account'], vo=vo)
if 'scope' in filters:
filters['scope'] = InternalScope(filters['scope'], vo=vo)
result = did.list_dids_extended(scope=scope, filters=filters, did_type=did_type, ignore_case=ignore_case,
limit=limit, offset=offset, long=long, recursive=recursive)
for d in result:
yield api_update_return_dict(d)
def add_subscription(name,
account,
filter,
replication_rules,
comments,
lifetime,
retroactive,
dry_run,
priority=None,
issuer=None):
"""
Adds a new subscription which will be verified against every new added file and dataset
:param account: Account identifier
:type account: String
:param name: Name of the subscription
:type: String
:param filter: Dictionary of attributes by which the input data should be filtered
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
:type filter: Dict
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
:type replication_rules: Dict
:param comments: Comments for the subscription
:type comments: String
:param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime
:type lifetime: Integer or False
:param retroactive: Flag to know if the subscription should be applied on previous data
:type retroactive: Boolean
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
:type dry_run: Boolean
:param priority: The priority of the subscription
:type priority: Integer
:param issuer: The account issuing this operation.
:type comments: String
:returns: subscription_id
:rtype: String
"""
if not has_permission(issuer=issuer,
action='add_subscription',
kwargs={'account': account}):
raise AccessDenied('Account %s can not add subscription' % (issuer))
try:
if filter:
if type(filter) != dict:
raise TypeError('filter should be a dict')
validate_schema(name='subscription_filter', obj=filter)
if replication_rules:
if type(replication_rules) != list:
raise TypeError('replication_rules should be a list')
else:
for rule in replication_rules:
validate_schema(name='activity',
obj=rule.get('activity', 'default'))
else:
raise InvalidObject('You must specify a rule')
except ValueError, error:
raise TypeError(error)
def resurrect(dids, issuer):
"""
Resurrect DIDs.
:param dids: A list of dids.
:param issuer: The issuer account.
"""
kwargs = {'issuer': issuer}
if not rucio.api.permission.has_permission(issuer=issuer, action='resurrect', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not resurrect data identifiers' % (issuer))
validate_schema(name='dids', obj=dids)
return did.resurrect(dids=dids)
def list_replicas(dids, schemes=None, unavailable=False, request_id=None, ignore_availability=True, all_states=False):
"""
List file replicas for a list of data identifiers.
:param dids: The list of data identifiers (DIDs).
:param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...)
:param unavailable: Also include unavailable replicas in the list.
:param request_id: ID associated with the request for debugging.
:param all_states: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
"""
validate_schema(name='r_dids', obj=dids)
return replica.list_replicas(dids=dids, schemes=schemes, unavailable=unavailable, request_id=request_id, ignore_availability=ignore_availability, all_states=all_states)
def attach_dids_to_dids(attachments, issuer):
"""
Append content to dids.
:param attachments: The contents.
:param issuer: The issuer account.
"""
validate_schema(name='attachments', obj=attachments)
if not rucio.api.permission.has_permission(issuer=issuer, action='attach_dids_to_dids', kwargs={'attachments': attachments}):
raise rucio.common.exception.AccessDenied('Account %s can not add data identifiers' % (issuer))
return did.attach_dids_to_dids(attachments=attachments, account=issuer)
def set_account_status(account, status, issuer='root'):
""" Set the status of an account_core.
:param account: Name of the account_core.
:param status: The status for the account_core.
"""
validate_schema(name='account', obj=account)
kwargs = {}
if not rucio.api.permission.has_permission(
issuer=issuer, action='set_account_status', kwargs=kwargs):
raise rucio.common.exception.AccessDenied(
'Account %s can not change the status of the account' % (issuer))
return account_core.set_account_status(account, status)
def get_metadata_bulk(dids, vo='def', session=None):
"""
Get metadata for a list of dids
:param dids: A list of dids.
:param session: The database session in use.
"""
validate_schema(name='dids', obj=dids, vo=vo)
for entry in dids:
entry['scope'] = InternalScope(entry['scope'], vo=vo)
meta = did.get_metadata_bulk(dids)
for met in meta:
yield api_update_return_dict(met)
def attach_dids(scope, name, attachment, issuer, vo='def', session=None):
"""
Append content to data did.
:param attachment: The attachment.
:param issuer: The issuer account.
:param vo: The VO to act on.
:param session: The database session in use.
"""
validate_schema(name='attachment', obj=attachment, vo=vo)
rse_id = None
if 'rse' in attachment:
if attachment['rse'] is not None:
rse_id = get_rse_id(rse=attachment['rse'], vo=vo, session=session)
attachment['rse_id'] = rse_id
kwargs = {'scope': scope, 'name': name, 'attachment': attachment}
if not rucio.api.permission.has_permission(issuer=issuer,
vo=vo,
action='attach_dids',
kwargs=kwargs,
session=session):
raise rucio.common.exception.AccessDenied(
'Account %s can not add data identifiers to %s:%s' %
(issuer, scope, name))
scope = InternalScope(scope, vo=vo)
issuer = InternalAccount(issuer, vo=vo)
if 'account' in attachment.keys():
attachment['account'] = InternalAccount(attachment['account'], vo=vo)
for d in attachment['dids']:
d['scope'] = InternalScope(d['scope'], vo=vo)
if 'account' in d.keys():
d['account'] = InternalAccount(d['account'], vo=vo)
if rse_id is not None:
dids = did.attach_dids(scope=scope,
name=name,
dids=attachment['dids'],
account=attachment.get('account', issuer),
rse_id=rse_id,
session=session)
else:
dids = did.attach_dids(scope=scope,
name=name,
dids=attachment['dids'],
account=attachment.get('account', issuer),
session=session)
return dids
def import_data(data, issuer, vo='def'):
"""
Import data to add/update/delete records in Rucio.
:param data: data to be imported.
:param issuer: the issuer.
:param vo: the VO of the issuer.
"""
kwargs = {'issuer': issuer}
validate_schema(name='import', obj=data)
if not permission.has_permission(issuer=issuer, vo=vo, action='import', kwargs=kwargs):
raise exception.AccessDenied('Account %s can not import data' % issuer)
return importer.import_data(data, vo=vo)
def update_account(account, key, value, issuer='root'):
""" Update a property of an account_core.
:param account: Name of the account_core.
:param key: Account property like status.
:param value: Property value.
"""
validate_schema(name='account', obj=account)
kwargs = {}
if not rucio.api.permission.has_permission(
issuer=issuer, action='update_account', kwargs=kwargs):
raise rucio.common.exception.AccessDenied(
'Account %s can not change %s of the account' % (issuer, key))
return account_core.update_account(account, key, value)
def get_metadata_bulk(dids, inherit=False, vo='def'):
"""
Get metadata for a list of dids
:param dids: A list of dids.
:param inherit: A boolean. If set to true, the metadata of the parent are concatenated.
:param vo: The VO to act on.
"""
validate_schema(name='dids', obj=dids, vo=vo)
for entry in dids:
entry['scope'] = InternalScope(entry['scope'], vo=vo)
meta = did.get_metadata_bulk(dids, inherit=inherit)
for met in meta:
yield api_update_return_dict(met)
def add_replication_rule(dids, copies, rse_expression, weight, lifetime, grouping, account, locked, subscription_id, source_replica_expression, activity, notify, purge_replicas, ignore_availability, issuer):
"""
Adds a replication rule.
:param dids: The data identifier set.
:param copies: The number of replicas.
:param rse_expression: Boolean string expression to give the list of RSEs.
:param weight: If the weighting option of the replication rule is used, the choice of RSEs takes their weight into account.
:param lifetime: The lifetime of the replication rules (in seconds).
:param grouping: ALL - All files will be replicated to the same RSE.
DATASET - All files in the same dataset will be replicated to the same RSE.
NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all.
:param account: The account owning the rule.
:param locked: If the rule is locked, it cannot be deleted.
:param subscription_id: The subscription_id, if the rule is created by a subscription.
:param source_replica_expression: Only use replicas from this RSE as sources.
:param activity: Activity to be passed on to the conveyor.
:param notify: Notification setting of the rule.
:purge purge_replicas: The purge setting to delete replicas immediately after rule deletion.
:param ignore_availability: Option to ignore the availability of RSEs.
:param issuer: The issuing account of this operation.
:returns: List of created replication rules.
"""
if account is None:
account = issuer
kwargs = {'dids': dids, 'copies': copies, 'rse_expression': rse_expression, 'weight': weight, 'lifetime': lifetime,
'grouping': grouping, 'account': account, 'locked': locked, 'subscription_id': subscription_id,
'source_replica_expression': source_replica_expression, 'notify': notify, 'activity': activity,
'purge_replicas': purge_replicas, 'ignore_availability': ignore_availability}
validate_schema(name='activity', obj=kwargs['activity'])
if not has_permission(issuer=issuer, action='add_rule', kwargs=kwargs):
raise AccessDenied('Account %s can not add replication rule' % (issuer))
return rule.add_rule(account=account,
dids=dids,
copies=copies,
rse_expression=rse_expression,
grouping=grouping,
weight=weight,
lifetime=lifetime,
locked=locked,
subscription_id=subscription_id,
source_replica_expression=source_replica_expression,
activity=activity,
notify=notify,
purge_replicas=purge_replicas,
ignore_availability=ignore_availability)
def attach_dids_to_dids(attachments, issuer, ignore_duplicate=False):
"""
Append content to dids.
:param attachments: The contents.
:param issuer: The issuer account.
:param ignore_duplicate: If True, ignore duplicate entries.
"""
validate_schema(name='attachments', obj=attachments)
if not rucio.api.permission.has_permission(issuer=issuer, action='attach_dids_to_dids', kwargs={'attachments': attachments}):
raise rucio.common.exception.AccessDenied('Account %s can not add data identifiers' % (issuer))
return did.attach_dids_to_dids(attachments=attachments, account=issuer,
ignore_duplicate=ignore_duplicate)
def attach_dids(scope, name, attachment, issuer):
"""
Append content to data did.
:param attachment: The attachment.
:param issuer: The issuer account.
"""
validate_schema(name='attachment', obj=attachment)
kwargs = {'scope': scope, 'name': name, 'attachment': attachment}
if not rucio.api.permission.has_permission(issuer=issuer, action='attach_dids', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not add data identifiers to %s:%s' % (issuer, scope, name))
return did.attach_dids(scope=scope, name=name, dids=attachment['dids'],
account=attachment.get('account', issuer), rse=attachment.get('rse'))
def add_scope(scope, account, issuer):
"""
Creates a scope for an account.
:param account: The account name.
:param scope: The scope identifier.
:param issuer: The issuer account.
"""
validate_schema(name='scope', obj=scope)
kwargs = {'scope': scope, 'account': account}
if not rucio.api.permission.has_permission(issuer=issuer, action='add_scope', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not add scope' % (issuer))
core_scope.add_scope(scope, account)
def add_account_attribute(key, value, account, issuer):
"""
Add an attribute to an account.
:param key: attribute key.
:param value: attribute value.
:param account: The account name.
:param issuer: The issuer account_core.
"""
validate_schema(name="account_attribute", obj=key)
validate_schema(name="account_attribute", obj=value)
kwargs = {"account": account, "key": key, "value": value}
if not rucio.api.permission.has_permission(issuer=issuer, action="add_attribute", kwargs=kwargs):
raise rucio.common.exception.AccessDenied("Account %s can not add attributes" % (issuer))
account_core.add_account_attribute(account, key, value)
def add_account(account, type, issuer):
"""
Creates an account with the provided account name, contact information, etc.
:param account: The account name.
:param type: The account type
:param issuer: The issuer account_core.
"""
validate_schema(name="account", obj=account)
kwargs = {"account": account, "type": type}
if not rucio.api.permission.has_permission(issuer=issuer, action="add_account", kwargs=kwargs):
raise rucio.common.exception.AccessDenied("Account %s can not add account" % (issuer))
account_core.add_account(account, AccountType.from_sym(type))
def resurrect(dids, issuer, vo='def'):
"""
Resurrect DIDs.
:param dids: A list of dids.
:param issuer: The issuer account.
:param vo: The VO to act on.
"""
kwargs = {'issuer': issuer}
if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='resurrect', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not resurrect data identifiers' % (issuer))
validate_schema(name='dids', obj=dids, vo=vo)
for d in dids:
d['scope'] = InternalScope(d['scope'], vo=vo)
return did.resurrect(dids=dids)
def delete_replicas(rse, files, issuer, ignore_availability=False):
"""
Bulk delete file replicas.
:param rse: The RSE name.
:param files: The list of files.
:param issuer: The issuer account.
:param ignore_availability: Ignore the RSE blacklisting.
:returns: True is successful, False otherwise
"""
validate_schema(name='r_dids', obj=files)
kwargs = {'rse': rse}
if not permission.has_permission(issuer=issuer, action='delete_replicas', kwargs=kwargs):
raise exception.AccessDenied('Account %s can not delete file replicas on %s' % (issuer, rse))
if not permission.has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs):
ignore_availability = False
replica.delete_replicas(rse=rse, files=files, ignore_availability=ignore_availability)
def update_replicas_states(rse, files, issuer):
"""
Update File replica information and state.
:param rse: The RSE name.
:param files: The list of files.
:param issuer: The issuer account.
"""
validate_schema(name='dids', obj=files)
kwargs = {'rse': rse}
if not permission.has_permission(issuer=issuer, action='update_replicas_states', kwargs=kwargs):
raise exception.AccessDenied('Account %s can not update file replicas state on %s' % (issuer, rse))
replicas = []
for file in files:
rep = file
rep['rse'] = rse
replicas.append(rep)
replica.update_replicas_states(replicas=replicas)
def add_subscription(name, account, filter, replication_rules, comments, lifetime, retroactive, dry_run):
"""
Adds a new subscription which will be verified against every new added file and dataset
:param account: Account identifier
:type account: String
:param name: Name of the subscription
:type: String
:param filter: Dictionary of attributes by which the input data should be filtered
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
:type filter: Dict
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
:type replication_rules: Dict
:param comments: Comments for the subscription
:type comments: String
:param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime
:type lifetime: Integer or False
:param retroactive: Flag to know if the subscription should be applied on previous data
:type retroactive: Boolean
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
:type dry_run: Boolean
:returns: subscription_id
:rtype: String
"""
accepted_keys = ['datatype', 'prod_step', 'stream_name', 'project', 'scope', 'pattern', 'type', 'account', 'excluded_pattern', 'grouping', 'group', 'provenance']
try:
if filter:
if type(filter) != dict:
raise TypeError('filter should be a dict')
else:
for key in filter:
if key not in accepted_keys:
raise InvalidMetadata('Invalid metadata <%s>' % (key))
validate_schema(name='subscription_filter', obj=filter)
if replication_rules and type(replication_rules) != list:
raise TypeError('replication_rules should be a list')
except ValueError, e:
raise TypeError(e)
def add_rse(rse, issuer, deterministic=True, volatile=False, city=None, region_code=None, country_name=None, continent=None, time_zone=None, ISP=None, staging_area=False):
"""
Creates a new Rucio Storage Element(RSE).
:param rse: The RSE name.
:param issuer: The issuer account.
:param deterministic: Boolean to know if the pfn is generated deterministically.
:param volatile: Boolean for RSE cache.
:param city: City for the RSE.
:param region_code: The region code for the RSE.
:param country_name: The country.
:param continent: The continent.
:param time_zone: Timezone.
:param staging_area: staging area.
:param ISP: Internet service provider.
"""
validate_schema(name='rse', obj=rse)
kwargs = {'rse': rse}
if not permission.has_permission(issuer=issuer, action='add_rse', kwargs=kwargs):
raise exception.AccessDenied('Account %s can not add RSE' % (issuer))
return rse_module.add_rse(rse, deterministic=deterministic, volatile=volatile, city=city,
region_code=region_code, country_name=country_name, staging_area=staging_area,
continent=continent, time_zone=time_zone, ISP=ISP)