def examine_replication_rule(rule_id, issuer, vo='def', session=None):
"""
Examine a replication rule.
:param rule_id: The rule_id to get.
:param issuer: The issuing account of this operation.
:param vo: The VO of the issuer.
:param session: The database session in use.
"""
kwargs = {'rule_id': rule_id}
if is_multi_vo(
session=session) and not has_permission(issuer=issuer,
vo=vo,
action='access_rule_vo',
kwargs=kwargs,
session=session):
raise AccessDenied('Account %s can not access rules at other VOs.' %
(issuer))
result = rule.examine_rule(rule_id, session=session)
result = api_update_return_dict(result, session=session)
if 'transfers' in result:
result['transfers'] = [
api_update_return_dict(t, session=session)
for t in result['transfers']
]
return result
def examine_replication_rule(rule_id):
"""
Examine a replication rule.
:param rule_id: The rule_id to get.
"""
result = rule.examine_rule(rule_id)
result = api_update_return_dict(result)
if 'transfers' in result:
result['transfers'] = [api_update_return_dict(t) for t in result['transfers']]
return result
def queue_requests(requests, issuer, vo='def', session=None):
"""
Submit transfer or deletion requests on destination RSEs for data identifiers.
:param requests: List of dictionaries containing 'scope', 'name', 'dest_rse_id', 'request_type', 'attributes'
:param issuer: Issuing account as a string.
:param vo: The VO to act on.
:param session: The database session in use.
:returns: List of Request-IDs as 32 character hex strings
"""
kwargs = {'requests': requests, 'issuer': issuer}
if not permission.has_permission(issuer=issuer,
vo=vo,
action='queue_requests',
kwargs=kwargs,
session=session):
raise exception.AccessDenied('%(issuer)s can not queue request' %
locals())
for req in requests:
req['scope'] = InternalScope(req['scope'], vo=vo)
if 'account' in req:
req['account'] = InternalAccount(req['account'], vo=vo)
new_requests = request.queue_requests(requests, session=session)
return [api_update_return_dict(r, session=session) for r in new_requests]
def list_bad_replicas_status(state=BadFilesStatus.BAD,
rse=None,
younger_than=None,
older_than=None,
limit=None,
list_pfns=False):
"""
List the bad file replicas history states. Method used by the rucio-ui.
:param state: The state of the file (SUSPICIOUS or BAD).
:param rse: The RSE name.
:param younger_than: datetime object to select bad replicas younger than this date.
:param older_than: datetime object to select bad replicas older than this date.
:param limit: The maximum number of replicas returned.
"""
rse_id = None
if rse is not None:
rse_id = get_rse_id(rse=rse)
replicas = replica.list_bad_replicas_status(state=state,
rse_id=rse_id,
younger_than=younger_than,
older_than=older_than,
limit=limit,
list_pfns=list_pfns)
return [api_update_return_dict(r) for r in replicas]
def get_next(request_type, state, issuer, account, vo='def', session=None):
"""
Retrieve the next request matching the request type and state.
:param request_type: Type of the request as a string.
:param state: State of the request as a string.
:param issuer: Issuing account as a string.
:param account: Account identifier as a string.
:param vo: The VO to act on.
:param session: The database session in use.
:returns: Request as a dictionary.
"""
kwargs = {
'account': account,
'issuer': issuer,
'request_type': request_type,
'state': state
}
if not permission.has_permission(issuer=issuer,
vo=vo,
action='get_next',
kwargs=kwargs,
session=session):
raise exception.AccessDenied(
'%(account)s cannot get the next request of type %(request_type)s in state %(state)s'
% locals())
reqs = request.get_next(request_type, state, session=session)
return [api_update_return_dict(r, session=session) for r in reqs]
def get_account_usage(account, rse, issuer):
"""
Get the account usage and connect it with (if available) the account limits of the account.
:param account: The account to read.
:param rse: The rse to read (If none, get all).
:param issuer: The issuer account.
:returns: List of dicts {'rse_id', 'bytes_used', 'files_used', 'bytes_limit'}
"""
rse_id = None
if rse:
rse_id = get_rse_id(rse=rse)
kwargs = {'account': account, 'rse': rse, 'rse_id': rse_id}
if not rucio.api.permission.has_permission(
issuer=issuer, action='get_account_usage', kwargs=kwargs):
raise rucio.common.exception.AccessDenied(
'Account %s can not list account usage.' % (issuer))
account = InternalAccount(account)
if not account_exists(account=account):
raise rucio.common.exception.AccountNotFound(
'Account %s does not exist' % (account))
return [
api_update_return_dict(d)
for d in account_limit_core.get_account_usage(account=account,
rse_id=rse_id)
]
def test_api_update_return_dict(self):
""" API: Test the conversion of dictionaries to external representation """
test_dict = {
'account': self.account,
'scope': self.scope,
'rse_expression': 'MOCK|MOCK2',
'rse_id': self.rse_id,
'src_rse_id': self.rse_id,
'source_rse_id': self.rse_id,
'dest_rse_id': self.rse_id,
'destination_rse_id': self.rse_id
}
value = api_update_return_dict(test_dict)
expected = {
'account': self.account_name,
'scope': self.scope_name,
'rse_expression': 'MOCK|MOCK2',
'rse_id': self.rse_id,
'rse': self.rse_name,
'src_rse_id': self.rse_id,
'src_rse': self.rse_name,
'source_rse_id': self.rse_id,
'source_rse': self.rse_name,
'dest_rse_id': self.rse_id,
'dest_rse': self.rse_name,
'destination_rse_id': self.rse_id,
'destination_rse': self.rse_name
}
assert value == expected
def list_dids_extended(scope, filters, did_type='collection', ignore_case=False, limit=None, offset=None, long=False, recursive=False, vo='def'):
"""
List dids in a scope.
:param scope: The scope name.
:param pattern: The wildcard pattern.
:param did_type: The type of the did: all(container, dataset, file), collection(dataset or container), dataset, container
:param ignore_case: Ignore case distinctions.
:param limit: The maximum number of DIDs returned.
:param offset: Offset number.
:param long: Long format option to display more information for each DID.
:param recursive: Recursively list DIDs content.
"""
validate_schema(name='did_filters', obj=filters, vo=vo)
scope = InternalScope(scope, vo=vo)
if 'account' in filters:
filters['account'] = InternalAccount(filters['account'], vo=vo)
if 'scope' in filters:
filters['scope'] = InternalScope(filters['scope'], vo=vo)
result = did.list_dids_extended(scope=scope, filters=filters, did_type=did_type, ignore_case=ignore_case,
limit=limit, offset=offset, long=long, recursive=recursive)
for d in result:
yield api_update_return_dict(d)
def list_dataset_replicas_bulk(dids, vo='def'):
"""
:param dids: The list of did dictionaries with scope and name.
:param vo: The VO to act on.
:returns: A list of dict dataset replicas
"""
validate_schema(name='r_dids', obj=dids, vo=vo)
names_by_scope = dict()
for d in dids:
if d['scope'] in names_by_scope:
names_by_scope[d['scope']].append(d['name'])
else:
names_by_scope[d['scope']] = [
d['name'],
]
names_by_intscope = dict()
for scope in names_by_scope:
internal_scope = InternalScope(scope, vo=vo)
names_by_intscope[internal_scope] = names_by_scope[scope]
replicas = replica.list_dataset_replicas_bulk(names_by_intscope)
for r in replicas:
yield api_update_return_dict(r)
def list_replication_rules(filters={}, vo='def', session=None):
"""
Lists replication rules based on a filter.
:param filters: dictionary of attributes by which the results should be filtered.
:param vo: The VO to act on.
:param session: The database session in use.
"""
# If filters is empty, create a new dict to avoid overwriting the function's default
if not filters:
filters = {}
if 'scope' in filters:
scope = filters['scope']
else:
scope = '*'
filters['scope'] = InternalScope(scope=scope, vo=vo)
if 'account' in filters:
account = filters['account']
else:
account = '*'
filters['account'] = InternalAccount(account=account, vo=vo)
rules = rule.list_rules(filters, session=session)
for r in rules:
yield api_update_return_dict(r, session=session)
def list_dids(scope, filters, did_type='collection', ignore_case=False, limit=None, offset=None, long=False, recursive=False, vo='def', session=None):
"""
List dids in a scope.
:param scope: The scope name.
:param filters: Filter arguments in form supported by the filter engine.
:param did_type: The type of the did: all(container, dataset, file), collection(dataset or container), dataset, container
:param ignore_case: Ignore case distinctions.
:param limit: The maximum number of DIDs returned.
:param offset: Offset number.
:param long: Long format option to display more information for each DID.
:param recursive: Recursively list DIDs content.
:param vo: The VO to act on.
:param session: The database session in use.
"""
scope = InternalScope(scope, vo=vo)
# replace account and scope in filters with internal representation
for or_group in filters:
if 'account' in or_group:
or_group['account'] = InternalAccount(or_group['account'], vo=vo)
if 'scope' in or_group:
or_group['account'] = InternalScope(or_group['scope'], vo=vo)
result = did.list_dids(scope=scope, filters=filters, did_type=did_type, ignore_case=ignore_case,
limit=limit, offset=offset, long=long, recursive=recursive, session=session)
for d in result:
yield api_update_return_dict(d, session=session)
def get_request_by_did(scope, name, rse, issuer, vo='def', session=None):
"""
Retrieve a request by its DID for a destination RSE.
:param scope: The scope of the data identifier as a string.
:param name: The name of the data identifier as a string.
:param rse: The destination RSE of the request as a string.
:param issuer: Issuing account as a string.
:param vo: The VO to act on.
:param session: The database session in use.
:returns: Request as a dictionary.
"""
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
kwargs = {
'scope': scope,
'name': name,
'rse': rse,
'rse_id': rse_id,
'issuer': issuer
}
if not permission.has_permission(issuer=issuer,
vo=vo,
action='get_request_by_did',
kwargs=kwargs,
session=session):
raise exception.AccessDenied(
'%(issuer)s cannot retrieve the request DID %(scope)s:%(name)s to RSE %(rse)s'
% locals())
scope = InternalScope(scope, vo=vo)
req = request.get_request_by_did(scope, name, rse_id, session=session)
return api_update_return_dict(req, session=session)
def list_requests(src_rses, dst_rses, states, issuer, vo='def'):
"""
List all requests in a specific state from a source RSE to a destination RSE.
:param src_rses: source RSEs.
:param dst_rses: destination RSEs.
:param states: list of request states.
:param issuer: Issuing account as a string.
"""
src_rse_ids = [get_rse_id(rse=rse, vo=vo) for rse in src_rses]
dst_rse_ids = [get_rse_id(rse=rse, vo=vo) for rse in dst_rses]
kwargs = {
'src_rse_id': src_rse_ids,
'dst_rse_id': dst_rse_ids,
'issuer': issuer
}
if not permission.has_permission(
issuer=issuer, vo=vo, action='list_requests', kwargs=kwargs):
raise exception.AccessDenied(
'%(issuer)s cannot list requests from RSE %(src_rse)s to RSE %(dst_rse)s'
% locals())
for req in request.list_requests(src_rse_ids, dst_rse_ids, states):
req = req.to_dict()
yield api_update_return_dict(req)
def get_rse_usage(rse,
issuer,
source=None,
per_account=False,
vo='def',
session=None):
"""
get RSE usage information.
:param rse: The RSE name.
:param issuer: The issuer account.
:param source: dictionary of attributes by which the results should be filtered
:param vo: The VO to act on.
:param session: The database session in use.
:returns: List of RSE usage data.
"""
rse_id = rse_module.get_rse_id(rse=rse, vo=vo, session=session)
usages = rse_module.get_rse_usage(rse_id=rse_id,
source=source,
per_account=per_account,
session=session)
for u in usages:
u['rse'] = rse
if 'account_usages' in u:
for account_usage in u['account_usages']:
account_usage['account'] = account_usage['account'].external
return [api_update_return_dict(u, session=session) for u in usages]
def get_global_account_usage(account, rse_expression, issuer, vo='def'):
"""
Get the account usage and connect it with (if available) the account limits of the account.
:param account: The account to read.
:param rse_expression: The rse expression to read (If none, get all).
:param issuer: The issuer account.
:param vo: The VO to act on.
:returns: List of dicts {'rse_id', 'bytes_used', 'files_used', 'bytes_limit'}
"""
kwargs = {'account': account, 'rse_expression': rse_expression}
if not rucio.api.permission.has_permission(
issuer=issuer, vo=vo, action='get_global_account_usage',
kwargs=kwargs):
raise rucio.common.exception.AccessDenied(
'Account %s can not list global account usage.' % (issuer))
account = InternalAccount(account, vo=vo)
if not account_exists(account=account):
raise rucio.common.exception.AccountNotFound(
'Account %s does not exist' % (account))
return [
api_update_return_dict(d)
for d in account_limit_core.get_global_account_usage(
account=account, rse_expression=rse_expression)
]
def get_replication_rule(rule_id, estimate_ttc=None):
"""
Get replication rule by it's id.
:param rule_id: The rule_id to get.
"""
result = rule.get_rule(rule_id, estimate_ttc)
return api_update_return_dict(result)
def list_new_dids(type=None, thread=None, total_threads=None, chunk_size=1000):
"""
List recent identifiers.
:param type : The DID type.
"""
dids = did.list_new_dids(did_type=type and DIDType.from_sym(type), thread=thread, total_threads=total_threads, chunk_size=chunk_size)
for d in dids:
yield api_update_return_dict(d)
def validate_auth_token(token):
"""
Validate an authentication token.
:param token: Authentication token as a variable-length string.
:returns: Tuple(account identifier, token lifetime) if successful, None otherwise.
"""
return api_update_return_dict(authentication.validate_auth_token(token))
def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None):
"""
List the bad file replicas summary. Method used by the rucio-ui.
:param rse_expression: The RSE expression.
:param from_date: The start date.
:param to_date: The end date.
:param session: The database session in use.
"""
replicas = replica.get_bad_replicas_summary(rse_expression=rse_expression, from_date=from_date, to_date=to_date)
return [api_update_return_dict(r) for r in replicas]
def get_suspicious_files(rse_expression, younger_than=None, nattempts=None, vo='def'):
"""
List the list of suspicious files on a list of RSEs
:param rse_expression: The RSE expression where the suspicious files are located
:param younger_than: datetime object to select the suspicious replicas younger than this date.
:param nattempts: The number of time the replicas have been declared suspicious
:param vo: The VO to act on.
"""
replicas = replica.get_suspicious_files(rse_expression=rse_expression, younger_than=younger_than, nattempts=nattempts, filter={'vo': vo})
return [api_update_return_dict(r) for r in replicas]
def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None, vo='def'):
"""
List the bad file replicas summary. Method used by the rucio-ui.
:param rse_expression: The RSE expression.
:param from_date: The start date.
:param to_date: The end date.
:param vo: the VO to act on.
"""
replicas = replica.get_bad_replicas_summary(rse_expression=rse_expression, from_date=from_date, to_date=to_date, filter={'vo': vo})
return [api_update_return_dict(r) for r in replicas]
def get_rse_account_usage(rse):
"""
Returns the account limit and usage for all for all accounts on a RSE.
:param rse: The RSE name.
:return: List of dictionnaries.
"""
rse_id = get_rse_id(rse=rse)
return [api_update_return_dict(d) for d in account_limit_core.get_rse_account_usage(rse_id=rse_id)]
def list_dids_by_meta(scope, select):
"""
List all data identifiers in a scope(optional) which match a given metadata.
:param scope: the scope to search in(optional)
:param select: the list of key value pairs to filter on
"""
scope = InternalScope(scope)
return [api_update_return_dict(d) for d in did.list_dids_by_meta(scope=scope, select=select)]
def get_dataset_by_guid(guid):
"""
Get the parent datasets for a given GUID.
:param guid: The GUID.
:returns: A did
"""
dids = did.get_dataset_by_guid(guid=guid)
for d in dids:
yield api_update_return_dict(d)
def list_replication_rule_full_history(scope, name):
"""
List the rule history of a DID.
:param scope: The scope of the DID.
:param name: The name of the DID.
"""
scope = InternalScope(scope)
rules = rule.list_rule_full_history(scope, name)
for r in rules:
yield api_update_return_dict(r)
def list_associated_replication_rules_for_file(scope, name):
"""
Lists associated replication rules by file.
:param scope: Scope of the file..
:param name: Name of the file.
"""
scope = InternalScope(scope)
rules = rule.list_associated_rules_for_file(scope=scope, name=name)
for r in rules:
yield api_update_return_dict(r)
def list_exceptions(exception_id=None, states=None):
"""
List exceptions to Lifetime Model.
:param id: The id of the exception
:param states: The states to filter
"""
exceptions = lifetime_exception.list_exceptions(exception_id=exception_id, states=states)
for e in exceptions:
yield api_update_return_dict(e)
def get_rse_account_usage(rse, vo='def', session=None):
"""
Returns the account limit and usage for all for all accounts on a RSE.
:param rse: The RSE name.
:param vo: The VO to act on.
:param session: The database session in use.
:return: List of dictionnaries.
"""
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
return [api_update_return_dict(d, session=session) for d in account_limit_core.get_rse_account_usage(rse_id=rse_id, session=session)]
def get_metadata(scope, name):
"""
Get data identifier metadata
:param scope: The scope name.
:param name: The data identifier name.
"""
scope = InternalScope(scope)
d = did.get_metadata(scope=scope, name=name)
return api_update_return_dict(d)
def list_archive_content(scope, name):
"""
List archive contents.
:param scope: The archive scope name.
:param name: The archive data identifier name.
"""
scope = InternalScope(scope)
dids = did.list_archive_content(scope=scope, name=name)
for d in dids:
yield api_update_return_dict(d)
