def examine_replication_rule(rule_id, issuer, vo='def', session=None): """ Examine a replication rule. :param rule_id: The rule_id to get. :param issuer: The issuing account of this operation. :param vo: The VO of the issuer. :param session: The database session in use. """ kwargs = {'rule_id': rule_id} if is_multi_vo( session=session) and not has_permission(issuer=issuer, vo=vo, action='access_rule_vo', kwargs=kwargs, session=session): raise AccessDenied('Account %s can not access rules at other VOs.' % (issuer)) result = rule.examine_rule(rule_id, session=session) result = api_update_return_dict(result, session=session) if 'transfers' in result: result['transfers'] = [ api_update_return_dict(t, session=session) for t in result['transfers'] ] return result
def examine_replication_rule(rule_id): """ Examine a replication rule. :param rule_id: The rule_id to get. """ result = rule.examine_rule(rule_id) result = api_update_return_dict(result) if 'transfers' in result: result['transfers'] = [api_update_return_dict(t) for t in result['transfers']] return result
def queue_requests(requests, issuer, vo='def', session=None): """ Submit transfer or deletion requests on destination RSEs for data identifiers. :param requests: List of dictionaries containing 'scope', 'name', 'dest_rse_id', 'request_type', 'attributes' :param issuer: Issuing account as a string. :param vo: The VO to act on. :param session: The database session in use. :returns: List of Request-IDs as 32 character hex strings """ kwargs = {'requests': requests, 'issuer': issuer} if not permission.has_permission(issuer=issuer, vo=vo, action='queue_requests', kwargs=kwargs, session=session): raise exception.AccessDenied('%(issuer)s can not queue request' % locals()) for req in requests: req['scope'] = InternalScope(req['scope'], vo=vo) if 'account' in req: req['account'] = InternalAccount(req['account'], vo=vo) new_requests = request.queue_requests(requests, session=session) return [api_update_return_dict(r, session=session) for r in new_requests]
def list_bad_replicas_status(state=BadFilesStatus.BAD, rse=None, younger_than=None, older_than=None, limit=None, list_pfns=False): """ List the bad file replicas history states. Method used by the rucio-ui. :param state: The state of the file (SUSPICIOUS or BAD). :param rse: The RSE name. :param younger_than: datetime object to select bad replicas younger than this date. :param older_than: datetime object to select bad replicas older than this date. :param limit: The maximum number of replicas returned. """ rse_id = None if rse is not None: rse_id = get_rse_id(rse=rse) replicas = replica.list_bad_replicas_status(state=state, rse_id=rse_id, younger_than=younger_than, older_than=older_than, limit=limit, list_pfns=list_pfns) return [api_update_return_dict(r) for r in replicas]
def get_next(request_type, state, issuer, account, vo='def', session=None): """ Retrieve the next request matching the request type and state. :param request_type: Type of the request as a string. :param state: State of the request as a string. :param issuer: Issuing account as a string. :param account: Account identifier as a string. :param vo: The VO to act on. :param session: The database session in use. :returns: Request as a dictionary. """ kwargs = { 'account': account, 'issuer': issuer, 'request_type': request_type, 'state': state } if not permission.has_permission(issuer=issuer, vo=vo, action='get_next', kwargs=kwargs, session=session): raise exception.AccessDenied( '%(account)s cannot get the next request of type %(request_type)s in state %(state)s' % locals()) reqs = request.get_next(request_type, state, session=session) return [api_update_return_dict(r, session=session) for r in reqs]
def get_account_usage(account, rse, issuer): """ Get the account usage and connect it with (if available) the account limits of the account. :param account: The account to read. :param rse: The rse to read (If none, get all). :param issuer: The issuer account. :returns: List of dicts {'rse_id', 'bytes_used', 'files_used', 'bytes_limit'} """ rse_id = None if rse: rse_id = get_rse_id(rse=rse) kwargs = {'account': account, 'rse': rse, 'rse_id': rse_id} if not rucio.api.permission.has_permission( issuer=issuer, action='get_account_usage', kwargs=kwargs): raise rucio.common.exception.AccessDenied( 'Account %s can not list account usage.' % (issuer)) account = InternalAccount(account) if not account_exists(account=account): raise rucio.common.exception.AccountNotFound( 'Account %s does not exist' % (account)) return [ api_update_return_dict(d) for d in account_limit_core.get_account_usage(account=account, rse_id=rse_id) ]
def test_api_update_return_dict(self): """ API: Test the conversion of dictionaries to external representation """ test_dict = { 'account': self.account, 'scope': self.scope, 'rse_expression': 'MOCK|MOCK2', 'rse_id': self.rse_id, 'src_rse_id': self.rse_id, 'source_rse_id': self.rse_id, 'dest_rse_id': self.rse_id, 'destination_rse_id': self.rse_id } value = api_update_return_dict(test_dict) expected = { 'account': self.account_name, 'scope': self.scope_name, 'rse_expression': 'MOCK|MOCK2', 'rse_id': self.rse_id, 'rse': self.rse_name, 'src_rse_id': self.rse_id, 'src_rse': self.rse_name, 'source_rse_id': self.rse_id, 'source_rse': self.rse_name, 'dest_rse_id': self.rse_id, 'dest_rse': self.rse_name, 'destination_rse_id': self.rse_id, 'destination_rse': self.rse_name } assert value == expected
def list_dids_extended(scope, filters, did_type='collection', ignore_case=False, limit=None, offset=None, long=False, recursive=False, vo='def'): """ List dids in a scope. :param scope: The scope name. :param pattern: The wildcard pattern. :param did_type: The type of the did: all(container, dataset, file), collection(dataset or container), dataset, container :param ignore_case: Ignore case distinctions. :param limit: The maximum number of DIDs returned. :param offset: Offset number. :param long: Long format option to display more information for each DID. :param recursive: Recursively list DIDs content. """ validate_schema(name='did_filters', obj=filters, vo=vo) scope = InternalScope(scope, vo=vo) if 'account' in filters: filters['account'] = InternalAccount(filters['account'], vo=vo) if 'scope' in filters: filters['scope'] = InternalScope(filters['scope'], vo=vo) result = did.list_dids_extended(scope=scope, filters=filters, did_type=did_type, ignore_case=ignore_case, limit=limit, offset=offset, long=long, recursive=recursive) for d in result: yield api_update_return_dict(d)
def list_dataset_replicas_bulk(dids, vo='def'): """ :param dids: The list of did dictionaries with scope and name. :param vo: The VO to act on. :returns: A list of dict dataset replicas """ validate_schema(name='r_dids', obj=dids, vo=vo) names_by_scope = dict() for d in dids: if d['scope'] in names_by_scope: names_by_scope[d['scope']].append(d['name']) else: names_by_scope[d['scope']] = [ d['name'], ] names_by_intscope = dict() for scope in names_by_scope: internal_scope = InternalScope(scope, vo=vo) names_by_intscope[internal_scope] = names_by_scope[scope] replicas = replica.list_dataset_replicas_bulk(names_by_intscope) for r in replicas: yield api_update_return_dict(r)
def list_replication_rules(filters={}, vo='def', session=None): """ Lists replication rules based on a filter. :param filters: dictionary of attributes by which the results should be filtered. :param vo: The VO to act on. :param session: The database session in use. """ # If filters is empty, create a new dict to avoid overwriting the function's default if not filters: filters = {} if 'scope' in filters: scope = filters['scope'] else: scope = '*' filters['scope'] = InternalScope(scope=scope, vo=vo) if 'account' in filters: account = filters['account'] else: account = '*' filters['account'] = InternalAccount(account=account, vo=vo) rules = rule.list_rules(filters, session=session) for r in rules: yield api_update_return_dict(r, session=session)
def list_dids(scope, filters, did_type='collection', ignore_case=False, limit=None, offset=None, long=False, recursive=False, vo='def', session=None): """ List dids in a scope. :param scope: The scope name. :param filters: Filter arguments in form supported by the filter engine. :param did_type: The type of the did: all(container, dataset, file), collection(dataset or container), dataset, container :param ignore_case: Ignore case distinctions. :param limit: The maximum number of DIDs returned. :param offset: Offset number. :param long: Long format option to display more information for each DID. :param recursive: Recursively list DIDs content. :param vo: The VO to act on. :param session: The database session in use. """ scope = InternalScope(scope, vo=vo) # replace account and scope in filters with internal representation for or_group in filters: if 'account' in or_group: or_group['account'] = InternalAccount(or_group['account'], vo=vo) if 'scope' in or_group: or_group['account'] = InternalScope(or_group['scope'], vo=vo) result = did.list_dids(scope=scope, filters=filters, did_type=did_type, ignore_case=ignore_case, limit=limit, offset=offset, long=long, recursive=recursive, session=session) for d in result: yield api_update_return_dict(d, session=session)
def get_request_by_did(scope, name, rse, issuer, vo='def', session=None): """ Retrieve a request by its DID for a destination RSE. :param scope: The scope of the data identifier as a string. :param name: The name of the data identifier as a string. :param rse: The destination RSE of the request as a string. :param issuer: Issuing account as a string. :param vo: The VO to act on. :param session: The database session in use. :returns: Request as a dictionary. """ rse_id = get_rse_id(rse=rse, vo=vo, session=session) kwargs = { 'scope': scope, 'name': name, 'rse': rse, 'rse_id': rse_id, 'issuer': issuer } if not permission.has_permission(issuer=issuer, vo=vo, action='get_request_by_did', kwargs=kwargs, session=session): raise exception.AccessDenied( '%(issuer)s cannot retrieve the request DID %(scope)s:%(name)s to RSE %(rse)s' % locals()) scope = InternalScope(scope, vo=vo) req = request.get_request_by_did(scope, name, rse_id, session=session) return api_update_return_dict(req, session=session)
def list_requests(src_rses, dst_rses, states, issuer, vo='def'): """ List all requests in a specific state from a source RSE to a destination RSE. :param src_rses: source RSEs. :param dst_rses: destination RSEs. :param states: list of request states. :param issuer: Issuing account as a string. """ src_rse_ids = [get_rse_id(rse=rse, vo=vo) for rse in src_rses] dst_rse_ids = [get_rse_id(rse=rse, vo=vo) for rse in dst_rses] kwargs = { 'src_rse_id': src_rse_ids, 'dst_rse_id': dst_rse_ids, 'issuer': issuer } if not permission.has_permission( issuer=issuer, vo=vo, action='list_requests', kwargs=kwargs): raise exception.AccessDenied( '%(issuer)s cannot list requests from RSE %(src_rse)s to RSE %(dst_rse)s' % locals()) for req in request.list_requests(src_rse_ids, dst_rse_ids, states): req = req.to_dict() yield api_update_return_dict(req)
def get_rse_usage(rse, issuer, source=None, per_account=False, vo='def', session=None): """ get RSE usage information. :param rse: The RSE name. :param issuer: The issuer account. :param source: dictionary of attributes by which the results should be filtered :param vo: The VO to act on. :param session: The database session in use. :returns: List of RSE usage data. """ rse_id = rse_module.get_rse_id(rse=rse, vo=vo, session=session) usages = rse_module.get_rse_usage(rse_id=rse_id, source=source, per_account=per_account, session=session) for u in usages: u['rse'] = rse if 'account_usages' in u: for account_usage in u['account_usages']: account_usage['account'] = account_usage['account'].external return [api_update_return_dict(u, session=session) for u in usages]
def get_global_account_usage(account, rse_expression, issuer, vo='def'): """ Get the account usage and connect it with (if available) the account limits of the account. :param account: The account to read. :param rse_expression: The rse expression to read (If none, get all). :param issuer: The issuer account. :param vo: The VO to act on. :returns: List of dicts {'rse_id', 'bytes_used', 'files_used', 'bytes_limit'} """ kwargs = {'account': account, 'rse_expression': rse_expression} if not rucio.api.permission.has_permission( issuer=issuer, vo=vo, action='get_global_account_usage', kwargs=kwargs): raise rucio.common.exception.AccessDenied( 'Account %s can not list global account usage.' % (issuer)) account = InternalAccount(account, vo=vo) if not account_exists(account=account): raise rucio.common.exception.AccountNotFound( 'Account %s does not exist' % (account)) return [ api_update_return_dict(d) for d in account_limit_core.get_global_account_usage( account=account, rse_expression=rse_expression) ]
def get_replication_rule(rule_id, estimate_ttc=None): """ Get replication rule by it's id. :param rule_id: The rule_id to get. """ result = rule.get_rule(rule_id, estimate_ttc) return api_update_return_dict(result)
def list_new_dids(type=None, thread=None, total_threads=None, chunk_size=1000): """ List recent identifiers. :param type : The DID type. """ dids = did.list_new_dids(did_type=type and DIDType.from_sym(type), thread=thread, total_threads=total_threads, chunk_size=chunk_size) for d in dids: yield api_update_return_dict(d)
def validate_auth_token(token): """ Validate an authentication token. :param token: Authentication token as a variable-length string. :returns: Tuple(account identifier, token lifetime) if successful, None otherwise. """ return api_update_return_dict(authentication.validate_auth_token(token))
def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None): """ List the bad file replicas summary. Method used by the rucio-ui. :param rse_expression: The RSE expression. :param from_date: The start date. :param to_date: The end date. :param session: The database session in use. """ replicas = replica.get_bad_replicas_summary(rse_expression=rse_expression, from_date=from_date, to_date=to_date) return [api_update_return_dict(r) for r in replicas]
def get_suspicious_files(rse_expression, younger_than=None, nattempts=None, vo='def'): """ List the list of suspicious files on a list of RSEs :param rse_expression: The RSE expression where the suspicious files are located :param younger_than: datetime object to select the suspicious replicas younger than this date. :param nattempts: The number of time the replicas have been declared suspicious :param vo: The VO to act on. """ replicas = replica.get_suspicious_files(rse_expression=rse_expression, younger_than=younger_than, nattempts=nattempts, filter={'vo': vo}) return [api_update_return_dict(r) for r in replicas]
def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None, vo='def'): """ List the bad file replicas summary. Method used by the rucio-ui. :param rse_expression: The RSE expression. :param from_date: The start date. :param to_date: The end date. :param vo: the VO to act on. """ replicas = replica.get_bad_replicas_summary(rse_expression=rse_expression, from_date=from_date, to_date=to_date, filter={'vo': vo}) return [api_update_return_dict(r) for r in replicas]
def get_rse_account_usage(rse): """ Returns the account limit and usage for all for all accounts on a RSE. :param rse: The RSE name. :return: List of dictionnaries. """ rse_id = get_rse_id(rse=rse) return [api_update_return_dict(d) for d in account_limit_core.get_rse_account_usage(rse_id=rse_id)]
def list_dids_by_meta(scope, select): """ List all data identifiers in a scope(optional) which match a given metadata. :param scope: the scope to search in(optional) :param select: the list of key value pairs to filter on """ scope = InternalScope(scope) return [api_update_return_dict(d) for d in did.list_dids_by_meta(scope=scope, select=select)]
def get_dataset_by_guid(guid): """ Get the parent datasets for a given GUID. :param guid: The GUID. :returns: A did """ dids = did.get_dataset_by_guid(guid=guid) for d in dids: yield api_update_return_dict(d)
def list_replication_rule_full_history(scope, name): """ List the rule history of a DID. :param scope: The scope of the DID. :param name: The name of the DID. """ scope = InternalScope(scope) rules = rule.list_rule_full_history(scope, name) for r in rules: yield api_update_return_dict(r)
def list_associated_replication_rules_for_file(scope, name): """ Lists associated replication rules by file. :param scope: Scope of the file.. :param name: Name of the file. """ scope = InternalScope(scope) rules = rule.list_associated_rules_for_file(scope=scope, name=name) for r in rules: yield api_update_return_dict(r)
def list_exceptions(exception_id=None, states=None): """ List exceptions to Lifetime Model. :param id: The id of the exception :param states: The states to filter """ exceptions = lifetime_exception.list_exceptions(exception_id=exception_id, states=states) for e in exceptions: yield api_update_return_dict(e)
def get_rse_account_usage(rse, vo='def', session=None): """ Returns the account limit and usage for all for all accounts on a RSE. :param rse: The RSE name. :param vo: The VO to act on. :param session: The database session in use. :return: List of dictionnaries. """ rse_id = get_rse_id(rse=rse, vo=vo, session=session) return [api_update_return_dict(d, session=session) for d in account_limit_core.get_rse_account_usage(rse_id=rse_id, session=session)]
def get_metadata(scope, name): """ Get data identifier metadata :param scope: The scope name. :param name: The data identifier name. """ scope = InternalScope(scope) d = did.get_metadata(scope=scope, name=name) return api_update_return_dict(d)
def list_archive_content(scope, name): """ List archive contents. :param scope: The archive scope name. :param name: The archive data identifier name. """ scope = InternalScope(scope) dids = did.list_archive_content(scope=scope, name=name) for d in dids: yield api_update_return_dict(d)