def list_accounts(self, account_type=None, identity=None):
"""
Sends the request to list all rucio accounts.
:param type: The account type
:param identity: The identity key name. For example x509 DN, or a username.
:return: a list containing account info dictionary for all rucio accounts.
:raises AccountNotFound: if account doesn't exist.
"""
path = '/'.join([self.ACCOUNTS_BASEURL])
url = build_url(choice(self.list_hosts), path=path)
params = {}
if account_type:
params['account_type'] = account_type
if identity:
params['identity'] = identity
r = self._send_request(url, params=params)
if r.status_code == codes.ok:
accounts = self._load_json_data(r)
return accounts
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def delete_protocols(self, rse, scheme, hostname=None, port=None):
"""
Deletes matching protocols from RSE. Protocols using the same identifier can be
distinguished by hostname and port.
:param rse: the RSE name.
:param scheme: identifier of the protocol.
:param hostname: hostname of the protocol.
:param port: port of the protocol.
:returns: True if success.
:raises RSEProtocolNotSupported: if no matching protocol entry could be found.
:raises RSENotFound: if the RSE doesn't exist.
:raises AccessDenied: if not authorized.
"""
path = [self.RSE_BASEURL, rse, 'protocols', scheme]
if hostname:
path.append(hostname)
if port:
path.append(str(port))
path = '/'.join(path)
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def update_protocols(self, rse, scheme, data, hostname=None, port=None):
"""
Updates matching protocols from RSE. Protocol using the same identifier can be
distinguished by hostname and port.
:param rse: the RSE name.
:param scheme: identifier of the protocol.
:param data: A dict providing the new values of the protocol attibutes.
Keys must match column names in database.
:param hostname: hostname of the protocol.
:param port: port of the protocol.
:returns: True if success.
:raises RSEProtocolNotSupported: if no matching protocol entry could be found.
:raises RSENotFound: if the RSE doesn't exist.
:raises KeyNotFound: if invalid data was provided for update.
:raises AccessDenied: if not authorized.
"""
path = [self.RSE_BASEURL, rse, 'protocols', scheme]
if hostname:
path.append(hostname)
if port:
path.append(str(port))
path = '/'.join(path)
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='PUT', data=dumps(data))
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def get_config(self, section=None, option=None):
"""
Sends the request to get the matching configuration.
:param section: the optional name of the section.
:param option: the optional option within the section.
:return: dictionary containing the configuration.
"""
if section is None and option is not None:
raise ValueError('--section not specified')
path = self.CONFIG_BASEURL
if section is not None and option is None:
path += '/' + section
elif section is not None and option is not None:
path += '/'.join(['', section, option])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return r.json()
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def add_replication_rule(self, dids, copies, rse_expression, weight=None, lifetime=None, grouping='DATASET', account=None, locked=False, source_replica_expression=None, activity=None, notify='N', purge_replicas=False, ignore_availability=False):
"""
:param dids: The data identifier set.
:param copies: The number of replicas.
:param rse_expression: Boolean string expression to give the list of RSEs.
:param weight: If the weighting option of the replication rule is used, the choice of RSEs takes their weight into account.
:param lifetime: The lifetime of the replication rules (in seconds).
:param grouping: ALL - All files will be replicated to the same RSE.
DATASET - All files in the same dataset will be replicated to the same RSE.
NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all.
:param account: The account owning the rule.
:param locked: If the rule is locked, it cannot be deleted.
:param source_replica_expression: RSE Expression for RSEs to be considered for source replicas.
:param activity: Transfer Activity to be passed to FTS.
:param notify: Notification setting for the rule (Y, N, C).
:param purge_replicas: When the rule gets deleted purge the associated replicas immediately.
:param ignore_availability: Option to ignore the availability of RSEs.
"""
path = self.RULE_BASEURL + '/'
url = build_url(choice(self.list_hosts), path=path)
# TODO remove the subscription_id from the client; It will only be used by the core;
data = dumps({'dids': dids, 'copies': copies, 'rse_expression': rse_expression,
'weight': weight, 'lifetime': lifetime, 'grouping': grouping,
'account': account, 'locked': locked, 'source_replica_expression': source_replica_expression,
'activity': activity, 'notify': notify, 'purge_replicas': purge_replicas,
'ignore_availability': ignore_availability})
r = self._send_request(url, type='POST', data=data)
if r.status_code == codes.created:
return loads(r.text)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def add_did(self, scope, name, type, statuses=None, meta=None, rules=None, lifetime=None):
"""
Add data identifier for a dataset or container.
:param scope: The scope name.
:param name: The data identifier name.
:paran type: The data identifier type (file|dataset|container).
:param statuses: Dictionary with statuses, e.g.g {'monotonic':True}.
:meta: Meta-data associated with the data identifier is represented using key/value pairs in a dictionary.
:rules: Replication rules associated with the data identifier. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ].
:param lifetime: DID's lifetime (in seconds).
"""
path = '/'.join([self.DIDS_BASEURL, scope, name])
url = build_url(choice(self.list_hosts), path=path)
# Build json
data = {'type': type}
if statuses:
data['statuses'] = statuses
if meta:
data['meta'] = meta
if rules:
data['rules'] = rules
if lifetime:
data['lifetime'] = lifetime
r = self._send_request(url, type='POST', data=render_json(**data))
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_subscriptions(self, name=None, account=None):
"""
Returns a dictionary with the subscription information :
Examples: ``{'status': 'INACTIVE/ACTIVE/BROKEN', 'last_modified_date': ...}``
:param name: Name of the subscription
:type: String
:param account: Account identifier
:type account: String
:returns: Dictionary containing subscription parameter
:rtype: Dict
:raises: exception.NotFound if subscription is not found
"""
path = self.SUB_BASEURL
if account:
path += '/%s' % (account)
if name:
path += '/%s' % (name)
else:
path += '/'
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def add_key(self, key, key_type, value_type=None, value_regexp=None):
"""
Sends the request to add a new key.
:param key: the name for the new key.
:param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
:param value_type: the type of the value, if defined.
:param value_regexp: the regular expression that values should match, if defined.
:return: True if key was created successfully.
:raises Duplicate: if key already exists.
"""
path = '/'.join([self.META_BASEURL, key])
url = build_url(choice(self.list_hosts), path=path)
data = dumps({'value_type': value_type and str(value_type),
'value_regexp': value_regexp,
'key_type': key_type})
r = self._send_request(url, type='POST', data=data)
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def add_rse(self, rse, **kwargs):
"""
Sends the request to create a new RSE.
:param rse: the name of the rse.
:param deterministic: Boolean to know if the pfn is generated deterministically.
:param volatile: Boolean for RSE cache.
:param city: City for the RSE.
:param region_code: The region code for the RSE.
:param country_name: The country.
:param continent: The continent.
:param time_zone: Timezone.
:param staging_area: Staging area.
:param ISP: Internet service provider.
:return: True if location was created successfully else False.
:raises Duplicate: if rse already exists.
"""
path = 'rses/' + rse
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='POST', data=dumps(kwargs))
if r.status_code == codes.created:
return True
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def add_dids(self, dids):
"""
Bulk add datasets/containers.
"""
path = '/'.join([self.DIDS_BASEURL])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='POST', data=render_json_list(dids))
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def update_rse(self, rse, parameters):
"""
Update RSE properties like availability or name.
:param rse: the name of the new rse.
:param parameters: A dictionnary with property (name, read, write, delete as keys).
"""
path = 'rses/' + rse
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='PUT', data=dumps(parameters))
if r.status_code == codes.created:
return True
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def get_replication_rule(self, rule_id):
"""
Get a replication rule.
:param rule_id: The id of the rule to be retrieved.
:raises: RuleNotFound
"""
path = self.RULE_BASEURL + '/' + rule_id
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r).next()
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_values(self, key):
"""
Sends the request to list all values for a key.
:return: a list containing the names of all values for a key.
"""
path = self.META_BASEURL + '/' + key + '/'
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url)
if r.status_code == codes.ok:
values = loads(r.text)
return values
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def update_replication_rule(self, rule_id, options):
"""
:param rule_id: The id of the rule to be retrieved.
:param options: Options dictionary.
:raises: RuleNotFound
"""
path = self.RULE_BASEURL + '/' + rule_id
url = build_url(choice(self.list_hosts), path=path)
data = dumps({'options': options})
r = self._send_request(url, type='PUT', data=data)
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def get_dataset_by_guid(self, guid):
"""
Get the parent datasets for a given GUID.
:param guid: The GUID.
:returns: A did
"""
path = '/'.join([self.DIDS_BASEURL, guid, 'guid'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def delete_replication_rule(self, rule_id):
"""
Deletes a replication rule and all associated locks.
:param rule_id: The id of the rule to be deleted
:raises: RuleNotFound, AccessDenied
"""
path = self.RULE_BASEURL + '/' + rule_id
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_identities(self, account):
"""
List all identities on an account.
:param account: The account name.
"""
path = '/'.join([self.ACCOUNTS_BASEURL, account, 'identities'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url)
if r.status_code == codes.ok:
identities = self._load_json_data(r)
return identities
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def delete_rse(self, rse):
"""
Sends the request to delete a rse.
:param rse: the name of the rse.
:return: True if location was created successfully else False.
"""
path = 'rses/' + rse
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def get_account_limits(self, account):
"""
List the account rse limits of this account.
:param account: The account name.
"""
path = '/'.join([self.ACCOUNTS_BASEURL, account, 'limits'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r).next()
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_subscription_rules(self, account, name):
"""
List the associated rules of a subscription.
:param account: Account of the subscription.
:param name: Name of the subscription.
"""
path = '/'.join([self.SUB_BASEURL, account, name, 'Rules'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_content(self, scope, name):
"""
List data identifier contents.
:param scope: The scope name.
:param name: The data identifier name.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'dids'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_scopes(self):
"""
Sends the request to list all scopes.
:return: a list containing the names of all scopes.
"""
path = '/'.join(['scopes/'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url)
if r.status_code == codes.ok:
scopes = loads(r.text)
return scopes
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def list_associated_rules_for_file(self, scope, name):
"""
List the associated rules a file is affected from..
:param scope: The scope name.
:param name: The file name.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'associated_rules'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def get_metadata(self, scope, name):
"""
Get data identifier metadata
:param scope: The scope name.
:param name: The data identifier name.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'meta'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
meta = self._load_json_data(r)
return meta.next()
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_rse_attributes(self, rse):
"""
Sends the request to get RSE attributes.
:param rse: The RSE name.
:return: True if RSE attribute was created successfully else False.
"""
path = '/'.join([self.RSE_BASEURL, rse, 'attr/'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
attributes = loads(r.text)
return attributes
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def detach_dids(self, scope, name, dids):
"""
Detach data identifier
:param scope: The scope name.
:param name: The data identifier name.
:param dids: The content.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'dids'])
url = build_url(choice(self.list_hosts), path=path)
data = {'dids': dids}
r = self._send_request(url, type='DEL', data=render_json(**data))
if r.status_code == codes.ok:
return True
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def delete_metadata(self, scope, name, key):
"""
Delete data identifier metadata
:param scope: The scope name.
:param name: The data identifier.
:param key: the key.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'meta', key])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def attach_dids_to_dids(self, attachments):
"""
Add dids to dids.
:param attachments: The attachments.
attachments is: [attachment, attachment, ...]
attachment is: {'scope': scope, 'name': name, 'dids': dids}
dids is: [{'scope': scope, 'name': name}, ...]
"""
path = '/'.join([self.DIDS_BASEURL, 'attachments'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='POST', data=render_json_list(attachments))
if r.status_code in (codes.ok, codes.no_content, codes.created):
return True
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def set_status(self, scope, name, **kwargs):
"""
Set data identifier status
:param scope: The scope name.
:param name: The data identifier name.
:param kwargs: Keyword arguments of the form status_name=value.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'status'])
url = build_url(choice(self.list_hosts), path=path)
data = dumps(kwargs)
r = self._send_request(url, type='PUT', data=data)
if r.status_code in (codes.ok, codes.no_content, codes.created):
return True
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def get_account_usage(self, account, rse=None):
"""
List the account usage for one or all rses of this account.
:param account: The account name.
:param rse: The rse name.
"""
if rse:
path = '/'.join([self.ACCOUNTS_BASEURL, account, 'usage', rse])
else:
path = '/'.join([self.ACCOUNTS_BASEURL, account, 'usage/'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def add_identity(self, account, identity, authtype, email, default=False):
"""
Adds a membership association between identity and account.
:param account: The account name.
:param identity: The identity key name. For example x509 DN, or a username.
:param authtype: The type of the authentication (x509, gss, userpass).
:param default: If True, the account should be used by default with the provided identity.
:param email: The Email address associated with the identity.
"""
data = dumps({'identity': identity, 'authtype': authtype, 'default': default, 'email': email})
path = '/'.join([self.ACCOUNTS_BASEURL, account, 'identities'])
url = build_url(choice(self.list_hosts), path=path)
res = self._send_request(url, type='POST', data=data)
if res.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
raise exc_cls(exc_msg)
def list_parent_dids(self, scope, name):
"""
List parent dataset/containers of a did.
:param scope: The scope.
:param name: The name.
"""
path = '/'.join([
self.DIDS_BASEURL,
quote_plus(scope),
quote_plus(name), 'parents'
])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def delete_metadata(self, scope, name, key):
"""
Delete data identifier metadata
:param scope: The scope name.
:param name: The data identifier.
:param key: the key.
"""
path = '/'.join([
self.DIDS_BASEURL,
quote_plus(scope),
quote_plus(name), 'meta', key
])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def delete_replication_rule(self, rule_id, purge_replicas=None):
"""
Deletes a replication rule and all associated locks.
:param rule_id: The id of the rule to be deleted
:param purge_replicas: Immediately delete the replicas.
:raises: RuleNotFound, AccessDenied
"""
path = self.RULE_BASEURL + '/' + rule_id
url = build_url(choice(self.list_hosts), path=path)
data = dumps({'purge_replicas': purge_replicas})
r = self._send_request(url, type='DEL', data=data)
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def add_did_meta(self, scope, name, meta):
"""
Insert metadata to the json column of a did, updates key if already present
:param scope: the scope of did
:param name: the name of the did
:param meta: the metadata to be inserted or updated(in json format)
"""
path = '/'.join([
self.DIDS_BASEURL,
quote_plus(scope),
quote_plus(name), 'did_meta'
])
url = build_url(choice(self.list_hosts), path=path)
data = dumps(meta)
r = self._send_request(url, type='POST', data=data)
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def set_rse_usage(self, rse, source, used, free):
"""
Set RSE usage information.
:param rse: the RSE name.
:param source: the information source, e.g. srm.
:param used: the used space in bytes.
:param free: the free in bytes.
:returns: True if successful, otherwise false.
"""
path = [self.RSE_BASEURL, rse, 'usage']
path = '/'.join(path)
url = build_url(choice(self.list_hosts), path=path)
data = {'source': source, 'used': used, 'free': free}
r = self._send_request(url, type='PUT', data=dumps(data))
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def add_account(self, account, type, email):
"""
Sends the request to create a new account.
:param account: the name of the account.
:param type: The account type
:param email: The Email address associated with the account.
:return: True if account was created successfully else False.
:raises Duplicate: if account already exists.
"""
data = dumps({'type': type, 'email': email})
path = '/'.join([self.ACCOUNTS_BASEURL, account])
url = build_url(choice(self.list_hosts), path=path)
res = self._send_request(url, type='POST', data=data)
if res.status_code == codes.created:
return True
exc_cls, exc_msg = self._get_exception(headers=res.headers,
status_code=res.status_code,
data=res.content)
raise exc_cls(exc_msg)
def scope_list(self, scope, name=None, recursive=False):
"""
List data identifiers in a scope.
:param scope: The scope name.
:param name: The data identifier name.
:param recursive: boolean, True or False.
"""
payload = {}
path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), ''])
if name:
payload['name'] = name
if recursive:
payload['recursive'] = True
url = build_url(choice(self.list_hosts), path=path, params=payload)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def add_qos_policy(self, rse, qos_policy):
"""
Add a QoS policy from an RSE.
:param rse_id: The id of the RSE.
:param qos_policy: The QoS policy to add.
:param session: The database session in use.
:raises Duplicate: If the QoS policy already exists.
:returns: True if successful, except otherwise.
"""
path = [self.RSE_BASEURL, rse, 'qos_policy', qos_policy]
path = '/'.join(path)
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='POST')
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def add_rse_attribute(self, rse, key, value):
"""
Sends the request to add a RSE attribute.
:param rse: the name of the rse.
:param key: the attribute key.
:param value: the attribute value.
:return: True if RSE attribute was created successfully else False.
:raises Duplicate: if RSE attribute already exists.
"""
path = '/'.join([self.RSE_BASEURL, rse, 'attr', key])
url = build_url(choice(self.list_hosts), path=path)
data = dumps({'value': value})
r = self._send_request(url, type='POST', data=data)
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def get_metadata(self, scope, name, plugin='DID_COLUMN'):
"""
Get data identifier metadata
:param scope: The scope name.
:param name: The data identifier name.
"""
path = '/'.join(
[self.DIDS_BASEURL,
quote_plus(scope),
quote_plus(name), 'meta'])
url = build_url(choice(self.list_hosts), path=path)
payload = {}
payload['plugin'] = plugin
r = self._send_request(url, type='GET', params=payload)
if r.status_code == codes.ok:
meta = self._load_json_data(r)
return next(meta)
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def list_file_replicas(self, scope, lfn):
"""
List file replicas.
:param scope: the scope.
:param lfn: the lfn.
:return: List of replicas.
"""
path = '/'.join([self.BASEURL, scope, lfn, 'rses'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
rses = loads(r.text)
return rses
else:
print r.status_code
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def list_request_by_did(self, name, rse, scope=None):
"""Return latest request details for a DID
:param name: DID
:type name: str
:param rse: Destination RSE name
:type rse: str
:param scope: rucio scope, defaults to None
:param scope: str, optional
:raises exc_cls: from BaseClient._get_exception
:return: request information
:rtype: dict
"""
path = '/'.join(['requests', quote_plus(scope), quote_plus(name), rse])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return next(self._load_json_data(r))
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def set_local_account_limit(self, account, rse, bytes):
"""
Sends the request to set an account limit for an account.
:param account: The name of the account.
:param rse: The rse name.
:param bytes: An integer with the limit in bytes.
:return: True if quota was created successfully else False.
"""
data = dumps({'bytes': bytes})
path = '/'.join([self.ACCOUNTLIMIT_BASEURL, 'local', account, rse])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='POST', data=data)
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def delete_local_account_limit(self, account, rse):
"""
Sends the request to remove an account limit.
:param account: The name of the account.
:param rse: The rse name.
:return: True if quota was removed successfully. False otherwise.
:raises AccountNotFound: if account doesn't exist.
"""
path = '/'.join([self.ACCOUNTLIMIT_BASEURL, 'local', account, rse])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def add_scope(self, account, scope):
"""
Sends the request to add a new scope.
:param account: the name of the account to add the scope to.
:param scope: the name of the new scope.
:return: True if scope was created successfully.
:raises Duplicate: if scope already exists.
:raises AccountNotFound: if account doesn't exist.
"""
path = '/'.join(
[self.SCOPE_BASEURL, account, 'scopes',
quote_plus(scope)])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type_='POST')
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def add_protocol(self, rse, params):
"""
Sends the request to create a new protocol for the given RSE.
:param rse: the name of the rse.
:param scheme: identifier of this protocol
:param params: Attributes of the protocol. Supported are:
hostname: hostname for this protocol (default = localhost)
port: port for this protocol (default = 0)
prefix: string used as a prfeix for this protocol when generating the PFN (default = None)
impl: qualified name of the implementation class for this protocol (mandatory)
read: integer representing the priority of this procotol for read operations (default = -1)
write: integer representing the priority of this procotol for write operations (default = -1)
delete: integer representing the priority of this procotol for delete operations (default = -1)
extended_attributes: miscellaneous protocol specific information e.g. spacetoken for SRM (default = None)
:return: True if protocol was created successfully else False.
:raises Duplicate: if protocol with same hostname, port and protocol identifier
already exists for the given RSE.
:raises RSENotFound: if the RSE doesn't exist.
:raises KeyNotFound: if params is missing manadtory attributes to create the
protocol.
:raises AccessDenied: if not authorized.
"""
scheme = params['scheme']
path = '/'.join([self.RSE_BASEURL, rse, 'protocols', scheme])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='POST', data=dumps(params))
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def move_replication_rule(self, rule_id, rse_expression, activity, source_replica_expression):
"""
Move a replication rule to another RSE and, once done, delete the original one.
:param rule_id: Rule to be moved.
:param rse_expression: RSE expression of the new rule.
:param activity: Activity of the new rule.
:param source_replica_expression: Source-Replica-Expression of the new rule.
:raises: RuleNotFound, RuleReplaceFailed
"""
path = self.RULE_BASEURL + '/' + rule_id + '/move'
url = build_url(choice(self.list_hosts), path=path)
data = dumps({
'rule_id': rule_id,
'rse_expression': rse_expression,
'activity': activity,
'source_replica_expression': source_replica_expression
})
r = self._send_request(url, type_='POST', data=data)
if r.status_code == codes.created:
return loads(r.text)
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def get_global_account_usage(self, account, rse_expression=None):
"""
List the account usage for one or all RSE expressions of this account.
:param account: The account name.
:param rse_expression: The rse expression.
"""
if rse_expression:
path = '/'.join([
self.ACCOUNTS_BASEURL, account, 'usage', 'global',
rse_expression
])
else:
path = '/'.join(
[self.ACCOUNTS_BASEURL, account, 'usage', 'global'])
url = build_url(choice(self.list_hosts), path=path)
res = self._send_request(url, type='GET')
if res.status_code == codes.ok:
return self._load_json_data(res)
else:
exc_cls, exc_msg = self._get_exception(headers=res.headers,
status_code=res.status_code,
data=res.content)
raise exc_cls(exc_msg)
def get_dataset_locks(self, scope, name):
"""
Get a dataset locks of the specified dataset.
:param scope: the scope of the did of the locks to list.
:param name: the name of the did of the locks to list.
"""
path = '/'.join(
[self.LOCKS_BASEURL,
quote_plus(scope),
quote_plus(name)])
url = build_url(choice(self.list_hosts),
path=path,
params={'did_type': 'dataset'})
result = self._send_request(url)
if result.status_code == codes.ok: # pylint: disable-msg=E1101
locks = self._load_json_data(result)
return locks
else:
exc_cls, exc_msg = self._get_exception(
headers=result.headers, status_code=result.status_code)
raise exc_cls(exc_msg)
def list_exceptions(self, exception_id=None, states=None):
"""
List exceptions to Lifetime Model.
:param id: The id of the exception
:param states: The states to filter
"""
path = self.LIFETIME_BASEURL + '/'
params = {}
if exception_id:
params['exception_id'] = exception_id
if states:
params['states'] = exception_id
url = build_url(choice(self.list_hosts), path=path, params=params)
result = self._send_request(url)
if result.status_code == codes.ok:
lifetime_exceptions = self._load_json_data(result)
return lifetime_exceptions
else:
exc_cls, exc_msg = self._get_exception(
headers=result.headers, status_code=result.status_code)
raise exc_cls(exc_msg)
def reduce_replication_rule(self,
rule_id,
copies,
exclude_expression=None):
"""
:param rule_id: Rule to be reduced.
:param copies: Number of copies of the new rule.
:param exclude_expression: RSE Expression of RSEs to exclude.
:raises: RuleReplaceFailed, RuleNotFound
"""
path = self.RULE_BASEURL + '/' + rule_id + '/reduce'
url = build_url(choice(self.list_hosts), path=path)
data = dumps({
'copies': copies,
'exclude_expression': exclude_expression
})
r = self._send_request(url, type='POST', data=data)
if r.status_code == codes.ok:
return loads(r.text)
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def add_did(self, scope, name, type, statuses=None, meta=None, rules=None, lifetime=None, dids=None, rse=None):
"""
Add data identifier for a dataset or container.
:param scope: The scope name.
:param name: The data identifier name.
:paran type: The data identifier type (file|dataset|container).
:param statuses: Dictionary with statuses, e.g.g {'monotonic':True}.
:meta: Meta-data associated with the data identifier is represented using key/value pairs in a dictionary.
:rules: Replication rules associated with the data identifier. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ].
:param lifetime: DID's lifetime (in seconds).
:param dids: The content.
:param rse: The RSE name when registering replicas.
"""
path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), quote_plus(name)])
url = build_url(choice(self.list_hosts), path=path)
# Build json
data = {'type': type}
if statuses:
data['statuses'] = statuses
if meta:
data['meta'] = meta
if rules:
data['rules'] = rules
if lifetime:
data['lifetime'] = lifetime
if dids:
data['dids'] = dids
if rse:
data['rse'] = rse
r = self._send_request(url, type='POST', data=render_json(**data))
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def set_metadata_bulk(self, scope, name, meta, recursive=False):
"""
Set data identifier metadata in bulk.
:param scope: The scope name.
:param name: The data identifier name.
:param meta: the metadata key-values.
:type meta: dict
:param recursive: Option to propagate the metadata change to content.
"""
path = '/'.join(
[self.DIDS_BASEURL,
quote_plus(scope),
quote_plus(name), 'meta'])
url = build_url(choice(self.list_hosts), path=path)
data = dumps({'meta': meta, 'recursive': recursive})
r = self._send_request(url, type_='POST', data=data)
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def delete_replicas(self, rse, files, ignore_availability=True):
"""
Bulk delete file replicas from a RSE.
:param rse: the RSE name.
:param files: The list of files. This is a list of DIDs like :
[{'scope': <scope1>, 'name': <name1>}, {'scope': <scope2>, 'name': <name2>}, ...]
:param ignore_availability: Ignore the RSE blacklisting.
:return: True if files have been deleted successfully.
"""
url = build_url(choice(self.list_hosts), path=self.REPLICAS_BASEURL)
data = {
'rse': rse,
'files': files,
'ignore_availability': ignore_availability
}
r = self._send_request(url, type='DEL', data=render_json(**data))
if r.status_code == codes.ok:
return True
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def get_did(self, scope, name, dynamic=False):
"""
Retrieve a single data identifier.
:param scope: The scope name.
:param name: The data identifier name.
:param dynamic: Calculate sizes dynamically when True
"""
path = '/'.join(
[self.DIDS_BASEURL,
quote_plus(scope),
quote_plus(name)])
if dynamic:
path += '?dynamic=True'
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type_='GET')
if r.status_code == codes.ok:
return next(self._load_json_data(r))
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def attach_dids_to_dids(self, attachments, ignore_duplicate=False):
"""
Add dids to dids.
:param attachments: The attachments.
attachments is: [attachment, attachment, ...]
attachment is: {'scope': scope, 'name': name, 'dids': dids}
dids is: [{'scope': scope, 'name': name}, ...]
:param ignore_duplicate: If True, ignore duplicate entries.
"""
path = '/'.join([self.DIDS_BASEURL, 'attachments'])
url = build_url(choice(self.list_hosts), path=path)
data = {
'ignore_duplicate': ignore_duplicate,
'attachments': attachments
}
r = self._send_request(url, type_='POST', data=dumps(data))
if r.status_code in (codes.ok, codes.no_content, codes.created):
return True
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def del_identity(self, account, identity, authtype):
"""
Delete an identity's membership association with an account.
:param account: The account name.
:param identity: The identity key name. For example x509 DN, or a username.
:param authtype: The type of the authentication (x509, gss, userpass).
:param default: If True, the account should be used by default with the provided identity.
"""
data = dumps({'identity': identity, 'authtype': authtype})
path = '/'.join([self.ACCOUNTS_BASEURL, account, 'identities'])
url = build_url(choice(self.list_hosts), path=path)
res = self._send_request(url, type='DEL', data=data)
if res.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=res.headers,
status_code=res.status_code,
data=res.content)
raise exc_cls(exc_msg)
def list_replicas(self,
dids,
schemes=None,
unavailable=False,
all_states=False,
metalink=False,
rse_expression=None,
client_location=None,
sort=None,
domain=None):
"""
List file replicas for a list of data identifiers (DIDs).
:param dids: The list of data identifiers (DIDs) like :
[{'scope': <scope1>, 'name': <name1>}, {'scope': <scope2>, 'name': <name2>}, ...]
:param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...)
:param unavailable: Also include unavailable replicas in the list.
:param metalink: ``False`` (default) retrieves as JSON,
``True`` retrieves as metalink4+xml.
:param rse_expression: The RSE expression to restrict replicas on a set of RSEs.
:param client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'}
:param sort: Sort the replicas: ``geoip`` - based on src/dst IP topographical distance
``closeness`` - based on src/dst closeness
``dynamic`` - Rucio Dynamic Smart Sort (tm)
:param domain: Define the domain. None is fallback to 'wan', otherwise 'wan, 'lan', or 'all'
"""
data = {'dids': dids, 'domain': domain}
if schemes:
data['schemes'] = schemes
if unavailable:
data['unavailable'] = True
data['all_states'] = all_states
if rse_expression:
data['rse_expression'] = rse_expression
if client_location:
data['client_location'] = client_location
if sort:
data['sort'] = sort
url = build_url(choice(self.list_hosts),
path='/'.join([self.REPLICAS_BASEURL, 'list']))
headers = {}
if metalink:
headers['Accept'] = 'application/metalink4+xml'
# pass json dict in querystring
r = self._send_request(url,
headers=headers,
type='POST',
data=dumps(data))
if r.status_code == codes.ok:
if not metalink:
return self._load_json_data(r)
return r.text
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def list_replicas(self, dids, schemes=None, unavailable=False, ignore_availability=True,
all_states=False, metalink=False, rse_expression=None,
client_location=None, sort=None, domain=None,
signature_lifetime=None,
resolve_archives=True, resolve_parents=False,
updated_after=None):
"""
List file replicas for a list of data identifiers (DIDs).
:param dids: The list of data identifiers (DIDs) like :
[{'scope': <scope1>, 'name': <name1>}, {'scope': <scope2>, 'name': <name2>}, ...]
:param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...)
:param unavailable: Also include unavailable replicas in the list (deprecated)
:param ignore_availability: Also include blacklisted replicas into the list
:param metalink: ``False`` (default) retrieves as JSON,
``True`` retrieves as metalink4+xml.
:param rse_expression: The RSE expression to restrict replicas on a set of RSEs.
:param client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'}
:param sort: Sort the replicas: ``geoip`` - based on src/dst IP topographical distance
``closeness`` - based on src/dst closeness
``dynamic`` - Rucio Dynamic Smart Sort (tm)
:param domain: Define the domain. None is fallback to 'wan', otherwise 'wan, 'lan', or 'all'
:param signature_lifetime: If supported, in seconds, restrict the lifetime of the signed PFN.
:param resolve_archives: When set to True, find archives which contain the replicas.
:param resolve_parents: When set to True, find all parent datasets which contain the replicas.
:param updated_after: epoch timestamp or datetime object (UTC time), only return replicas updated after this time
:returns: A list of dictionaries with replica information.
"""
data = {'dids': dids,
'domain': domain}
if schemes:
data['schemes'] = schemes
if unavailable:
data['unavailable'] = True
if ignore_availability is not None:
data['ignore_availability'] = ignore_availability
data['all_states'] = all_states
if rse_expression:
data['rse_expression'] = rse_expression
if client_location:
data['client_location'] = client_location
if sort:
data['sort'] = sort
if updated_after:
if isinstance(updated_after, datetime):
# encode in UTC string with format '%Y-%m-%dT%H:%M:%S' e.g. '2020-03-02T12:01:38'
data['updated_after'] = updated_after.strftime('%Y-%m-%dT%H:%M:%S')
else:
data['updated_after'] = updated_after
if signature_lifetime:
data['signature_lifetime'] = signature_lifetime
data['resolve_archives'] = resolve_archives
data['resolve_parents'] = resolve_parents
url = build_url(choice(self.list_hosts),
path='/'.join([self.REPLICAS_BASEURL, 'list']))
headers = {}
if metalink:
headers['Accept'] = 'application/metalink4+xml'
# pass json dict in querystring
r = self._send_request(url, headers=headers, type='POST', data=dumps(data), stream=True)
if r.status_code == codes.ok:
if not metalink:
return self._load_json_data(r)
return r.text
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
