def delete_protocols(self, rse, scheme, hostname=None, port=None):
"""
Deletes matching protocols from RSE. Protocols using the same identifier can be
distinguished by hostname and port.
:param rse: the RSE name.
:param scheme: identifier of the protocol.
:param hostname: hostname of the protocol.
:param port: port of the protocol.
:returns: True if success.
:raises RSEProtocolNotSupported: if no matching protocol entry could be found.
:raises RSENotFound: if the RSE doesn't exist.
:raises AccessDenied: if not authorized.
"""
path = [self.RSE_BASEURL, rse, 'protocols', scheme]
if hostname:
path.append(hostname)
if port:
path.append(str(port))
path = '/'.join(path)
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def get_config(self, section=None, option=None):
"""
Sends the request to get the matching configuration.
:param section: the optional name of the section.
:param option: the optional option within the section.
:return: dictionary containing the configuration.
"""
if section is None and option is not None:
raise ValueError('--section not specified')
path = self.CONFIG_BASEURL
if section is not None and option is None:
path += '/' + section
elif section is not None and option is not None:
path += '/'.join(['', section, option])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return r.json()
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def add_key(self, key, key_type, value_type=None, value_regexp=None):
"""
Sends the request to add a new key.
:param key: the name for the new key.
:param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
:param value_type: the type of the value, if defined.
:param value_regexp: the regular expression that values should match, if defined.
:return: True if key was created successfully.
:raises Duplicate: if key already exists.
"""
path = '/'.join([self.META_BASEURL, key])
url = build_url(choice(self.list_hosts), path=path)
data = dumps({'value_type': value_type and str(value_type),
'value_regexp': value_regexp,
'key_type': key_type})
r = self._send_request(url, type='POST', data=data)
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def list_accounts(self, account_type=None, identity=None):
"""
Sends the request to list all rucio accounts.
:param type: The account type
:param identity: The identity key name. For example x509 DN, or a username.
:return: a list containing account info dictionary for all rucio accounts.
:raises AccountNotFound: if account doesn't exist.
"""
path = '/'.join([self.ACCOUNTS_BASEURL])
url = build_url(choice(self.list_hosts), path=path)
params = {}
if account_type:
params['account_type'] = account_type
if identity:
params['identity'] = identity
r = self._send_request(url, params=params)
if r.status_code == codes.ok:
accounts = self._load_json_data(r)
return accounts
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def list_subscriptions(self, name=None, account=None):
"""
Returns a dictionary with the subscription information :
Examples: ``{'status': 'INACTIVE/ACTIVE/BROKEN', 'last_modified_date': ...}``
:param name: Name of the subscription
:type: String
:param account: Account identifier
:type account: String
:returns: Dictionary containing subscription parameter
:rtype: Dict
:raises: exception.NotFound if subscription is not found
"""
path = self.SUB_BASEURL
if account:
path += '/%s' % (account)
if name:
path += '/%s' % (name)
else:
path += '/'
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def add_did(self, scope, name, type, statuses=None, meta=None, rules=None, lifetime=None):
"""
Add data identifier for a dataset or container.
:param scope: The scope name.
:param name: The data identifier name.
:paran type: The data identifier type (file|dataset|container).
:param statuses: Dictionary with statuses, e.g.g {'monotonic':True}.
:meta: Meta-data associated with the data identifier is represented using key/value pairs in a dictionary.
:rules: Replication rules associated with the data identifier. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ].
:param lifetime: DID's lifetime (in seconds).
"""
path = '/'.join([self.DIDS_BASEURL, scope, name])
url = build_url(choice(self.list_hosts), path=path)
# Build json
data = {'type': type}
if statuses:
data['statuses'] = statuses
if meta:
data['meta'] = meta
if rules:
data['rules'] = rules
if lifetime:
data['lifetime'] = lifetime
r = self._send_request(url, type='POST', data=render_json(**data))
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def add_replication_rule(self, dids, copies, rse_expression, weight=None, lifetime=None, grouping='DATASET', account=None, locked=False, source_replica_expression=None, activity=None, notify='N', purge_replicas=False, ignore_availability=False):
"""
:param dids: The data identifier set.
:param copies: The number of replicas.
:param rse_expression: Boolean string expression to give the list of RSEs.
:param weight: If the weighting option of the replication rule is used, the choice of RSEs takes their weight into account.
:param lifetime: The lifetime of the replication rules (in seconds).
:param grouping: ALL - All files will be replicated to the same RSE.
DATASET - All files in the same dataset will be replicated to the same RSE.
NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all.
:param account: The account owning the rule.
:param locked: If the rule is locked, it cannot be deleted.
:param source_replica_expression: RSE Expression for RSEs to be considered for source replicas.
:param activity: Transfer Activity to be passed to FTS.
:param notify: Notification setting for the rule (Y, N, C).
:param purge_replicas: When the rule gets deleted purge the associated replicas immediately.
:param ignore_availability: Option to ignore the availability of RSEs.
"""
path = self.RULE_BASEURL + '/'
url = build_url(choice(self.list_hosts), path=path)
# TODO remove the subscription_id from the client; It will only be used by the core;
data = dumps({'dids': dids, 'copies': copies, 'rse_expression': rse_expression,
'weight': weight, 'lifetime': lifetime, 'grouping': grouping,
'account': account, 'locked': locked, 'source_replica_expression': source_replica_expression,
'activity': activity, 'notify': notify, 'purge_replicas': purge_replicas,
'ignore_availability': ignore_availability})
r = self._send_request(url, type='POST', data=data)
if r.status_code == codes.created:
return loads(r.text)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def update_protocols(self, rse, scheme, data, hostname=None, port=None):
"""
Updates matching protocols from RSE. Protocol using the same identifier can be
distinguished by hostname and port.
:param rse: the RSE name.
:param scheme: identifier of the protocol.
:param data: A dict providing the new values of the protocol attibutes.
Keys must match column names in database.
:param hostname: hostname of the protocol.
:param port: port of the protocol.
:returns: True if success.
:raises RSEProtocolNotSupported: if no matching protocol entry could be found.
:raises RSENotFound: if the RSE doesn't exist.
:raises KeyNotFound: if invalid data was provided for update.
:raises AccessDenied: if not authorized.
"""
path = [self.RSE_BASEURL, rse, 'protocols', scheme]
if hostname:
path.append(hostname)
if port:
path.append(str(port))
path = '/'.join(path)
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='PUT', data=dumps(data))
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def add_rse(self, rse, **kwargs):
"""
Sends the request to create a new RSE.
:param rse: the name of the rse.
:param deterministic: Boolean to know if the pfn is generated deterministically.
:param volatile: Boolean for RSE cache.
:param city: City for the RSE.
:param region_code: The region code for the RSE.
:param country_name: The country.
:param continent: The continent.
:param time_zone: Timezone.
:param staging_area: Staging area.
:param ISP: Internet service provider.
:return: True if location was created successfully else False.
:raises Duplicate: if rse already exists.
"""
path = 'rses/' + rse
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='POST', data=dumps(kwargs))
if r.status_code == codes.created:
return True
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def __get_token_x509(self):
"""
Sends a request to get an auth token from the server and stores it as a class attribute. Uses x509 authentication.
:returns: True if the token was successfully received. False otherwise.
"""
headers = {'X-Rucio-Account': self.account}
client_cert = None
client_key = None
if self.auth_type == 'x509':
url = build_url(self.auth_host, path='auth/x509')
client_cert = self.creds['client_cert']
if 'client_key' in self.creds:
client_key = self.creds['client_key']
elif self.auth_type == 'x509_proxy':
url = build_url(self.auth_host, path='auth/x509_proxy')
client_cert = self.creds['client_proxy']
if not path.exists(client_cert):
LOG.error('given client cert (%s) doesn\'t exist' % client_cert)
return False
if client_key is not None and not path.exists(client_key):
LOG.error('given client key (%s) doesn\'t exist' % client_key)
retry = 0
if client_key is None:
cert = client_cert
else:
cert = (client_cert, client_key)
while retry <= self.AUTH_RETRIES:
try:
r = self.session.get(url, headers=headers, cert=cert, verify=self.ca_cert)
except SSLError, e:
if 'alert certificate expired' in str(e.message):
raise CannotAuthenticate(str(e))
LOG.warning('SSLError: ' + str(e))
self.ca_cert = False
retry += 1
if retry > self.request_retries:
raise
continue
break
def add_dids(self, dids):
"""
Bulk add datasets/containers.
"""
path = '/'.join([self.DIDS_BASEURL])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='POST', data=render_json_list(dids))
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def ping(self):
"""
Sends a ping request to the rucio server.
:return: Dictonnary with server information
"""
headers = None
path = 'ping'
url = build_url(self.host, path=path)
r = self._send_request(url, headers=headers, type='GET')
if r.status_code == codes.ok:
server_info = loads(r.text)
return server_info
def update_rse(self, rse, parameters):
"""
Update RSE properties like availability or name.
:param rse: the name of the new rse.
:param parameters: A dictionnary with property (name, read, write, delete as keys).
"""
path = 'rses/' + rse
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='PUT', data=dumps(parameters))
if r.status_code == codes.created:
return True
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def delete_rse(self, rse):
"""
Sends the request to delete a rse.
:param rse: the name of the rse.
:return: True if location was created successfully else False.
"""
path = 'rses/' + rse
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def get_dataset_by_guid(self, guid):
"""
Get the parent datasets for a given GUID.
:param guid: The GUID.
:returns: A did
"""
path = '/'.join([self.DIDS_BASEURL, guid, 'guid'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def delete_replication_rule(self, rule_id):
"""
Deletes a replication rule and all associated locks.
:param rule_id: The id of the rule to be deleted
:raises: RuleNotFound, AccessDenied
"""
path = self.RULE_BASEURL + '/' + rule_id
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def get_replication_rule(self, rule_id):
"""
Get a replication rule.
:param rule_id: The id of the rule to be retrieved.
:raises: RuleNotFound
"""
path = self.RULE_BASEURL + '/' + rule_id
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r).next()
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def update_replication_rule(self, rule_id, options):
"""
:param rule_id: The id of the rule to be retrieved.
:param options: Options dictionary.
:raises: RuleNotFound
"""
path = self.RULE_BASEURL + '/' + rule_id
url = build_url(choice(self.list_hosts), path=path)
data = dumps({'options': options})
r = self._send_request(url, type='PUT', data=data)
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def get_account_limits(self, account):
"""
List the account rse limits of this account.
:param account: The account name.
"""
path = '/'.join([self.ACCOUNTS_BASEURL, account, 'limits'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r).next()
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_identities(self, account):
"""
List all identities on an account.
:param account: The account name.
"""
path = '/'.join([self.ACCOUNTS_BASEURL, account, 'identities'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url)
if r.status_code == codes.ok:
identities = self._load_json_data(r)
return identities
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_values(self, key):
"""
Sends the request to list all values for a key.
:return: a list containing the names of all values for a key.
"""
path = self.META_BASEURL + '/' + key + '/'
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url)
if r.status_code == codes.ok:
values = loads(r.text)
return values
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def list_subscription_rules(self, account, name):
"""
List the associated rules of a subscription.
:param account: Account of the subscription.
:param name: Name of the subscription.
"""
path = '/'.join([self.SUB_BASEURL, account, name, 'Rules'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_scopes(self):
"""
Sends the request to list all scopes.
:return: a list containing the names of all scopes.
"""
path = '/'.join(['scopes/'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url)
if r.status_code == codes.ok:
scopes = loads(r.text)
return scopes
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def list_content(self, scope, name):
"""
List data identifier contents.
:param scope: The scope name.
:param name: The data identifier name.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'dids'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def get_metadata(self, scope, name):
"""
Get data identifier metadata
:param scope: The scope name.
:param name: The data identifier name.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'meta'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
meta = self._load_json_data(r)
return meta.next()
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_associated_rules_for_file(self, scope, name):
"""
List the associated rules a file is affected from..
:param scope: The scope name.
:param name: The file name.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'associated_rules'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def delete_metadata(self, scope, name, key):
"""
Delete data identifier metadata
:param scope: The scope name.
:param name: The data identifier.
:param key: the key.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'meta', key])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
raise exc_cls(exc_msg)
def set_status(self, scope, name, **kwargs):
"""
Set data identifier status
:param scope: The scope name.
:param name: The data identifier name.
:param kwargs: Keyword arguments of the form status_name=value.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'status'])
url = build_url(choice(self.list_hosts), path=path)
data = dumps(kwargs)
r = self._send_request(url, type='PUT', data=data)
if r.status_code in (codes.ok, codes.no_content, codes.created):
return True
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def detach_dids(self, scope, name, dids):
"""
Detach data identifier
:param scope: The scope name.
:param name: The data identifier name.
:param dids: The content.
"""
path = '/'.join([self.DIDS_BASEURL, scope, name, 'dids'])
url = build_url(choice(self.list_hosts), path=path)
data = {'dids': dids}
r = self._send_request(url, type='DEL', data=render_json(**data))
if r.status_code == codes.ok:
return True
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def attach_dids_to_dids(self, attachments):
"""
Add dids to dids.
:param attachments: The attachments.
attachments is: [attachment, attachment, ...]
attachment is: {'scope': scope, 'name': name, 'dids': dids}
dids is: [{'scope': scope, 'name': name}, ...]
"""
path = '/'.join([self.DIDS_BASEURL, 'attachments'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='POST', data=render_json_list(attachments))
if r.status_code in (codes.ok, codes.no_content, codes.created):
return True
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
raise exc_cls(exc_msg)
def list_content(self, scope, name):
"""
List data identifier contents.
:param scope: The scope name.
:param name: The data identifier name.
"""
path = '/'.join(
[self.DIDS_BASEURL,
quote_plus(scope),
quote_plus(name), 'dids'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def get_account(self, account):
"""
Sends the request to get information about a given account.
:param account: the name of the account.
:return: a list of attributes for the account. None if failure.
:raises AccountNotFound: if account doesn't exist.
"""
path = '/'.join([self.ACCOUNTS_BASEURL, account])
url = build_url(choice(self.list_hosts), path=path)
res = self._send_request(url)
if res.status_code == codes.ok:
acc = self._load_json_data(res)
return next(acc)
exc_cls, exc_msg = self._get_exception(headers=res.headers,
status_code=res.status_code,
data=res.content)
raise exc_cls(exc_msg)
def add_account_attribute(self, account, key, value):
"""
Adds an attribute to an account.
:param account: The account name.
:param key: The attribute key.
:param value: The attribute value.
"""
data = dumps({'key': key, 'value': value})
path = '/'.join([self.ACCOUNTS_BASEURL, account, 'attr', key])
url = build_url(choice(self.list_hosts), path=path)
res = self._send_request(url, type='POST', data=data)
if res.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=res.headers,
status_code=res.status_code,
data=res.content)
raise exc_cls(exc_msg)
def delete_rse_attribute(self, rse, key):
"""
Sends the request to delete a RSE attribute.
:param rse: the RSE name.
:param key: the attribute key.
:return: True if RSE attribute was deleted successfully else False.
"""
path = '/'.join([self.RSE_BASEURL, rse, 'attr', key])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def update_replicas_states(self, rse, files):
"""
Bulk update the file replicas states from a RSE.
:param rse: the RSE name.
:param files: The list of files. This is a list of DIDs like :
[{'scope': <scope1>, 'name': <name1>}, {'scope': <scope2>, 'name': <name2>}, ...]
:return: True if files have been deleted successfully.
"""
url = build_url(choice(self.list_hosts), path=self.REPLICAS_BASEURL)
data = {'rse': rse, 'files': files}
r = self._send_request(url, type='PUT', data=render_json(**data))
if r.status_code == codes.ok:
return True
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def delete_replication_rule(self, rule_id, purge_replicas=None):
"""
Deletes a replication rule and all associated locks.
:param rule_id: The id of the rule to be deleted
:param purge_replicas: Immediately delete the replicas.
:raises: RuleNotFound, AccessDenied
"""
path = self.RULE_BASEURL + '/' + rule_id
url = build_url(choice(self.list_hosts), path=path)
data = dumps({'purge_replicas': purge_replicas})
r = self._send_request(url, type_='DEL', data=data)
if r.status_code == codes.ok:
return True
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def update_account(self, account, key, value):
""" Update a property of an account.
:param account: Name of the account.
:param key: Account property like status.
:param value: Property value.
"""
data = dumps({key: value})
path = '/'.join([self.ACCOUNTS_BASEURL, account])
url = build_url(choice(self.list_hosts), path=path)
res = self._send_request(url, type='PUT', data=data)
if res.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=res.headers,
status_code=res.status_code,
data=res.content)
raise exc_cls(exc_msg)
def get_global_account_limit(self, account, rse_expression):
"""
List the account limit for the specific RSE expression.
:param account: The account name.
:param rse_expression: The rse expression.
"""
path = '/'.join([
self.ACCOUNTS_BASEURL, account, 'limits', 'global',
quote_plus(rse_expression)
])
url = build_url(choice(self.list_hosts), path=path)
res = self._send_request(url, type='GET')
if res.status_code == codes.ok:
return next(self._load_json_data(res))
exc_cls, exc_msg = self._get_exception(headers=res.headers,
status_code=res.status_code,
data=res.content)
raise exc_cls(exc_msg)
def get_account_usage(self, account, rse=None):
"""
List the account usage for one or all rses of this account.
:param account: The account name.
:param rse: The rse name.
"""
if rse:
path = '/'.join([self.ACCOUNTS_BASEURL, account, 'usage', rse])
else:
path = '/'.join([self.ACCOUNTS_BASEURL, account, 'usage/'])
url = build_url(choice(self.list_hosts), path=path)
res = self._send_request(url, type='GET')
if res.status_code == codes.ok:
return self._load_json_data(res)
else:
exc_cls, exc_msg = self._get_exception(headers=res.headers,
status_code=res.status_code,
data=res.content)
raise exc_cls(exc_msg)
def add_bad_pfns(self, pfns, reason, state, expires_at):
"""
Declare a list of bad replicas.
:param pfns: The list of PFNs.
:param reason: The reason of the loss.
:param state: The state of the replica. Either BAD, SUSPICIOUS, TEMPORARY_UNAVAILABLE
:param expires_at: Specify a timeout for the TEMPORARY_UNAVAILABLE replicas. None for BAD files.
:return: True if PFNs were created successfully.
"""
data = {'reason': reason, 'pfns': pfns, 'state': state, 'expires_at': expires_at}
url = build_url(self.host, path='/'.join([self.REPLICAS_BASEURL, 'bad/pfns']))
headers = {}
r = self._send_request(url, headers=headers, type='POST', data=dumps(data))
if r.status_code == codes.created:
return True
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def list_requests(self, src_rse, dst_rse, request_states):
"""Return latest request details
:return: request information
:rtype: dict
"""
path = '/'.join([self.REQUEST_BASEURL, 'list']) + '?' + '&'.join([
'src_rse={}'.format(src_rse), 'dst_rse={}'.format(dst_rse),
'request_states={}'.format(request_states)
])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type_='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def delete_rse_limits(self, rse, name):
"""
Delete RSE limit information.
:param rse: The RSE name.
:param name: The name of the limit.
:returns: True if successful, otherwise false.
"""
path = [self.RSE_BASEURL, rse, 'limits']
path = '/'.join(path)
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='DEL', data=dumps({'name': name}))
if r.status_code == codes.ok:
return True
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
return exc_cls(exc_msg)
def list_rse_usage_history(self, rse, filters=None):
"""
List RSE usage history information.
:param rse: The RSE name.
:param filters: dictionary of attributes by which the results should be filtered.
:returns: list of dictionnaries.
"""
path = [self.RSE_BASEURL, rse, 'usage', 'history']
path = '/'.join(path)
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET', params=filters)
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def add_value(self, key, value):
"""
Sends the request to add a value to a key.
:param key: the name for key.
:param value: the value.
:return: True if value was created successfully.
:raises Duplicate: if valid already exists.
"""
path = '/'.join([self.META_BASEURL, quote_plus(key)]) + '/'
data = dumps({'value': value})
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='POST', data=data)
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def set_config_option(self, section, option, value):
"""
Sends the request to create or set an option within a section. Missing sections will be created.
:param section: the name of the section.
:param option: the name of the option.
:return: True if option was removed successfully. False otherwise.
"""
path = '/'.join([self.CONFIG_BASEURL, section, option, value])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='PUT')
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def delete_qos_policy(self, rse, qos_policy):
"""
Delete a QoS policy from an RSE.
:param rse_id: The id of the RSE.
:param qos_policy: The QoS policy to delete.
:param session: The database session in use.
:returns: True if successful, silent failure if QoS policy does not exist.
"""
path = [self.RSE_BASEURL, rse, 'qos_policy', qos_policy]
path = '/'.join(path)
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type_='DEL')
if r.status_code == codes.ok:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def delete_account(self, account):
"""
Sends the request to disable an account.
:param account: the name of the account.
:return: True is account was disabled successfully. False otherwise.
:raises AccountNotFound: if account doesn't exist.
"""
path = '/'.join([self.ACCOUNTS_BASEURL, account])
url = build_url(choice(self.list_hosts), path=path)
res = self._send_request(url, type='DEL')
if res.status_code == codes.ok:
return True
exc_cls, exc_msg = self._get_exception(headers=res.headers,
status_code=res.status_code,
data=res.content)
raise exc_cls(exc_msg)
def get_rse(self, rse):
"""
Returns details about the referred RSE.
:param rse: Name of the referred RSE
:returns: A dict containing all attributes of the referred RSE
:raises RSENotFound: if the referred RSE was not found in the database
"""
path = '/'.join([self.RSE_BASEURL, rse])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
rse = loads(r.text)
return rse
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def list_scopes_for_account(self, account):
"""
Sends the request to list all scopes for a rucio account.
:param account: the rucio account to list scopes for.
:return: a list containing the names of all scopes for a rucio account.
:raises AccountNotFound: if account doesn't exist.
:raises ScopeNotFound: if no scopes exist for account.
"""
path = '/'.join([self.SCOPE_BASEURL, account, 'scopes/'])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url)
if r.status_code == codes.ok:
scopes = loads(r.text)
return scopes
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def list_dids_by_meta(self, scope=None, select={}):
"""
Gets all dids matching the values of the provided metadata keys
:param scope: the scope of the search
:param select: the key value pairs to search with(query in json format)
"""
path = '/'.join([self.DIDS_BASEURL, 'list_dids_by_meta'])
payload = {}
if scope is not None:
payload['scope'] = scope
payload['select'] = dumps(select)
url = build_url(choice(self.list_hosts), path=path, params=payload)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return loads(next(self._load_json_data(r)))
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def attach_dids(self, scope, name, dids, rse=None):
"""
Attach data identifier.
:param scope: The scope name.
:param name: The data identifier name.
:param dids: The content.
:param rse: The RSE name when registering replicas.
"""
path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'dids'])
url = build_url(choice(self.list_hosts), path=path)
data = {'dids': dids}
if rse:
data['rse'] = rse
r = self._send_request(url, type_='POST', data=render_json(**data))
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def list_dids(self,
scope,
filters,
type='collection',
long=False,
recursive=False):
"""
List all data identifiers in a scope which match a given pattern.
:param scope: The scope name.
:param filters: A dictionary of key/value pairs like {'name': 'file_name','rse-expression': 'tier0'}.
:param type: The type of the did: 'all'(container, dataset or file)|'collection'(dataset or container)|'dataset'|'container'|'file'
:param long: Long format option to display more information for each DID.
:param recursive: Recursively list DIDs content.
"""
path = '/'.join(
[self.DIDS_BASEURL,
quote_plus(scope), 'dids', 'search'])
payload = {}
if long:
payload['long'] = 1
for k, v in list(filters.items()):
if k in ('created_before', 'created_after'):
payload[k] = date_to_str(v)
else:
payload[k] = v
payload['type'] = type
payload['recursive'] = recursive
url = build_url(choice(self.list_hosts), path=path, params=payload)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
dids = self._load_json_data(r)
return dids
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def __get_token_gss(self):
"""
Sends a request to get an auth token from the server and stores it as a class attribute. Uses Kerberos authentication.
:returns: True if the token was successfully received. False otherwise.
"""
if not EXTRA_MODULES['requests_kerberos']:
raise MissingModuleException(
'The requests-kerberos module is not installed.')
headers = {'X-Rucio-Account': self.account}
url = build_url(self.auth_host, path='auth/gss')
result = None
for retry in range(self.AUTH_RETRIES + 1):
try:
result = self.session.get(url,
headers=headers,
verify=self.ca_cert,
auth=HTTPKerberosAuth())
break
except ConnectionError as error:
LOG.warning('ConnectionError: ' + str(error))
self.ca_cert = False
if retry > self.request_retries:
raise
if not result:
LOG.error('cannot get auth_token')
return False
if result.status_code != codes.ok: # pylint: disable-msg=E1101
exc_cls, exc_msg = self._get_exception(
headers=result.headers,
status_code=result.status_code,
data=result.content)
raise exc_cls(exc_msg)
self.auth_token = result.headers['x-rucio-auth-token']
LOG.debug('got new token')
return True
def __get_token_userpass(self):
"""
Sends a request to get an auth token from the server and stores it as a class attribute. Uses username/password.
:returns: True if the token was successfully received. False otherwise.
"""
headers = {
'X-Rucio-Account': self.account,
'X-Rucio-Username': self.creds['username'],
'X-Rucio-Password': self.creds['password']
}
url = build_url(self.auth_host, path='auth/userpass')
result = None
for retry in range(self.AUTH_RETRIES + 1):
try:
result = self.session.get(url,
headers=headers,
verify=self.ca_cert)
break
except ConnectionError as error:
LOG.warning('ConnectionError: ' + str(error))
self.ca_cert = False
if retry > self.request_retries:
raise
if not result or 'result' not in locals():
LOG.error('cannot get auth_token')
return False
if result.status_code != codes.ok: # pylint: disable-msg=E1101
exc_cls, exc_msg = self._get_exception(
headers=result.headers,
status_code=result.status_code,
data=result.content)
raise exc_cls(exc_msg)
self.auth_token = result.headers['x-rucio-auth-token']
LOG.debug('got new token')
return True
def update_subscription(self, name, account=None, filter=None, replication_rules=None, comments=None, lifetime=None, retroactive=None, dry_run=None, priority=None):
"""
Updates a subscription
:param name: Name of the subscription
:type: String
:param account: Account identifier
:type account: String
:param filter: Dictionary of attributes by which the input data should be filtered
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
:type filter: Dict
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
:type replication_rules: Dict
:param comments: Comments for the subscription
:type comments: String
:param lifetime: Subscription's lifetime (days); False if subscription has no lifetime
:type lifetime: Integer or False
:param retroactive: Flag to know if the subscription should be applied on previous data
:type retroactive: Boolean
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
:type dry_run: Boolean
:param priority: The priority of the subscription
:type priority: Integer
:raises: exception.NotFound if subscription is not found
"""
if not account:
account = self.account
path = self.SUB_BASEURL + '/' + account + '/' + name
url = build_url(choice(self.list_hosts), path=path)
if filter and type(filter) != dict:
raise TypeError('filter should be a dict')
if replication_rules and type(replication_rules) != list:
raise TypeError('replication_rules should be a list')
data = dumps({'options': {'filter': filter, 'replication_rules': replication_rules, 'comments': comments,
'lifetime': lifetime, 'retroactive': retroactive, 'dry_run': dry_run, 'priority': priority}})
r = self._send_request(url, type='PUT', data=data)
if r.status_code == codes.created:
return True
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
raise exc_cls(exc_msg)
def declare_suspicious_file_replicas(self, pfns, reason):
"""
Declare a list of bad replicas.
:param pfns: The list of PFNs.
:param reason: The reason of the loss.
"""
data = {'reason': reason, 'pfns': pfns}
url = build_url(self.host,
path='/'.join([self.REPLICAS_BASEURL, 'suspicious']))
headers = {}
r = self._send_request(url,
headers=headers,
type='POST',
data=dumps(data))
if r.status_code == codes.created:
return loads(r.text)
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def get_did_meta(self, scope, name):
"""
Get all metadata for a given did
:param scope: the scope of did
:param name: the name of the did
"""
path = '/'.join([
self.DIDS_BASEURL,
quote_plus(scope),
quote_plus(name), 'did_meta'
])
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
meta = self._load_json_data(r)
return next(meta)
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def get_dataset_locks_by_rse(self, rse):
"""
Get all dataset locks of the specified rse.
:param rse: the rse of the locks to list.
"""
path = '/'.join([self.LOCKS_BASEURL, rse])
url = build_url(choice(self.list_hosts),
path=path,
params={'did_type': 'dataset'})
result = self._send_request(url)
if result.status_code == codes.ok: # pylint: disable-msg=E1101
locks = self._load_json_data(result)
return locks
else:
exc_cls, exc_msg = self._get_exception(
headers=result.headers, status_code=result.status_code)
raise exc_cls(exc_msg)
def list_rses(self, rse_expression=None):
"""
Sends the request to list all rucio locations(RSEs).
:rse_expression: RSE Expression to use as filter.
:return: a list containing the names of all rucio locations.
"""
if rse_expression:
path = ['rses', "?expression=" + quote(rse_expression)]
path = '/'.join(path)
else:
path = 'rses/'
url = build_url(choice(self.list_hosts), path=path)
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
else:
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)
def list_datasets_per_rse(self, rse, filters=None, limit=None):
"""
List datasets at a RSE.
:param rse: the rse name.
:param filters: dictionary of attributes by which the results should be filtered.
:param limit: limit number.
:returns: A list of dict dataset replicas.
"""
url = build_url(self.host,
path='/'.join([self.REPLICAS_BASEURL, 'rse', rse]))
r = self._send_request(url, type='GET')
if r.status_code == codes.ok:
return self._load_json_data(r)
exc_cls, exc_msg = self._get_exception(headers=r.headers,
status_code=r.status_code,
data=r.content)
raise exc_cls(exc_msg)