def list_accounts(self, account_type=None, identity=None): """ Sends the request to list all rucio accounts. :param type: The account type :param identity: The identity key name. For example x509 DN, or a username. :return: a list containing account info dictionary for all rucio accounts. :raises AccountNotFound: if account doesn't exist. """ path = '/'.join([self.ACCOUNTS_BASEURL]) url = build_url(choice(self.list_hosts), path=path) params = {} if account_type: params['account_type'] = account_type if identity: params['identity'] = identity r = self._send_request(url, params=params) if r.status_code == codes.ok: accounts = self._load_json_data(r) return accounts else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def delete_protocols(self, rse, scheme, hostname=None, port=None): """ Deletes matching protocols from RSE. Protocols using the same identifier can be distinguished by hostname and port. :param rse: the RSE name. :param scheme: identifier of the protocol. :param hostname: hostname of the protocol. :param port: port of the protocol. :returns: True if success. :raises RSEProtocolNotSupported: if no matching protocol entry could be found. :raises RSENotFound: if the RSE doesn't exist. :raises AccessDenied: if not authorized. """ path = [self.RSE_BASEURL, rse, 'protocols', scheme] if hostname: path.append(hostname) if port: path.append(str(port)) path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def update_protocols(self, rse, scheme, data, hostname=None, port=None): """ Updates matching protocols from RSE. Protocol using the same identifier can be distinguished by hostname and port. :param rse: the RSE name. :param scheme: identifier of the protocol. :param data: A dict providing the new values of the protocol attibutes. Keys must match column names in database. :param hostname: hostname of the protocol. :param port: port of the protocol. :returns: True if success. :raises RSEProtocolNotSupported: if no matching protocol entry could be found. :raises RSENotFound: if the RSE doesn't exist. :raises KeyNotFound: if invalid data was provided for update. :raises AccessDenied: if not authorized. """ path = [self.RSE_BASEURL, rse, 'protocols', scheme] if hostname: path.append(hostname) if port: path.append(str(port)) path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='PUT', data=dumps(data)) if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def get_config(self, section=None, option=None): """ Sends the request to get the matching configuration. :param section: the optional name of the section. :param option: the optional option within the section. :return: dictionary containing the configuration. """ if section is None and option is not None: raise ValueError('--section not specified') path = self.CONFIG_BASEURL if section is not None and option is None: path += '/' + section elif section is not None and option is not None: path += '/'.join(['', section, option]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return r.json() else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def add_replication_rule(self, dids, copies, rse_expression, weight=None, lifetime=None, grouping='DATASET', account=None, locked=False, source_replica_expression=None, activity=None, notify='N', purge_replicas=False, ignore_availability=False): """ :param dids: The data identifier set. :param copies: The number of replicas. :param rse_expression: Boolean string expression to give the list of RSEs. :param weight: If the weighting option of the replication rule is used, the choice of RSEs takes their weight into account. :param lifetime: The lifetime of the replication rules (in seconds). :param grouping: ALL - All files will be replicated to the same RSE. DATASET - All files in the same dataset will be replicated to the same RSE. NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all. :param account: The account owning the rule. :param locked: If the rule is locked, it cannot be deleted. :param source_replica_expression: RSE Expression for RSEs to be considered for source replicas. :param activity: Transfer Activity to be passed to FTS. :param notify: Notification setting for the rule (Y, N, C). :param purge_replicas: When the rule gets deleted purge the associated replicas immediately. :param ignore_availability: Option to ignore the availability of RSEs. """ path = self.RULE_BASEURL + '/' url = build_url(choice(self.list_hosts), path=path) # TODO remove the subscription_id from the client; It will only be used by the core; data = dumps({'dids': dids, 'copies': copies, 'rse_expression': rse_expression, 'weight': weight, 'lifetime': lifetime, 'grouping': grouping, 'account': account, 'locked': locked, 'source_replica_expression': source_replica_expression, 'activity': activity, 'notify': notify, 'purge_replicas': purge_replicas, 'ignore_availability': ignore_availability}) r = self._send_request(url, type='POST', data=data) if r.status_code == codes.created: return loads(r.text) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def add_did(self, scope, name, type, statuses=None, meta=None, rules=None, lifetime=None): """ Add data identifier for a dataset or container. :param scope: The scope name. :param name: The data identifier name. :paran type: The data identifier type (file|dataset|container). :param statuses: Dictionary with statuses, e.g.g {'monotonic':True}. :meta: Meta-data associated with the data identifier is represented using key/value pairs in a dictionary. :rules: Replication rules associated with the data identifier. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ]. :param lifetime: DID's lifetime (in seconds). """ path = '/'.join([self.DIDS_BASEURL, scope, name]) url = build_url(choice(self.list_hosts), path=path) # Build json data = {'type': type} if statuses: data['statuses'] = statuses if meta: data['meta'] = meta if rules: data['rules'] = rules if lifetime: data['lifetime'] = lifetime r = self._send_request(url, type='POST', data=render_json(**data)) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_subscriptions(self, name=None, account=None): """ Returns a dictionary with the subscription information : Examples: ``{'status': 'INACTIVE/ACTIVE/BROKEN', 'last_modified_date': ...}`` :param name: Name of the subscription :type: String :param account: Account identifier :type account: String :returns: Dictionary containing subscription parameter :rtype: Dict :raises: exception.NotFound if subscription is not found """ path = self.SUB_BASEURL if account: path += '/%s' % (account) if name: path += '/%s' % (name) else: path += '/' url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def add_key(self, key, key_type, value_type=None, value_regexp=None): """ Sends the request to add a new key. :param key: the name for the new key. :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection). :param value_type: the type of the value, if defined. :param value_regexp: the regular expression that values should match, if defined. :return: True if key was created successfully. :raises Duplicate: if key already exists. """ path = '/'.join([self.META_BASEURL, key]) url = build_url(choice(self.list_hosts), path=path) data = dumps({'value_type': value_type and str(value_type), 'value_regexp': value_regexp, 'key_type': key_type}) r = self._send_request(url, type='POST', data=data) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def add_rse(self, rse, **kwargs): """ Sends the request to create a new RSE. :param rse: the name of the rse. :param deterministic: Boolean to know if the pfn is generated deterministically. :param volatile: Boolean for RSE cache. :param city: City for the RSE. :param region_code: The region code for the RSE. :param country_name: The country. :param continent: The continent. :param time_zone: Timezone. :param staging_area: Staging area. :param ISP: Internet service provider. :return: True if location was created successfully else False. :raises Duplicate: if rse already exists. """ path = 'rses/' + rse url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST', data=dumps(kwargs)) if r.status_code == codes.created: return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def add_dids(self, dids): """ Bulk add datasets/containers. """ path = '/'.join([self.DIDS_BASEURL]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST', data=render_json_list(dids)) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def update_rse(self, rse, parameters): """ Update RSE properties like availability or name. :param rse: the name of the new rse. :param parameters: A dictionnary with property (name, read, write, delete as keys). """ path = 'rses/' + rse url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='PUT', data=dumps(parameters)) if r.status_code == codes.created: return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def get_replication_rule(self, rule_id): """ Get a replication rule. :param rule_id: The id of the rule to be retrieved. :raises: RuleNotFound """ path = self.RULE_BASEURL + '/' + rule_id url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r).next() else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_values(self, key): """ Sends the request to list all values for a key. :return: a list containing the names of all values for a key. """ path = self.META_BASEURL + '/' + key + '/' url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url) if r.status_code == codes.ok: values = loads(r.text) return values else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def update_replication_rule(self, rule_id, options): """ :param rule_id: The id of the rule to be retrieved. :param options: Options dictionary. :raises: RuleNotFound """ path = self.RULE_BASEURL + '/' + rule_id url = build_url(choice(self.list_hosts), path=path) data = dumps({'options': options}) r = self._send_request(url, type='PUT', data=data) if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def get_dataset_by_guid(self, guid): """ Get the parent datasets for a given GUID. :param guid: The GUID. :returns: A did """ path = '/'.join([self.DIDS_BASEURL, guid, 'guid']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def delete_replication_rule(self, rule_id): """ Deletes a replication rule and all associated locks. :param rule_id: The id of the rule to be deleted :raises: RuleNotFound, AccessDenied """ path = self.RULE_BASEURL + '/' + rule_id url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_identities(self, account): """ List all identities on an account. :param account: The account name. """ path = '/'.join([self.ACCOUNTS_BASEURL, account, 'identities']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url) if r.status_code == codes.ok: identities = self._load_json_data(r) return identities else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def delete_rse(self, rse): """ Sends the request to delete a rse. :param rse: the name of the rse. :return: True if location was created successfully else False. """ path = 'rses/' + rse url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def get_account_limits(self, account): """ List the account rse limits of this account. :param account: The account name. """ path = '/'.join([self.ACCOUNTS_BASEURL, account, 'limits']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r).next() else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_subscription_rules(self, account, name): """ List the associated rules of a subscription. :param account: Account of the subscription. :param name: Name of the subscription. """ path = '/'.join([self.SUB_BASEURL, account, name, 'Rules']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_content(self, scope, name): """ List data identifier contents. :param scope: The scope name. :param name: The data identifier name. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'dids']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_scopes(self): """ Sends the request to list all scopes. :return: a list containing the names of all scopes. """ path = '/'.join(['scopes/']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url) if r.status_code == codes.ok: scopes = loads(r.text) return scopes else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def list_associated_rules_for_file(self, scope, name): """ List the associated rules a file is affected from.. :param scope: The scope name. :param name: The file name. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'associated_rules']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def get_metadata(self, scope, name): """ Get data identifier metadata :param scope: The scope name. :param name: The data identifier name. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'meta']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: meta = self._load_json_data(r) return meta.next() else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_rse_attributes(self, rse): """ Sends the request to get RSE attributes. :param rse: The RSE name. :return: True if RSE attribute was created successfully else False. """ path = '/'.join([self.RSE_BASEURL, rse, 'attr/']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: attributes = loads(r.text) return attributes else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def detach_dids(self, scope, name, dids): """ Detach data identifier :param scope: The scope name. :param name: The data identifier name. :param dids: The content. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'dids']) url = build_url(choice(self.list_hosts), path=path) data = {'dids': dids} r = self._send_request(url, type='DEL', data=render_json(**data)) if r.status_code == codes.ok: return True exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def delete_metadata(self, scope, name, key): """ Delete data identifier metadata :param scope: The scope name. :param name: The data identifier. :param key: the key. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'meta', key]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def attach_dids_to_dids(self, attachments): """ Add dids to dids. :param attachments: The attachments. attachments is: [attachment, attachment, ...] attachment is: {'scope': scope, 'name': name, 'dids': dids} dids is: [{'scope': scope, 'name': name}, ...] """ path = '/'.join([self.DIDS_BASEURL, 'attachments']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST', data=render_json_list(attachments)) if r.status_code in (codes.ok, codes.no_content, codes.created): return True exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def set_status(self, scope, name, **kwargs): """ Set data identifier status :param scope: The scope name. :param name: The data identifier name. :param kwargs: Keyword arguments of the form status_name=value. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'status']) url = build_url(choice(self.list_hosts), path=path) data = dumps(kwargs) r = self._send_request(url, type='PUT', data=data) if r.status_code in (codes.ok, codes.no_content, codes.created): return True exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def get_account_usage(self, account, rse=None): """ List the account usage for one or all rses of this account. :param account: The account name. :param rse: The rse name. """ if rse: path = '/'.join([self.ACCOUNTS_BASEURL, account, 'usage', rse]) else: path = '/'.join([self.ACCOUNTS_BASEURL, account, 'usage/']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def add_identity(self, account, identity, authtype, email, default=False): """ Adds a membership association between identity and account. :param account: The account name. :param identity: The identity key name. For example x509 DN, or a username. :param authtype: The type of the authentication (x509, gss, userpass). :param default: If True, the account should be used by default with the provided identity. :param email: The Email address associated with the identity. """ data = dumps({'identity': identity, 'authtype': authtype, 'default': default, 'email': email}) path = '/'.join([self.ACCOUNTS_BASEURL, account, 'identities']) url = build_url(choice(self.list_hosts), path=path) res = self._send_request(url, type='POST', data=data) if res.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content) raise exc_cls(exc_msg)
def list_parent_dids(self, scope, name): """ List parent dataset/containers of a did. :param scope: The scope. :param name: The name. """ path = '/'.join([ self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'parents' ]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def delete_metadata(self, scope, name, key): """ Delete data identifier metadata :param scope: The scope name. :param name: The data identifier. :param key: the key. """ path = '/'.join([ self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'meta', key ]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def delete_replication_rule(self, rule_id, purge_replicas=None): """ Deletes a replication rule and all associated locks. :param rule_id: The id of the rule to be deleted :param purge_replicas: Immediately delete the replicas. :raises: RuleNotFound, AccessDenied """ path = self.RULE_BASEURL + '/' + rule_id url = build_url(choice(self.list_hosts), path=path) data = dumps({'purge_replicas': purge_replicas}) r = self._send_request(url, type='DEL', data=data) if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def add_did_meta(self, scope, name, meta): """ Insert metadata to the json column of a did, updates key if already present :param scope: the scope of did :param name: the name of the did :param meta: the metadata to be inserted or updated(in json format) """ path = '/'.join([ self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'did_meta' ]) url = build_url(choice(self.list_hosts), path=path) data = dumps(meta) r = self._send_request(url, type='POST', data=data) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def set_rse_usage(self, rse, source, used, free): """ Set RSE usage information. :param rse: the RSE name. :param source: the information source, e.g. srm. :param used: the used space in bytes. :param free: the free in bytes. :returns: True if successful, otherwise false. """ path = [self.RSE_BASEURL, rse, 'usage'] path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) data = {'source': source, 'used': used, 'free': free} r = self._send_request(url, type='PUT', data=dumps(data)) if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def add_account(self, account, type, email): """ Sends the request to create a new account. :param account: the name of the account. :param type: The account type :param email: The Email address associated with the account. :return: True if account was created successfully else False. :raises Duplicate: if account already exists. """ data = dumps({'type': type, 'email': email}) path = '/'.join([self.ACCOUNTS_BASEURL, account]) url = build_url(choice(self.list_hosts), path=path) res = self._send_request(url, type='POST', data=data) if res.status_code == codes.created: return True exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content) raise exc_cls(exc_msg)
def scope_list(self, scope, name=None, recursive=False): """ List data identifiers in a scope. :param scope: The scope name. :param name: The data identifier name. :param recursive: boolean, True or False. """ payload = {} path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), '']) if name: payload['name'] = name if recursive: payload['recursive'] = True url = build_url(choice(self.list_hosts), path=path, params=payload) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def add_qos_policy(self, rse, qos_policy): """ Add a QoS policy from an RSE. :param rse_id: The id of the RSE. :param qos_policy: The QoS policy to add. :param session: The database session in use. :raises Duplicate: If the QoS policy already exists. :returns: True if successful, except otherwise. """ path = [self.RSE_BASEURL, rse, 'qos_policy', qos_policy] path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST') if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def add_rse_attribute(self, rse, key, value): """ Sends the request to add a RSE attribute. :param rse: the name of the rse. :param key: the attribute key. :param value: the attribute value. :return: True if RSE attribute was created successfully else False. :raises Duplicate: if RSE attribute already exists. """ path = '/'.join([self.RSE_BASEURL, rse, 'attr', key]) url = build_url(choice(self.list_hosts), path=path) data = dumps({'value': value}) r = self._send_request(url, type='POST', data=data) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def get_metadata(self, scope, name, plugin='DID_COLUMN'): """ Get data identifier metadata :param scope: The scope name. :param name: The data identifier name. """ path = '/'.join( [self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'meta']) url = build_url(choice(self.list_hosts), path=path) payload = {} payload['plugin'] = plugin r = self._send_request(url, type='GET', params=payload) if r.status_code == codes.ok: meta = self._load_json_data(r) return next(meta) else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def list_file_replicas(self, scope, lfn): """ List file replicas. :param scope: the scope. :param lfn: the lfn. :return: List of replicas. """ path = '/'.join([self.BASEURL, scope, lfn, 'rses']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: rses = loads(r.text) return rses else: print r.status_code exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def list_request_by_did(self, name, rse, scope=None): """Return latest request details for a DID :param name: DID :type name: str :param rse: Destination RSE name :type rse: str :param scope: rucio scope, defaults to None :param scope: str, optional :raises exc_cls: from BaseClient._get_exception :return: request information :rtype: dict """ path = '/'.join(['requests', quote_plus(scope), quote_plus(name), rse]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return next(self._load_json_data(r)) else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def set_local_account_limit(self, account, rse, bytes): """ Sends the request to set an account limit for an account. :param account: The name of the account. :param rse: The rse name. :param bytes: An integer with the limit in bytes. :return: True if quota was created successfully else False. """ data = dumps({'bytes': bytes}) path = '/'.join([self.ACCOUNTLIMIT_BASEURL, 'local', account, rse]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST', data=data) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def delete_local_account_limit(self, account, rse): """ Sends the request to remove an account limit. :param account: The name of the account. :param rse: The rse name. :return: True if quota was removed successfully. False otherwise. :raises AccountNotFound: if account doesn't exist. """ path = '/'.join([self.ACCOUNTLIMIT_BASEURL, 'local', account, rse]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def add_scope(self, account, scope): """ Sends the request to add a new scope. :param account: the name of the account to add the scope to. :param scope: the name of the new scope. :return: True if scope was created successfully. :raises Duplicate: if scope already exists. :raises AccountNotFound: if account doesn't exist. """ path = '/'.join( [self.SCOPE_BASEURL, account, 'scopes', quote_plus(scope)]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type_='POST') if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def add_protocol(self, rse, params): """ Sends the request to create a new protocol for the given RSE. :param rse: the name of the rse. :param scheme: identifier of this protocol :param params: Attributes of the protocol. Supported are: hostname: hostname for this protocol (default = localhost) port: port for this protocol (default = 0) prefix: string used as a prfeix for this protocol when generating the PFN (default = None) impl: qualified name of the implementation class for this protocol (mandatory) read: integer representing the priority of this procotol for read operations (default = -1) write: integer representing the priority of this procotol for write operations (default = -1) delete: integer representing the priority of this procotol for delete operations (default = -1) extended_attributes: miscellaneous protocol specific information e.g. spacetoken for SRM (default = None) :return: True if protocol was created successfully else False. :raises Duplicate: if protocol with same hostname, port and protocol identifier already exists for the given RSE. :raises RSENotFound: if the RSE doesn't exist. :raises KeyNotFound: if params is missing manadtory attributes to create the protocol. :raises AccessDenied: if not authorized. """ scheme = params['scheme'] path = '/'.join([self.RSE_BASEURL, rse, 'protocols', scheme]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST', data=dumps(params)) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def move_replication_rule(self, rule_id, rse_expression, activity, source_replica_expression): """ Move a replication rule to another RSE and, once done, delete the original one. :param rule_id: Rule to be moved. :param rse_expression: RSE expression of the new rule. :param activity: Activity of the new rule. :param source_replica_expression: Source-Replica-Expression of the new rule. :raises: RuleNotFound, RuleReplaceFailed """ path = self.RULE_BASEURL + '/' + rule_id + '/move' url = build_url(choice(self.list_hosts), path=path) data = dumps({ 'rule_id': rule_id, 'rse_expression': rse_expression, 'activity': activity, 'source_replica_expression': source_replica_expression }) r = self._send_request(url, type_='POST', data=data) if r.status_code == codes.created: return loads(r.text) exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def get_global_account_usage(self, account, rse_expression=None): """ List the account usage for one or all RSE expressions of this account. :param account: The account name. :param rse_expression: The rse expression. """ if rse_expression: path = '/'.join([ self.ACCOUNTS_BASEURL, account, 'usage', 'global', rse_expression ]) else: path = '/'.join( [self.ACCOUNTS_BASEURL, account, 'usage', 'global']) url = build_url(choice(self.list_hosts), path=path) res = self._send_request(url, type='GET') if res.status_code == codes.ok: return self._load_json_data(res) else: exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content) raise exc_cls(exc_msg)
def get_dataset_locks(self, scope, name): """ Get a dataset locks of the specified dataset. :param scope: the scope of the did of the locks to list. :param name: the name of the did of the locks to list. """ path = '/'.join( [self.LOCKS_BASEURL, quote_plus(scope), quote_plus(name)]) url = build_url(choice(self.list_hosts), path=path, params={'did_type': 'dataset'}) result = self._send_request(url) if result.status_code == codes.ok: # pylint: disable-msg=E1101 locks = self._load_json_data(result) return locks else: exc_cls, exc_msg = self._get_exception( headers=result.headers, status_code=result.status_code) raise exc_cls(exc_msg)
def list_exceptions(self, exception_id=None, states=None): """ List exceptions to Lifetime Model. :param id: The id of the exception :param states: The states to filter """ path = self.LIFETIME_BASEURL + '/' params = {} if exception_id: params['exception_id'] = exception_id if states: params['states'] = exception_id url = build_url(choice(self.list_hosts), path=path, params=params) result = self._send_request(url) if result.status_code == codes.ok: lifetime_exceptions = self._load_json_data(result) return lifetime_exceptions else: exc_cls, exc_msg = self._get_exception( headers=result.headers, status_code=result.status_code) raise exc_cls(exc_msg)
def reduce_replication_rule(self, rule_id, copies, exclude_expression=None): """ :param rule_id: Rule to be reduced. :param copies: Number of copies of the new rule. :param exclude_expression: RSE Expression of RSEs to exclude. :raises: RuleReplaceFailed, RuleNotFound """ path = self.RULE_BASEURL + '/' + rule_id + '/reduce' url = build_url(choice(self.list_hosts), path=path) data = dumps({ 'copies': copies, 'exclude_expression': exclude_expression }) r = self._send_request(url, type='POST', data=data) if r.status_code == codes.ok: return loads(r.text) exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def add_did(self, scope, name, type, statuses=None, meta=None, rules=None, lifetime=None, dids=None, rse=None): """ Add data identifier for a dataset or container. :param scope: The scope name. :param name: The data identifier name. :paran type: The data identifier type (file|dataset|container). :param statuses: Dictionary with statuses, e.g.g {'monotonic':True}. :meta: Meta-data associated with the data identifier is represented using key/value pairs in a dictionary. :rules: Replication rules associated with the data identifier. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ]. :param lifetime: DID's lifetime (in seconds). :param dids: The content. :param rse: The RSE name when registering replicas. """ path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), quote_plus(name)]) url = build_url(choice(self.list_hosts), path=path) # Build json data = {'type': type} if statuses: data['statuses'] = statuses if meta: data['meta'] = meta if rules: data['rules'] = rules if lifetime: data['lifetime'] = lifetime if dids: data['dids'] = dids if rse: data['rse'] = rse r = self._send_request(url, type='POST', data=render_json(**data)) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def set_metadata_bulk(self, scope, name, meta, recursive=False): """ Set data identifier metadata in bulk. :param scope: The scope name. :param name: The data identifier name. :param meta: the metadata key-values. :type meta: dict :param recursive: Option to propagate the metadata change to content. """ path = '/'.join( [self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'meta']) url = build_url(choice(self.list_hosts), path=path) data = dumps({'meta': meta, 'recursive': recursive}) r = self._send_request(url, type_='POST', data=data) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def delete_replicas(self, rse, files, ignore_availability=True): """ Bulk delete file replicas from a RSE. :param rse: the RSE name. :param files: The list of files. This is a list of DIDs like : [{'scope': <scope1>, 'name': <name1>}, {'scope': <scope2>, 'name': <name2>}, ...] :param ignore_availability: Ignore the RSE blacklisting. :return: True if files have been deleted successfully. """ url = build_url(choice(self.list_hosts), path=self.REPLICAS_BASEURL) data = { 'rse': rse, 'files': files, 'ignore_availability': ignore_availability } r = self._send_request(url, type='DEL', data=render_json(**data)) if r.status_code == codes.ok: return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def get_did(self, scope, name, dynamic=False): """ Retrieve a single data identifier. :param scope: The scope name. :param name: The data identifier name. :param dynamic: Calculate sizes dynamically when True """ path = '/'.join( [self.DIDS_BASEURL, quote_plus(scope), quote_plus(name)]) if dynamic: path += '?dynamic=True' url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type_='GET') if r.status_code == codes.ok: return next(self._load_json_data(r)) else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def attach_dids_to_dids(self, attachments, ignore_duplicate=False): """ Add dids to dids. :param attachments: The attachments. attachments is: [attachment, attachment, ...] attachment is: {'scope': scope, 'name': name, 'dids': dids} dids is: [{'scope': scope, 'name': name}, ...] :param ignore_duplicate: If True, ignore duplicate entries. """ path = '/'.join([self.DIDS_BASEURL, 'attachments']) url = build_url(choice(self.list_hosts), path=path) data = { 'ignore_duplicate': ignore_duplicate, 'attachments': attachments } r = self._send_request(url, type_='POST', data=dumps(data)) if r.status_code in (codes.ok, codes.no_content, codes.created): return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def del_identity(self, account, identity, authtype): """ Delete an identity's membership association with an account. :param account: The account name. :param identity: The identity key name. For example x509 DN, or a username. :param authtype: The type of the authentication (x509, gss, userpass). :param default: If True, the account should be used by default with the provided identity. """ data = dumps({'identity': identity, 'authtype': authtype}) path = '/'.join([self.ACCOUNTS_BASEURL, account, 'identities']) url = build_url(choice(self.list_hosts), path=path) res = self._send_request(url, type='DEL', data=data) if res.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content) raise exc_cls(exc_msg)
def list_replicas(self, dids, schemes=None, unavailable=False, all_states=False, metalink=False, rse_expression=None, client_location=None, sort=None, domain=None): """ List file replicas for a list of data identifiers (DIDs). :param dids: The list of data identifiers (DIDs) like : [{'scope': <scope1>, 'name': <name1>}, {'scope': <scope2>, 'name': <name2>}, ...] :param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...) :param unavailable: Also include unavailable replicas in the list. :param metalink: ``False`` (default) retrieves as JSON, ``True`` retrieves as metalink4+xml. :param rse_expression: The RSE expression to restrict replicas on a set of RSEs. :param client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'} :param sort: Sort the replicas: ``geoip`` - based on src/dst IP topographical distance ``closeness`` - based on src/dst closeness ``dynamic`` - Rucio Dynamic Smart Sort (tm) :param domain: Define the domain. None is fallback to 'wan', otherwise 'wan, 'lan', or 'all' """ data = {'dids': dids, 'domain': domain} if schemes: data['schemes'] = schemes if unavailable: data['unavailable'] = True data['all_states'] = all_states if rse_expression: data['rse_expression'] = rse_expression if client_location: data['client_location'] = client_location if sort: data['sort'] = sort url = build_url(choice(self.list_hosts), path='/'.join([self.REPLICAS_BASEURL, 'list'])) headers = {} if metalink: headers['Accept'] = 'application/metalink4+xml' # pass json dict in querystring r = self._send_request(url, headers=headers, type='POST', data=dumps(data)) if r.status_code == codes.ok: if not metalink: return self._load_json_data(r) return r.text exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def list_replicas(self, dids, schemes=None, unavailable=False, ignore_availability=True, all_states=False, metalink=False, rse_expression=None, client_location=None, sort=None, domain=None, signature_lifetime=None, resolve_archives=True, resolve_parents=False, updated_after=None): """ List file replicas for a list of data identifiers (DIDs). :param dids: The list of data identifiers (DIDs) like : [{'scope': <scope1>, 'name': <name1>}, {'scope': <scope2>, 'name': <name2>}, ...] :param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...) :param unavailable: Also include unavailable replicas in the list (deprecated) :param ignore_availability: Also include blacklisted replicas into the list :param metalink: ``False`` (default) retrieves as JSON, ``True`` retrieves as metalink4+xml. :param rse_expression: The RSE expression to restrict replicas on a set of RSEs. :param client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'} :param sort: Sort the replicas: ``geoip`` - based on src/dst IP topographical distance ``closeness`` - based on src/dst closeness ``dynamic`` - Rucio Dynamic Smart Sort (tm) :param domain: Define the domain. None is fallback to 'wan', otherwise 'wan, 'lan', or 'all' :param signature_lifetime: If supported, in seconds, restrict the lifetime of the signed PFN. :param resolve_archives: When set to True, find archives which contain the replicas. :param resolve_parents: When set to True, find all parent datasets which contain the replicas. :param updated_after: epoch timestamp or datetime object (UTC time), only return replicas updated after this time :returns: A list of dictionaries with replica information. """ data = {'dids': dids, 'domain': domain} if schemes: data['schemes'] = schemes if unavailable: data['unavailable'] = True if ignore_availability is not None: data['ignore_availability'] = ignore_availability data['all_states'] = all_states if rse_expression: data['rse_expression'] = rse_expression if client_location: data['client_location'] = client_location if sort: data['sort'] = sort if updated_after: if isinstance(updated_after, datetime): # encode in UTC string with format '%Y-%m-%dT%H:%M:%S' e.g. '2020-03-02T12:01:38' data['updated_after'] = updated_after.strftime('%Y-%m-%dT%H:%M:%S') else: data['updated_after'] = updated_after if signature_lifetime: data['signature_lifetime'] = signature_lifetime data['resolve_archives'] = resolve_archives data['resolve_parents'] = resolve_parents url = build_url(choice(self.list_hosts), path='/'.join([self.REPLICAS_BASEURL, 'list'])) headers = {} if metalink: headers['Accept'] = 'application/metalink4+xml' # pass json dict in querystring r = self._send_request(url, headers=headers, type='POST', data=dumps(data), stream=True) if r.status_code == codes.ok: if not metalink: return self._load_json_data(r) return r.text exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)