def delete_protocols(self, rse, scheme, hostname=None, port=None): """ Deletes matching protocols from RSE. Protocols using the same identifier can be distinguished by hostname and port. :param rse: the RSE name. :param scheme: identifier of the protocol. :param hostname: hostname of the protocol. :param port: port of the protocol. :returns: True if success. :raises RSEProtocolNotSupported: if no matching protocol entry could be found. :raises RSENotFound: if the RSE doesn't exist. :raises AccessDenied: if not authorized. """ path = [self.RSE_BASEURL, rse, 'protocols', scheme] if hostname: path.append(hostname) if port: path.append(str(port)) path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def get_config(self, section=None, option=None): """ Sends the request to get the matching configuration. :param section: the optional name of the section. :param option: the optional option within the section. :return: dictionary containing the configuration. """ if section is None and option is not None: raise ValueError('--section not specified') path = self.CONFIG_BASEURL if section is not None and option is None: path += '/' + section elif section is not None and option is not None: path += '/'.join(['', section, option]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return r.json() else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def add_key(self, key, key_type, value_type=None, value_regexp=None): """ Sends the request to add a new key. :param key: the name for the new key. :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection). :param value_type: the type of the value, if defined. :param value_regexp: the regular expression that values should match, if defined. :return: True if key was created successfully. :raises Duplicate: if key already exists. """ path = '/'.join([self.META_BASEURL, key]) url = build_url(choice(self.list_hosts), path=path) data = dumps({'value_type': value_type and str(value_type), 'value_regexp': value_regexp, 'key_type': key_type}) r = self._send_request(url, type='POST', data=data) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def list_accounts(self, account_type=None, identity=None): """ Sends the request to list all rucio accounts. :param type: The account type :param identity: The identity key name. For example x509 DN, or a username. :return: a list containing account info dictionary for all rucio accounts. :raises AccountNotFound: if account doesn't exist. """ path = '/'.join([self.ACCOUNTS_BASEURL]) url = build_url(choice(self.list_hosts), path=path) params = {} if account_type: params['account_type'] = account_type if identity: params['identity'] = identity r = self._send_request(url, params=params) if r.status_code == codes.ok: accounts = self._load_json_data(r) return accounts else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def list_subscriptions(self, name=None, account=None): """ Returns a dictionary with the subscription information : Examples: ``{'status': 'INACTIVE/ACTIVE/BROKEN', 'last_modified_date': ...}`` :param name: Name of the subscription :type: String :param account: Account identifier :type account: String :returns: Dictionary containing subscription parameter :rtype: Dict :raises: exception.NotFound if subscription is not found """ path = self.SUB_BASEURL if account: path += '/%s' % (account) if name: path += '/%s' % (name) else: path += '/' url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def add_did(self, scope, name, type, statuses=None, meta=None, rules=None, lifetime=None): """ Add data identifier for a dataset or container. :param scope: The scope name. :param name: The data identifier name. :paran type: The data identifier type (file|dataset|container). :param statuses: Dictionary with statuses, e.g.g {'monotonic':True}. :meta: Meta-data associated with the data identifier is represented using key/value pairs in a dictionary. :rules: Replication rules associated with the data identifier. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ]. :param lifetime: DID's lifetime (in seconds). """ path = '/'.join([self.DIDS_BASEURL, scope, name]) url = build_url(choice(self.list_hosts), path=path) # Build json data = {'type': type} if statuses: data['statuses'] = statuses if meta: data['meta'] = meta if rules: data['rules'] = rules if lifetime: data['lifetime'] = lifetime r = self._send_request(url, type='POST', data=render_json(**data)) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def add_replication_rule(self, dids, copies, rse_expression, weight=None, lifetime=None, grouping='DATASET', account=None, locked=False, source_replica_expression=None, activity=None, notify='N', purge_replicas=False, ignore_availability=False): """ :param dids: The data identifier set. :param copies: The number of replicas. :param rse_expression: Boolean string expression to give the list of RSEs. :param weight: If the weighting option of the replication rule is used, the choice of RSEs takes their weight into account. :param lifetime: The lifetime of the replication rules (in seconds). :param grouping: ALL - All files will be replicated to the same RSE. DATASET - All files in the same dataset will be replicated to the same RSE. NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all. :param account: The account owning the rule. :param locked: If the rule is locked, it cannot be deleted. :param source_replica_expression: RSE Expression for RSEs to be considered for source replicas. :param activity: Transfer Activity to be passed to FTS. :param notify: Notification setting for the rule (Y, N, C). :param purge_replicas: When the rule gets deleted purge the associated replicas immediately. :param ignore_availability: Option to ignore the availability of RSEs. """ path = self.RULE_BASEURL + '/' url = build_url(choice(self.list_hosts), path=path) # TODO remove the subscription_id from the client; It will only be used by the core; data = dumps({'dids': dids, 'copies': copies, 'rse_expression': rse_expression, 'weight': weight, 'lifetime': lifetime, 'grouping': grouping, 'account': account, 'locked': locked, 'source_replica_expression': source_replica_expression, 'activity': activity, 'notify': notify, 'purge_replicas': purge_replicas, 'ignore_availability': ignore_availability}) r = self._send_request(url, type='POST', data=data) if r.status_code == codes.created: return loads(r.text) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def update_protocols(self, rse, scheme, data, hostname=None, port=None): """ Updates matching protocols from RSE. Protocol using the same identifier can be distinguished by hostname and port. :param rse: the RSE name. :param scheme: identifier of the protocol. :param data: A dict providing the new values of the protocol attibutes. Keys must match column names in database. :param hostname: hostname of the protocol. :param port: port of the protocol. :returns: True if success. :raises RSEProtocolNotSupported: if no matching protocol entry could be found. :raises RSENotFound: if the RSE doesn't exist. :raises KeyNotFound: if invalid data was provided for update. :raises AccessDenied: if not authorized. """ path = [self.RSE_BASEURL, rse, 'protocols', scheme] if hostname: path.append(hostname) if port: path.append(str(port)) path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='PUT', data=dumps(data)) if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def add_rse(self, rse, **kwargs): """ Sends the request to create a new RSE. :param rse: the name of the rse. :param deterministic: Boolean to know if the pfn is generated deterministically. :param volatile: Boolean for RSE cache. :param city: City for the RSE. :param region_code: The region code for the RSE. :param country_name: The country. :param continent: The continent. :param time_zone: Timezone. :param staging_area: Staging area. :param ISP: Internet service provider. :return: True if location was created successfully else False. :raises Duplicate: if rse already exists. """ path = 'rses/' + rse url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST', data=dumps(kwargs)) if r.status_code == codes.created: return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def __get_token_x509(self): """ Sends a request to get an auth token from the server and stores it as a class attribute. Uses x509 authentication. :returns: True if the token was successfully received. False otherwise. """ headers = {'X-Rucio-Account': self.account} client_cert = None client_key = None if self.auth_type == 'x509': url = build_url(self.auth_host, path='auth/x509') client_cert = self.creds['client_cert'] if 'client_key' in self.creds: client_key = self.creds['client_key'] elif self.auth_type == 'x509_proxy': url = build_url(self.auth_host, path='auth/x509_proxy') client_cert = self.creds['client_proxy'] if not path.exists(client_cert): LOG.error('given client cert (%s) doesn\'t exist' % client_cert) return False if client_key is not None and not path.exists(client_key): LOG.error('given client key (%s) doesn\'t exist' % client_key) retry = 0 if client_key is None: cert = client_cert else: cert = (client_cert, client_key) while retry <= self.AUTH_RETRIES: try: r = self.session.get(url, headers=headers, cert=cert, verify=self.ca_cert) except SSLError, e: if 'alert certificate expired' in str(e.message): raise CannotAuthenticate(str(e)) LOG.warning('SSLError: ' + str(e)) self.ca_cert = False retry += 1 if retry > self.request_retries: raise continue break
def add_dids(self, dids): """ Bulk add datasets/containers. """ path = '/'.join([self.DIDS_BASEURL]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST', data=render_json_list(dids)) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def ping(self): """ Sends a ping request to the rucio server. :return: Dictonnary with server information """ headers = None path = 'ping' url = build_url(self.host, path=path) r = self._send_request(url, headers=headers, type='GET') if r.status_code == codes.ok: server_info = loads(r.text) return server_info
def update_rse(self, rse, parameters): """ Update RSE properties like availability or name. :param rse: the name of the new rse. :param parameters: A dictionnary with property (name, read, write, delete as keys). """ path = 'rses/' + rse url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='PUT', data=dumps(parameters)) if r.status_code == codes.created: return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def delete_rse(self, rse): """ Sends the request to delete a rse. :param rse: the name of the rse. :return: True if location was created successfully else False. """ path = 'rses/' + rse url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def get_dataset_by_guid(self, guid): """ Get the parent datasets for a given GUID. :param guid: The GUID. :returns: A did """ path = '/'.join([self.DIDS_BASEURL, guid, 'guid']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def delete_replication_rule(self, rule_id): """ Deletes a replication rule and all associated locks. :param rule_id: The id of the rule to be deleted :raises: RuleNotFound, AccessDenied """ path = self.RULE_BASEURL + '/' + rule_id url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def get_replication_rule(self, rule_id): """ Get a replication rule. :param rule_id: The id of the rule to be retrieved. :raises: RuleNotFound """ path = self.RULE_BASEURL + '/' + rule_id url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r).next() else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def update_replication_rule(self, rule_id, options): """ :param rule_id: The id of the rule to be retrieved. :param options: Options dictionary. :raises: RuleNotFound """ path = self.RULE_BASEURL + '/' + rule_id url = build_url(choice(self.list_hosts), path=path) data = dumps({'options': options}) r = self._send_request(url, type='PUT', data=data) if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def get_account_limits(self, account): """ List the account rse limits of this account. :param account: The account name. """ path = '/'.join([self.ACCOUNTS_BASEURL, account, 'limits']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r).next() else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_identities(self, account): """ List all identities on an account. :param account: The account name. """ path = '/'.join([self.ACCOUNTS_BASEURL, account, 'identities']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url) if r.status_code == codes.ok: identities = self._load_json_data(r) return identities else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_values(self, key): """ Sends the request to list all values for a key. :return: a list containing the names of all values for a key. """ path = self.META_BASEURL + '/' + key + '/' url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url) if r.status_code == codes.ok: values = loads(r.text) return values else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def list_subscription_rules(self, account, name): """ List the associated rules of a subscription. :param account: Account of the subscription. :param name: Name of the subscription. """ path = '/'.join([self.SUB_BASEURL, account, name, 'Rules']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_scopes(self): """ Sends the request to list all scopes. :return: a list containing the names of all scopes. """ path = '/'.join(['scopes/']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url) if r.status_code == codes.ok: scopes = loads(r.text) return scopes else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def list_content(self, scope, name): """ List data identifier contents. :param scope: The scope name. :param name: The data identifier name. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'dids']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def get_metadata(self, scope, name): """ Get data identifier metadata :param scope: The scope name. :param name: The data identifier name. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'meta']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: meta = self._load_json_data(r) return meta.next() else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_associated_rules_for_file(self, scope, name): """ List the associated rules a file is affected from.. :param scope: The scope name. :param name: The file name. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'associated_rules']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def delete_metadata(self, scope, name, key): """ Delete data identifier metadata :param scope: The scope name. :param name: The data identifier. :param key: the key. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'meta', key]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
def set_status(self, scope, name, **kwargs): """ Set data identifier status :param scope: The scope name. :param name: The data identifier name. :param kwargs: Keyword arguments of the form status_name=value. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'status']) url = build_url(choice(self.list_hosts), path=path) data = dumps(kwargs) r = self._send_request(url, type='PUT', data=data) if r.status_code in (codes.ok, codes.no_content, codes.created): return True exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def detach_dids(self, scope, name, dids): """ Detach data identifier :param scope: The scope name. :param name: The data identifier name. :param dids: The content. """ path = '/'.join([self.DIDS_BASEURL, scope, name, 'dids']) url = build_url(choice(self.list_hosts), path=path) data = {'dids': dids} r = self._send_request(url, type='DEL', data=render_json(**data)) if r.status_code == codes.ok: return True exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def attach_dids_to_dids(self, attachments): """ Add dids to dids. :param attachments: The attachments. attachments is: [attachment, attachment, ...] attachment is: {'scope': scope, 'name': name, 'dids': dids} dids is: [{'scope': scope, 'name': name}, ...] """ path = '/'.join([self.DIDS_BASEURL, 'attachments']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST', data=render_json_list(attachments)) if r.status_code in (codes.ok, codes.no_content, codes.created): return True exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
def list_content(self, scope, name): """ List data identifier contents. :param scope: The scope name. :param name: The data identifier name. """ path = '/'.join( [self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'dids']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def get_account(self, account): """ Sends the request to get information about a given account. :param account: the name of the account. :return: a list of attributes for the account. None if failure. :raises AccountNotFound: if account doesn't exist. """ path = '/'.join([self.ACCOUNTS_BASEURL, account]) url = build_url(choice(self.list_hosts), path=path) res = self._send_request(url) if res.status_code == codes.ok: acc = self._load_json_data(res) return next(acc) exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content) raise exc_cls(exc_msg)
def add_account_attribute(self, account, key, value): """ Adds an attribute to an account. :param account: The account name. :param key: The attribute key. :param value: The attribute value. """ data = dumps({'key': key, 'value': value}) path = '/'.join([self.ACCOUNTS_BASEURL, account, 'attr', key]) url = build_url(choice(self.list_hosts), path=path) res = self._send_request(url, type='POST', data=data) if res.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content) raise exc_cls(exc_msg)
def delete_rse_attribute(self, rse, key): """ Sends the request to delete a RSE attribute. :param rse: the RSE name. :param key: the attribute key. :return: True if RSE attribute was deleted successfully else False. """ path = '/'.join([self.RSE_BASEURL, rse, 'attr', key]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def update_replicas_states(self, rse, files): """ Bulk update the file replicas states from a RSE. :param rse: the RSE name. :param files: The list of files. This is a list of DIDs like : [{'scope': <scope1>, 'name': <name1>}, {'scope': <scope2>, 'name': <name2>}, ...] :return: True if files have been deleted successfully. """ url = build_url(choice(self.list_hosts), path=self.REPLICAS_BASEURL) data = {'rse': rse, 'files': files} r = self._send_request(url, type='PUT', data=render_json(**data)) if r.status_code == codes.ok: return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def delete_replication_rule(self, rule_id, purge_replicas=None): """ Deletes a replication rule and all associated locks. :param rule_id: The id of the rule to be deleted :param purge_replicas: Immediately delete the replicas. :raises: RuleNotFound, AccessDenied """ path = self.RULE_BASEURL + '/' + rule_id url = build_url(choice(self.list_hosts), path=path) data = dumps({'purge_replicas': purge_replicas}) r = self._send_request(url, type_='DEL', data=data) if r.status_code == codes.ok: return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def update_account(self, account, key, value): """ Update a property of an account. :param account: Name of the account. :param key: Account property like status. :param value: Property value. """ data = dumps({key: value}) path = '/'.join([self.ACCOUNTS_BASEURL, account]) url = build_url(choice(self.list_hosts), path=path) res = self._send_request(url, type='PUT', data=data) if res.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content) raise exc_cls(exc_msg)
def get_global_account_limit(self, account, rse_expression): """ List the account limit for the specific RSE expression. :param account: The account name. :param rse_expression: The rse expression. """ path = '/'.join([ self.ACCOUNTS_BASEURL, account, 'limits', 'global', quote_plus(rse_expression) ]) url = build_url(choice(self.list_hosts), path=path) res = self._send_request(url, type='GET') if res.status_code == codes.ok: return next(self._load_json_data(res)) exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content) raise exc_cls(exc_msg)
def get_account_usage(self, account, rse=None): """ List the account usage for one or all rses of this account. :param account: The account name. :param rse: The rse name. """ if rse: path = '/'.join([self.ACCOUNTS_BASEURL, account, 'usage', rse]) else: path = '/'.join([self.ACCOUNTS_BASEURL, account, 'usage/']) url = build_url(choice(self.list_hosts), path=path) res = self._send_request(url, type='GET') if res.status_code == codes.ok: return self._load_json_data(res) else: exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content) raise exc_cls(exc_msg)
def add_bad_pfns(self, pfns, reason, state, expires_at): """ Declare a list of bad replicas. :param pfns: The list of PFNs. :param reason: The reason of the loss. :param state: The state of the replica. Either BAD, SUSPICIOUS, TEMPORARY_UNAVAILABLE :param expires_at: Specify a timeout for the TEMPORARY_UNAVAILABLE replicas. None for BAD files. :return: True if PFNs were created successfully. """ data = {'reason': reason, 'pfns': pfns, 'state': state, 'expires_at': expires_at} url = build_url(self.host, path='/'.join([self.REPLICAS_BASEURL, 'bad/pfns'])) headers = {} r = self._send_request(url, headers=headers, type='POST', data=dumps(data)) if r.status_code == codes.created: return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def list_requests(self, src_rse, dst_rse, request_states): """Return latest request details :return: request information :rtype: dict """ path = '/'.join([self.REQUEST_BASEURL, 'list']) + '?' + '&'.join([ 'src_rse={}'.format(src_rse), 'dst_rse={}'.format(dst_rse), 'request_states={}'.format(request_states) ]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type_='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def delete_rse_limits(self, rse, name): """ Delete RSE limit information. :param rse: The RSE name. :param name: The name of the limit. :returns: True if successful, otherwise false. """ path = [self.RSE_BASEURL, rse, 'limits'] path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL', data=dumps({'name': name})) if r.status_code == codes.ok: return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) return exc_cls(exc_msg)
def list_rse_usage_history(self, rse, filters=None): """ List RSE usage history information. :param rse: The RSE name. :param filters: dictionary of attributes by which the results should be filtered. :returns: list of dictionnaries. """ path = [self.RSE_BASEURL, rse, 'usage', 'history'] path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET', params=filters) if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def add_value(self, key, value): """ Sends the request to add a value to a key. :param key: the name for key. :param value: the value. :return: True if value was created successfully. :raises Duplicate: if valid already exists. """ path = '/'.join([self.META_BASEURL, quote_plus(key)]) + '/' data = dumps({'value': value}) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST', data=data) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def set_config_option(self, section, option, value): """ Sends the request to create or set an option within a section. Missing sections will be created. :param section: the name of the section. :param option: the name of the option. :return: True if option was removed successfully. False otherwise. """ path = '/'.join([self.CONFIG_BASEURL, section, option, value]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='PUT') if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def delete_qos_policy(self, rse, qos_policy): """ Delete a QoS policy from an RSE. :param rse_id: The id of the RSE. :param qos_policy: The QoS policy to delete. :param session: The database session in use. :returns: True if successful, silent failure if QoS policy does not exist. """ path = [self.RSE_BASEURL, rse, 'qos_policy', qos_policy] path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type_='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def delete_account(self, account): """ Sends the request to disable an account. :param account: the name of the account. :return: True is account was disabled successfully. False otherwise. :raises AccountNotFound: if account doesn't exist. """ path = '/'.join([self.ACCOUNTS_BASEURL, account]) url = build_url(choice(self.list_hosts), path=path) res = self._send_request(url, type='DEL') if res.status_code == codes.ok: return True exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content) raise exc_cls(exc_msg)
def get_rse(self, rse): """ Returns details about the referred RSE. :param rse: Name of the referred RSE :returns: A dict containing all attributes of the referred RSE :raises RSENotFound: if the referred RSE was not found in the database """ path = '/'.join([self.RSE_BASEURL, rse]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: rse = loads(r.text) return rse else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def list_scopes_for_account(self, account): """ Sends the request to list all scopes for a rucio account. :param account: the rucio account to list scopes for. :return: a list containing the names of all scopes for a rucio account. :raises AccountNotFound: if account doesn't exist. :raises ScopeNotFound: if no scopes exist for account. """ path = '/'.join([self.SCOPE_BASEURL, account, 'scopes/']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url) if r.status_code == codes.ok: scopes = loads(r.text) return scopes else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def list_dids_by_meta(self, scope=None, select={}): """ Gets all dids matching the values of the provided metadata keys :param scope: the scope of the search :param select: the key value pairs to search with(query in json format) """ path = '/'.join([self.DIDS_BASEURL, 'list_dids_by_meta']) payload = {} if scope is not None: payload['scope'] = scope payload['select'] = dumps(select) url = build_url(choice(self.list_hosts), path=path, params=payload) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return loads(next(self._load_json_data(r))) else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def attach_dids(self, scope, name, dids, rse=None): """ Attach data identifier. :param scope: The scope name. :param name: The data identifier name. :param dids: The content. :param rse: The RSE name when registering replicas. """ path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'dids']) url = build_url(choice(self.list_hosts), path=path) data = {'dids': dids} if rse: data['rse'] = rse r = self._send_request(url, type_='POST', data=render_json(**data)) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def list_dids(self, scope, filters, type='collection', long=False, recursive=False): """ List all data identifiers in a scope which match a given pattern. :param scope: The scope name. :param filters: A dictionary of key/value pairs like {'name': 'file_name','rse-expression': 'tier0'}. :param type: The type of the did: 'all'(container, dataset or file)|'collection'(dataset or container)|'dataset'|'container'|'file' :param long: Long format option to display more information for each DID. :param recursive: Recursively list DIDs content. """ path = '/'.join( [self.DIDS_BASEURL, quote_plus(scope), 'dids', 'search']) payload = {} if long: payload['long'] = 1 for k, v in list(filters.items()): if k in ('created_before', 'created_after'): payload[k] = date_to_str(v) else: payload[k] = v payload['type'] = type payload['recursive'] = recursive url = build_url(choice(self.list_hosts), path=path, params=payload) r = self._send_request(url, type='GET') if r.status_code == codes.ok: dids = self._load_json_data(r) return dids else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def __get_token_gss(self): """ Sends a request to get an auth token from the server and stores it as a class attribute. Uses Kerberos authentication. :returns: True if the token was successfully received. False otherwise. """ if not EXTRA_MODULES['requests_kerberos']: raise MissingModuleException( 'The requests-kerberos module is not installed.') headers = {'X-Rucio-Account': self.account} url = build_url(self.auth_host, path='auth/gss') result = None for retry in range(self.AUTH_RETRIES + 1): try: result = self.session.get(url, headers=headers, verify=self.ca_cert, auth=HTTPKerberosAuth()) break except ConnectionError as error: LOG.warning('ConnectionError: ' + str(error)) self.ca_cert = False if retry > self.request_retries: raise if not result: LOG.error('cannot get auth_token') return False if result.status_code != codes.ok: # pylint: disable-msg=E1101 exc_cls, exc_msg = self._get_exception( headers=result.headers, status_code=result.status_code, data=result.content) raise exc_cls(exc_msg) self.auth_token = result.headers['x-rucio-auth-token'] LOG.debug('got new token') return True
def __get_token_userpass(self): """ Sends a request to get an auth token from the server and stores it as a class attribute. Uses username/password. :returns: True if the token was successfully received. False otherwise. """ headers = { 'X-Rucio-Account': self.account, 'X-Rucio-Username': self.creds['username'], 'X-Rucio-Password': self.creds['password'] } url = build_url(self.auth_host, path='auth/userpass') result = None for retry in range(self.AUTH_RETRIES + 1): try: result = self.session.get(url, headers=headers, verify=self.ca_cert) break except ConnectionError as error: LOG.warning('ConnectionError: ' + str(error)) self.ca_cert = False if retry > self.request_retries: raise if not result or 'result' not in locals(): LOG.error('cannot get auth_token') return False if result.status_code != codes.ok: # pylint: disable-msg=E1101 exc_cls, exc_msg = self._get_exception( headers=result.headers, status_code=result.status_code, data=result.content) raise exc_cls(exc_msg) self.auth_token = result.headers['x-rucio-auth-token'] LOG.debug('got new token') return True
def update_subscription(self, name, account=None, filter=None, replication_rules=None, comments=None, lifetime=None, retroactive=None, dry_run=None, priority=None): """ Updates a subscription :param name: Name of the subscription :type: String :param account: Account identifier :type account: String :param filter: Dictionary of attributes by which the input data should be filtered **Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}`` :type filter: Dict :param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression :type replication_rules: Dict :param comments: Comments for the subscription :type comments: String :param lifetime: Subscription's lifetime (days); False if subscription has no lifetime :type lifetime: Integer or False :param retroactive: Flag to know if the subscription should be applied on previous data :type retroactive: Boolean :param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set) :type dry_run: Boolean :param priority: The priority of the subscription :type priority: Integer :raises: exception.NotFound if subscription is not found """ if not account: account = self.account path = self.SUB_BASEURL + '/' + account + '/' + name url = build_url(choice(self.list_hosts), path=path) if filter and type(filter) != dict: raise TypeError('filter should be a dict') if replication_rules and type(replication_rules) != list: raise TypeError('replication_rules should be a list') data = dumps({'options': {'filter': filter, 'replication_rules': replication_rules, 'comments': comments, 'lifetime': lifetime, 'retroactive': retroactive, 'dry_run': dry_run, 'priority': priority}}) r = self._send_request(url, type='PUT', data=data) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def declare_suspicious_file_replicas(self, pfns, reason): """ Declare a list of bad replicas. :param pfns: The list of PFNs. :param reason: The reason of the loss. """ data = {'reason': reason, 'pfns': pfns} url = build_url(self.host, path='/'.join([self.REPLICAS_BASEURL, 'suspicious'])) headers = {} r = self._send_request(url, headers=headers, type='POST', data=dumps(data)) if r.status_code == codes.created: return loads(r.text) exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def get_did_meta(self, scope, name): """ Get all metadata for a given did :param scope: the scope of did :param name: the name of the did """ path = '/'.join([ self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'did_meta' ]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: meta = self._load_json_data(r) return next(meta) else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def get_dataset_locks_by_rse(self, rse): """ Get all dataset locks of the specified rse. :param rse: the rse of the locks to list. """ path = '/'.join([self.LOCKS_BASEURL, rse]) url = build_url(choice(self.list_hosts), path=path, params={'did_type': 'dataset'}) result = self._send_request(url) if result.status_code == codes.ok: # pylint: disable-msg=E1101 locks = self._load_json_data(result) return locks else: exc_cls, exc_msg = self._get_exception( headers=result.headers, status_code=result.status_code) raise exc_cls(exc_msg)
def list_rses(self, rse_expression=None): """ Sends the request to list all rucio locations(RSEs). :rse_expression: RSE Expression to use as filter. :return: a list containing the names of all rucio locations. """ if rse_expression: path = ['rses', "?expression=" + quote(rse_expression)] path = '/'.join(path) else: path = 'rses/' url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)
def list_datasets_per_rse(self, rse, filters=None, limit=None): """ List datasets at a RSE. :param rse: the rse name. :param filters: dictionary of attributes by which the results should be filtered. :param limit: limit number. :returns: A list of dict dataset replicas. """ url = build_url(self.host, path='/'.join([self.REPLICAS_BASEURL, 'rse', rse])) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) raise exc_cls(exc_msg)