Beispiel #1
0
    def delete_protocols(self, rse, scheme, hostname=None, port=None):
        """
        Deletes matching protocols from RSE. Protocols using the same identifier can be
        distinguished by hostname and port.

        :param rse: the RSE name.
        :param scheme: identifier of the protocol.
        :param hostname: hostname of the protocol.
        :param port: port of the protocol.

        :returns: True if success.

        :raises RSEProtocolNotSupported: if no matching protocol entry could be found.
        :raises RSENotFound: if the RSE doesn't exist.
        :raises AccessDenied: if not authorized.
        """
        path = [self.RSE_BASEURL, rse, 'protocols', scheme]
        if hostname:
            path.append(hostname)
            if port:
                path.append(str(port))

        path = '/'.join(path)
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='DEL')
        if r.status_code == codes.ok:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #2
0
    def get_config(self, section=None, option=None):
        """
        Sends the request to get the matching configuration.

        :param section: the optional name of the section.
        :param option: the optional option within the section.
        :return: dictionary containing the configuration.
        """

        if section is None and option is not None:
            raise ValueError('--section not specified')

        path = self.CONFIG_BASEURL
        if section is not None and option is None:
            path += '/' + section
        elif section is not None and option is not None:
            path += '/'.join(['', section, option])

        url = build_url(choice(self.list_hosts), path=path)

        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            return r.json()
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #3
0
    def add_key(self, key, key_type, value_type=None, value_regexp=None):
        """
        Sends the request to add a new key.

        :param key: the name for the new key.
        :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
        :param value_type: the type of the value, if defined.
        :param value_regexp: the regular expression that values should match, if defined.

        :return: True if key was created successfully.
        :raises Duplicate: if key already exists.
        """

        path = '/'.join([self.META_BASEURL, key])
        url = build_url(choice(self.list_hosts), path=path)
        data = dumps({'value_type': value_type and str(value_type),
                      'value_regexp': value_regexp,
                      'key_type': key_type})

        r = self._send_request(url, type='POST', data=data)

        if r.status_code == codes.created:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
            raise exc_cls(exc_msg)
Beispiel #4
0
    def list_accounts(self, account_type=None, identity=None):
        """
        Sends the request to list all rucio accounts.

        :param type: The account type
        :param identity: The identity key name. For example x509 DN, or a username.

        :return: a list containing account info dictionary for all rucio accounts.
        :raises AccountNotFound: if account doesn't exist.
        """
        path = '/'.join([self.ACCOUNTS_BASEURL])
        url = build_url(choice(self.list_hosts), path=path)
        params = {}
        if account_type:
            params['account_type'] = account_type
        if identity:
            params['identity'] = identity

        r = self._send_request(url, params=params)

        if r.status_code == codes.ok:
            accounts = self._load_json_data(r)
            return accounts
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
            raise exc_cls(exc_msg)
    def list_subscriptions(self, name=None, account=None):
        """
        Returns a dictionary with the subscription information :
        Examples: ``{'status': 'INACTIVE/ACTIVE/BROKEN', 'last_modified_date': ...}``

        :param name: Name of the subscription
        :type:  String
        :param account: Account identifier
        :type account:  String
        :returns: Dictionary containing subscription parameter
        :rtype:   Dict
        :raises: exception.NotFound if subscription is not found
        """
        path = self.SUB_BASEURL
        if account:
            path += '/%s' % (account)
            if name:
                path += '/%s' % (name)
        else:
            path += '/'
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            return self._load_json_data(r)
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #6
0
    def add_did(self, scope, name, type, statuses=None, meta=None, rules=None, lifetime=None):
        """
        Add data identifier for a dataset or container.

        :param scope: The scope name.
        :param name: The data identifier name.
        :paran type: The data identifier type (file|dataset|container).
        :param statuses: Dictionary with statuses, e.g.g {'monotonic':True}.
        :meta: Meta-data associated with the data identifier is represented using key/value pairs in a dictionary.
        :rules: Replication rules associated with the data identifier. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ].
        :param lifetime: DID's lifetime (in seconds).
        """
        path = '/'.join([self.DIDS_BASEURL, scope, name])
        url = build_url(choice(self.list_hosts), path=path)
        # Build json
        data = {'type': type}
        if statuses:
            data['statuses'] = statuses
        if meta:
            data['meta'] = meta
        if rules:
            data['rules'] = rules
        if lifetime:
            data['lifetime'] = lifetime
        r = self._send_request(url, type='POST', data=render_json(**data))
        if r.status_code == codes.created:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #7
0
 def add_replication_rule(self, dids, copies, rse_expression, weight=None, lifetime=None, grouping='DATASET', account=None, locked=False, source_replica_expression=None, activity=None, notify='N', purge_replicas=False, ignore_availability=False):
     """
     :param dids:                       The data identifier set.
     :param copies:                     The number of replicas.
     :param rse_expression:             Boolean string expression to give the list of RSEs.
     :param weight:                     If the weighting option of the replication rule is used, the choice of RSEs takes their weight into account.
     :param lifetime:                   The lifetime of the replication rules (in seconds).
     :param grouping:                   ALL -  All files will be replicated to the same RSE.
                                        DATASET - All files in the same dataset will be replicated to the same RSE.
                                        NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all.
     :param account:                    The account owning the rule.
     :param locked:                     If the rule is locked, it cannot be deleted.
     :param source_replica_expression:  RSE Expression for RSEs to be considered for source replicas.
     :param activity:                   Transfer Activity to be passed to FTS.
     :param notify:                     Notification setting for the rule (Y, N, C).
     :param purge_replicas:             When the rule gets deleted purge the associated replicas immediately.
     :param ignore_availability:        Option to ignore the availability of RSEs.
     """
     path = self.RULE_BASEURL + '/'
     url = build_url(choice(self.list_hosts), path=path)
     # TODO remove the subscription_id from the client; It will only be used by the core;
     data = dumps({'dids': dids, 'copies': copies, 'rse_expression': rse_expression,
                   'weight': weight, 'lifetime': lifetime, 'grouping': grouping,
                   'account': account, 'locked': locked, 'source_replica_expression': source_replica_expression,
                   'activity': activity, 'notify': notify, 'purge_replicas': purge_replicas,
                   'ignore_availability': ignore_availability})
     r = self._send_request(url, type='POST', data=data)
     if r.status_code == codes.created:
         return loads(r.text)
     else:
         exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
         raise exc_cls(exc_msg)
Beispiel #8
0
    def update_protocols(self, rse, scheme, data, hostname=None, port=None):
        """
        Updates matching protocols from RSE. Protocol using the same identifier can be
        distinguished by hostname and port.

        :param rse: the RSE name.
        :param scheme: identifier of the protocol.
        :param data: A dict providing the new values of the protocol attibutes.
                     Keys must match column names in database.
        :param hostname: hostname of the protocol.
        :param port: port of the protocol.

        :returns: True if success.

        :raises RSEProtocolNotSupported: if no matching protocol entry could be found.
        :raises RSENotFound: if the RSE doesn't exist.
        :raises KeyNotFound: if invalid data was provided for update.
        :raises AccessDenied: if not authorized.
        """
        path = [self.RSE_BASEURL, rse, 'protocols', scheme]
        if hostname:
            path.append(hostname)
            if port:
                path.append(str(port))

        path = '/'.join(path)
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='PUT', data=dumps(data))
        if r.status_code == codes.ok:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #9
0
    def add_rse(self, rse, **kwargs):
        """
        Sends the request to create a new RSE.

        :param rse: the name of the rse.
        :param deterministic: Boolean to know if the pfn is generated deterministically.
        :param volatile: Boolean for RSE cache.
        :param city: City for the RSE.
        :param region_code: The region code for the RSE.
        :param country_name: The country.
        :param continent: The continent.
        :param time_zone: Timezone.
        :param staging_area: Staging area.
        :param ISP: Internet service provider.

        :return: True if location was created successfully else False.
        :raises Duplicate: if rse already exists.
        """
        path = 'rses/' + rse
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='POST', data=dumps(kwargs))
        if r.status_code == codes.created:
            return True
        exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
        raise exc_cls(exc_msg)
Beispiel #10
0
    def __get_token_x509(self):
        """
        Sends a request to get an auth token from the server and stores it as a class attribute. Uses x509 authentication.

        :returns: True if the token was successfully received. False otherwise.
        """

        headers = {'X-Rucio-Account': self.account}

        client_cert = None
        client_key = None
        if self.auth_type == 'x509':
            url = build_url(self.auth_host, path='auth/x509')
            client_cert = self.creds['client_cert']
            if 'client_key' in self.creds:
                client_key = self.creds['client_key']
        elif self.auth_type == 'x509_proxy':
            url = build_url(self.auth_host, path='auth/x509_proxy')
            client_cert = self.creds['client_proxy']

        if not path.exists(client_cert):
            LOG.error('given client cert (%s) doesn\'t exist' % client_cert)
            return False
        if client_key is not None and not path.exists(client_key):
            LOG.error('given client key (%s) doesn\'t exist' % client_key)

        retry = 0

        if client_key is None:
            cert = client_cert
        else:
            cert = (client_cert, client_key)

        while retry <= self.AUTH_RETRIES:
            try:
                r = self.session.get(url, headers=headers, cert=cert, verify=self.ca_cert)
            except SSLError, e:
                if 'alert certificate expired' in str(e.message):
                    raise CannotAuthenticate(str(e))
                LOG.warning('SSLError: ' + str(e))
                self.ca_cert = False
                retry += 1
                if retry > self.request_retries:
                    raise
                continue
            break
Beispiel #11
0
 def add_dids(self, dids):
     """
     Bulk add datasets/containers.
     """
     path = '/'.join([self.DIDS_BASEURL])
     url = build_url(choice(self.list_hosts), path=path)
     r = self._send_request(url, type='POST', data=render_json_list(dids))
     if r.status_code == codes.created:
         return True
     else:
         exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
         raise exc_cls(exc_msg)
Beispiel #12
0
    def ping(self):
        """
        Sends a ping request to the rucio server.

        :return: Dictonnary with server information
        """

        headers = None
        path = 'ping'
        url = build_url(self.host, path=path)
        r = self._send_request(url, headers=headers, type='GET')
        if r.status_code == codes.ok:
            server_info = loads(r.text)
            return server_info
Beispiel #13
0
    def update_rse(self, rse, parameters):
        """
        Update RSE properties like availability or name.

        :param rse: the name of the new rse.
        :param  parameters: A dictionnary with property (name, read, write, delete as keys).
        """
        path = 'rses/' + rse
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='PUT', data=dumps(parameters))
        if r.status_code == codes.created:
            return True
        exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
        raise exc_cls(exc_msg)
Beispiel #14
0
    def delete_rse(self, rse):
        """
        Sends the request to delete a rse.

        :param rse: the name of the rse.
        :return: True if location was created successfully else False.
        """
        path = 'rses/' + rse
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='DEL')
        if r.status_code == codes.ok:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #15
0
    def get_dataset_by_guid(self, guid):
        """
        Get the parent datasets for a given GUID.
       :param guid: The GUID.

        :returns: A did
        """
        path = '/'.join([self.DIDS_BASEURL, guid, 'guid'])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            return self._load_json_data(r)
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #16
0
    def delete_replication_rule(self, rule_id):
        """
        Deletes a replication rule and all associated locks.

        :param rule_id:  The id of the rule to be deleted
        :raises:         RuleNotFound, AccessDenied
        """
        path = self.RULE_BASEURL + '/' + rule_id
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='DEL')
        if r.status_code == codes.ok:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #17
0
    def get_replication_rule(self, rule_id):
        """
        Get a replication rule.

        :param rule_id:  The id of the rule to be retrieved.
        :raises:         RuleNotFound
        """
        path = self.RULE_BASEURL + '/' + rule_id
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            return self._load_json_data(r).next()
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #18
0
 def update_replication_rule(self, rule_id, options):
     """
     :param rule_id:   The id of the rule to be retrieved.
     :param options:   Options dictionary.
     :raises:          RuleNotFound
     """
     path = self.RULE_BASEURL + '/' + rule_id
     url = build_url(choice(self.list_hosts), path=path)
     data = dumps({'options': options})
     r = self._send_request(url, type='PUT', data=data)
     if r.status_code == codes.ok:
         return True
     else:
         exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
         raise exc_cls(exc_msg)
Beispiel #19
0
    def get_account_limits(self, account):
        """
        List the account rse limits of this account.

        :param account: The account name.
        """

        path = '/'.join([self.ACCOUNTS_BASEURL, account, 'limits'])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            return self._load_json_data(r).next()
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #20
0
    def list_identities(self, account):
        """
        List all identities on an account.

        :param account: The account name.
        """
        path = '/'.join([self.ACCOUNTS_BASEURL, account, 'identities'])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url)
        if r.status_code == codes.ok:
            identities = self._load_json_data(r)
            return identities
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #21
0
    def list_values(self, key):
        """
        Sends the request to list all values for a key.

        :return: a list containing the names of all values for a key.
        """
        path = self.META_BASEURL + '/' + key + '/'
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url)
        if r.status_code == codes.ok:
            values = loads(r.text)
            return values
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
            raise exc_cls(exc_msg)
Beispiel #22
0
    def list_subscription_rules(self, account, name):
        """
        List the associated rules of a subscription.

        :param account: Account of the subscription.
        :param name:    Name of the subscription.
        """

        path = '/'.join([self.SUB_BASEURL, account, name, 'Rules'])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            return self._load_json_data(r)
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #23
0
    def list_scopes(self):
        """
        Sends the request to list all scopes.

        :return: a list containing the names of all scopes.
        """

        path = '/'.join(['scopes/'])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url)
        if r.status_code == codes.ok:
            scopes = loads(r.text)
            return scopes
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
            raise exc_cls(exc_msg)
Beispiel #24
0
    def list_content(self, scope, name):
        """
        List data identifier contents.

        :param scope: The scope name.
        :param name: The data identifier name.
        """

        path = '/'.join([self.DIDS_BASEURL, scope, name, 'dids'])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            return self._load_json_data(r)
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #25
0
    def get_metadata(self, scope, name):
        """
        Get data identifier metadata

        :param scope: The scope name.
        :param name: The data identifier name.
        """
        path = '/'.join([self.DIDS_BASEURL, scope, name, 'meta'])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            meta = self._load_json_data(r)
            return meta.next()
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #26
0
    def list_associated_rules_for_file(self, scope, name):
        """
        List the associated rules a file is affected from..

        :param scope: The scope name.
        :param name:  The file name.
        """

        path = '/'.join([self.DIDS_BASEURL, scope, name, 'associated_rules'])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            return self._load_json_data(r)
        else:
            exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
            raise exc_cls(exc_msg)
Beispiel #27
0
    def delete_metadata(self, scope, name, key):
        """
        Delete data identifier metadata

        :param scope: The scope name.
        :param name: The data identifier.
        :param key: the key.
        """
        path = '/'.join([self.DIDS_BASEURL, scope, name, 'meta', key])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='DEL')

        if r.status_code == codes.ok:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code)
            raise exc_cls(exc_msg)
Beispiel #28
0
    def set_status(self, scope, name, **kwargs):
        """
        Set data identifier status

        :param scope: The scope name.
        :param name: The data identifier name.
        :param kwargs:  Keyword arguments of the form status_name=value.
        """
        path = '/'.join([self.DIDS_BASEURL, scope, name, 'status'])
        url = build_url(choice(self.list_hosts), path=path)
        data = dumps(kwargs)
        r = self._send_request(url, type='PUT', data=data)
        if r.status_code in (codes.ok, codes.no_content, codes.created):
            return True

        exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
        raise exc_cls(exc_msg)
Beispiel #29
0
    def detach_dids(self, scope, name, dids):
        """
        Detach data identifier

        :param scope: The scope name.
        :param name: The data identifier name.
        :param dids: The content.
        """

        path = '/'.join([self.DIDS_BASEURL, scope, name, 'dids'])
        url = build_url(choice(self.list_hosts), path=path)
        data = {'dids': dids}
        r = self._send_request(url, type='DEL', data=render_json(**data))
        if r.status_code == codes.ok:
            return True
        exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
        raise exc_cls(exc_msg)
Beispiel #30
0
    def attach_dids_to_dids(self, attachments):
        """
        Add dids to dids.

        :param attachments: The attachments.
            attachments is: [attachment, attachment, ...]
            attachment is: {'scope': scope, 'name': name, 'dids': dids}
            dids is: [{'scope': scope, 'name': name}, ...]
        """
        path = '/'.join([self.DIDS_BASEURL, 'attachments'])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='POST', data=render_json_list(attachments))
        if r.status_code in (codes.ok, codes.no_content, codes.created):
            return True

        exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
        raise exc_cls(exc_msg)
Beispiel #31
0
    def list_content(self, scope, name):
        """
        List data identifier contents.

        :param scope: The scope name.
        :param name: The data identifier name.
        """

        path = '/'.join(
            [self.DIDS_BASEURL,
             quote_plus(scope),
             quote_plus(name), 'dids'])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            return self._load_json_data(r)
        exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                               status_code=r.status_code,
                                               data=r.content)
        raise exc_cls(exc_msg)
Beispiel #32
0
    def get_account(self, account):
        """
        Sends the request to get information about a given account.

        :param account: the name of the account.
        :return: a list of attributes for the account. None if failure.
        :raises AccountNotFound: if account doesn't exist.
        """

        path = '/'.join([self.ACCOUNTS_BASEURL, account])
        url = build_url(choice(self.list_hosts), path=path)

        res = self._send_request(url)
        if res.status_code == codes.ok:
            acc = self._load_json_data(res)
            return next(acc)
        exc_cls, exc_msg = self._get_exception(headers=res.headers,
                                               status_code=res.status_code,
                                               data=res.content)
        raise exc_cls(exc_msg)
Beispiel #33
0
    def add_account_attribute(self, account, key, value):
        """
        Adds an attribute to an account.

        :param account: The account name.
        :param key: The attribute key.
        :param value: The attribute value.
        """

        data = dumps({'key': key, 'value': value})
        path = '/'.join([self.ACCOUNTS_BASEURL, account, 'attr', key])
        url = build_url(choice(self.list_hosts), path=path)
        res = self._send_request(url, type='POST', data=data)
        if res.status_code == codes.created:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(headers=res.headers,
                                                   status_code=res.status_code,
                                                   data=res.content)
            raise exc_cls(exc_msg)
Beispiel #34
0
    def delete_rse_attribute(self, rse, key):
        """
        Sends the request to delete a RSE attribute.

        :param rse: the RSE name.
        :param key: the attribute key.

        :return: True if RSE attribute was deleted successfully else False.
        """
        path = '/'.join([self.RSE_BASEURL, rse, 'attr', key])
        url = build_url(choice(self.list_hosts), path=path)

        r = self._send_request(url, type='DEL')
        if r.status_code == codes.ok:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                                   status_code=r.status_code,
                                                   data=r.content)
            raise exc_cls(exc_msg)
Beispiel #35
0
    def update_replicas_states(self, rse, files):
        """
        Bulk update the file replicas states from a RSE.

        :param rse: the RSE name.
        :param files: The list of files. This is a list of DIDs like :
            [{'scope': <scope1>, 'name': <name1>}, {'scope': <scope2>, 'name': <name2>}, ...]

        :return: True if files have been deleted successfully.

        """
        url = build_url(choice(self.list_hosts), path=self.REPLICAS_BASEURL)
        data = {'rse': rse, 'files': files}
        r = self._send_request(url, type='PUT', data=render_json(**data))
        if r.status_code == codes.ok:
            return True
        exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                               status_code=r.status_code,
                                               data=r.content)
        raise exc_cls(exc_msg)
Beispiel #36
0
    def delete_replication_rule(self, rule_id, purge_replicas=None):
        """
        Deletes a replication rule and all associated locks.

        :param rule_id:         The id of the rule to be deleted
        :param purge_replicas:  Immediately delete the replicas.
        :raises:                RuleNotFound, AccessDenied
        """

        path = self.RULE_BASEURL + '/' + rule_id
        url = build_url(choice(self.list_hosts), path=path)

        data = dumps({'purge_replicas': purge_replicas})

        r = self._send_request(url, type_='DEL', data=data)

        if r.status_code == codes.ok:
            return True
        exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
        raise exc_cls(exc_msg)
Beispiel #37
0
    def update_account(self, account, key, value):
        """ Update a property of an account.

        :param account: Name of the account.
        :param key: Account property like status.
        :param value: Property value.
        """
        data = dumps({key: value})
        path = '/'.join([self.ACCOUNTS_BASEURL, account])
        url = build_url(choice(self.list_hosts), path=path)

        res = self._send_request(url, type='PUT', data=data)

        if res.status_code == codes.ok:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(headers=res.headers,
                                                   status_code=res.status_code,
                                                   data=res.content)
            raise exc_cls(exc_msg)
Beispiel #38
0
    def get_global_account_limit(self, account, rse_expression):
        """
        List the account limit for the specific RSE expression.

        :param account:        The account name.
        :param rse_expression: The rse expression.
        """

        path = '/'.join([
            self.ACCOUNTS_BASEURL, account, 'limits', 'global',
            quote_plus(rse_expression)
        ])
        url = build_url(choice(self.list_hosts), path=path)
        res = self._send_request(url, type='GET')
        if res.status_code == codes.ok:
            return next(self._load_json_data(res))
        exc_cls, exc_msg = self._get_exception(headers=res.headers,
                                               status_code=res.status_code,
                                               data=res.content)
        raise exc_cls(exc_msg)
Beispiel #39
0
    def get_account_usage(self, account, rse=None):
        """
        List the account usage for one or all rses of this account.

        :param account: The account name.
        :param rse:     The rse name.
        """
        if rse:
            path = '/'.join([self.ACCOUNTS_BASEURL, account, 'usage', rse])
        else:
            path = '/'.join([self.ACCOUNTS_BASEURL, account, 'usage/'])
        url = build_url(choice(self.list_hosts), path=path)
        res = self._send_request(url, type='GET')
        if res.status_code == codes.ok:
            return self._load_json_data(res)
        else:
            exc_cls, exc_msg = self._get_exception(headers=res.headers,
                                                   status_code=res.status_code,
                                                   data=res.content)
            raise exc_cls(exc_msg)
    def add_bad_pfns(self, pfns, reason, state, expires_at):
        """
        Declare a list of bad replicas.

        :param pfns: The list of PFNs.
        :param reason: The reason of the loss.
        :param state: The state of the replica. Either BAD, SUSPICIOUS, TEMPORARY_UNAVAILABLE
        :param expires_at: Specify a timeout for the TEMPORARY_UNAVAILABLE replicas. None for BAD files.

        :return: True if PFNs were created successfully.

        """
        data = {'reason': reason, 'pfns': pfns, 'state': state, 'expires_at': expires_at}
        url = build_url(self.host, path='/'.join([self.REPLICAS_BASEURL, 'bad/pfns']))
        headers = {}
        r = self._send_request(url, headers=headers, type='POST', data=dumps(data))
        if r.status_code == codes.created:
            return True
        exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
        raise exc_cls(exc_msg)
Beispiel #41
0
    def list_requests(self, src_rse, dst_rse, request_states):
        """Return latest request details

        :return: request information
        :rtype: dict
        """
        path = '/'.join([self.REQUEST_BASEURL, 'list']) + '?' + '&'.join([
            'src_rse={}'.format(src_rse), 'dst_rse={}'.format(dst_rse),
            'request_states={}'.format(request_states)
        ])
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type_='GET')

        if r.status_code == codes.ok:
            return self._load_json_data(r)
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                                   status_code=r.status_code,
                                                   data=r.content)
            raise exc_cls(exc_msg)
Beispiel #42
0
    def delete_rse_limits(self, rse, name):
        """
        Delete RSE limit information.

        :param rse: The RSE name.
        :param name: The name of the limit.

        :returns: True if successful, otherwise false.
        """
        path = [self.RSE_BASEURL, rse, 'limits']
        path = '/'.join(path)
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='DEL', data=dumps({'name': name}))
        if r.status_code == codes.ok:
            return True
        exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                               status_code=r.status_code,
                                               data=r.content)

        return exc_cls(exc_msg)
Beispiel #43
0
    def list_rse_usage_history(self, rse, filters=None):
        """
        List RSE usage history information.

        :param rse: The RSE name.
        :param filters: dictionary of attributes by which the results should be filtered.

        :returns:  list of dictionnaries.
        """
        path = [self.RSE_BASEURL, rse, 'usage', 'history']
        path = '/'.join(path)
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='GET', params=filters)
        if r.status_code == codes.ok:
            return self._load_json_data(r)
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                                   status_code=r.status_code,
                                                   data=r.content)
            raise exc_cls(exc_msg)
Beispiel #44
0
    def add_value(self, key, value):
        """
        Sends the request to add a value to a key.

        :param key: the name for key.
        :param value: the value.

        :return: True if value was created successfully.
        :raises Duplicate: if valid already exists.
        """

        path = '/'.join([self.META_BASEURL, quote_plus(key)]) + '/'
        data = dumps({'value': value})
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='POST', data=data)
        if r.status_code == codes.created:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
            raise exc_cls(exc_msg)
    def set_config_option(self, section, option, value):
        """
        Sends the request to create or set an option within a section. Missing sections will be created.

        :param section: the name of the section.
        :param option: the name of the option.
        :return: True if option was removed successfully. False otherwise.
        """

        path = '/'.join([self.CONFIG_BASEURL, section, option, value])
        url = build_url(choice(self.list_hosts), path=path)

        r = self._send_request(url, type='PUT')
        if r.status_code == codes.created:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                                   status_code=r.status_code,
                                                   data=r.content)
            raise exc_cls(exc_msg)
Beispiel #46
0
    def delete_qos_policy(self, rse, qos_policy):
        """
        Delete a QoS policy from an RSE.

        :param rse_id: The id of the RSE.
        :param qos_policy: The QoS policy to delete.
        :param session: The database session in use.

        :returns: True if successful, silent failure if QoS policy does not exist.
        """

        path = [self.RSE_BASEURL, rse, 'qos_policy', qos_policy]
        path = '/'.join(path)
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type_='DEL')
        if r.status_code == codes.ok:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
            raise exc_cls(exc_msg)
Beispiel #47
0
    def delete_account(self, account):
        """
        Sends the request to disable an account.

        :param account: the name of the account.
        :return: True is account was disabled successfully. False otherwise.
        :raises AccountNotFound: if account doesn't exist.
        """

        path = '/'.join([self.ACCOUNTS_BASEURL, account])
        url = build_url(choice(self.list_hosts), path=path)

        res = self._send_request(url, type='DEL')

        if res.status_code == codes.ok:
            return True
        exc_cls, exc_msg = self._get_exception(headers=res.headers,
                                               status_code=res.status_code,
                                               data=res.content)
        raise exc_cls(exc_msg)
Beispiel #48
0
    def get_rse(self, rse):
        """
        Returns details about the referred RSE.

        :param rse: Name of the referred RSE

        :returns: A dict containing all attributes of the referred RSE

        :raises RSENotFound: if the referred RSE was not found in the database
        """
        path = '/'.join([self.RSE_BASEURL, rse])
        url = build_url(choice(self.list_hosts), path=path)

        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            rse = loads(r.text)
            return rse
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
            raise exc_cls(exc_msg)
Beispiel #49
0
    def list_scopes_for_account(self, account):
        """
        Sends the request to list all scopes for a rucio account.

        :param account: the rucio account to list scopes for.
        :return: a list containing the names of all scopes for a rucio account.
        :raises AccountNotFound: if account doesn't exist.
        :raises ScopeNotFound: if no scopes exist for account.
        """

        path = '/'.join([self.SCOPE_BASEURL, account, 'scopes/'])
        url = build_url(choice(self.list_hosts), path=path)

        r = self._send_request(url)
        if r.status_code == codes.ok:
            scopes = loads(r.text)
            return scopes
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
            raise exc_cls(exc_msg)
Beispiel #50
0
 def list_dids_by_meta(self, scope=None, select={}):
     """
     Gets all dids matching the values of the provided metadata keys
     :param scope: the scope of the search
     :param select: the key value pairs to search with(query in json format)
     """
     path = '/'.join([self.DIDS_BASEURL, 'list_dids_by_meta'])
     payload = {}
     if scope is not None:
         payload['scope'] = scope
     payload['select'] = dumps(select)
     url = build_url(choice(self.list_hosts), path=path, params=payload)
     r = self._send_request(url, type='GET')
     if r.status_code == codes.ok:
         return loads(next(self._load_json_data(r)))
     else:
         exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                                status_code=r.status_code,
                                                data=r.content)
         raise exc_cls(exc_msg)
Beispiel #51
0
    def attach_dids(self, scope, name, dids, rse=None):
        """
        Attach data identifier.

        :param scope: The scope name.
        :param name: The data identifier name.
        :param dids: The content.
        :param rse: The RSE name when registering replicas.
        """
        path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'dids'])
        url = build_url(choice(self.list_hosts), path=path)
        data = {'dids': dids}
        if rse:
            data['rse'] = rse
        r = self._send_request(url, type_='POST', data=render_json(**data))
        if r.status_code == codes.created:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
            raise exc_cls(exc_msg)
Beispiel #52
0
    def list_dids(self,
                  scope,
                  filters,
                  type='collection',
                  long=False,
                  recursive=False):
        """
        List all data identifiers in a scope which match a given pattern.

        :param scope: The scope name.
        :param filters: A dictionary of key/value pairs like {'name': 'file_name','rse-expression': 'tier0'}.
        :param type: The type of the did: 'all'(container, dataset or file)|'collection'(dataset or container)|'dataset'|'container'|'file'
        :param long: Long format option to display more information for each DID.
        :param recursive: Recursively list DIDs content.
        """
        path = '/'.join(
            [self.DIDS_BASEURL,
             quote_plus(scope), 'dids', 'search'])
        payload = {}
        if long:
            payload['long'] = 1

        for k, v in list(filters.items()):
            if k in ('created_before', 'created_after'):
                payload[k] = date_to_str(v)
            else:
                payload[k] = v
        payload['type'] = type
        payload['recursive'] = recursive

        url = build_url(choice(self.list_hosts), path=path, params=payload)

        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            dids = self._load_json_data(r)
            return dids
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                                   status_code=r.status_code,
                                                   data=r.content)
            raise exc_cls(exc_msg)
Beispiel #53
0
    def __get_token_gss(self):
        """
        Sends a request to get an auth token from the server and stores it as a class attribute. Uses Kerberos authentication.

        :returns: True if the token was successfully received. False otherwise.
        """
        if not EXTRA_MODULES['requests_kerberos']:
            raise MissingModuleException(
                'The requests-kerberos module is not installed.')

        headers = {'X-Rucio-Account': self.account}
        url = build_url(self.auth_host, path='auth/gss')

        result = None
        for retry in range(self.AUTH_RETRIES + 1):
            try:
                result = self.session.get(url,
                                          headers=headers,
                                          verify=self.ca_cert,
                                          auth=HTTPKerberosAuth())
                break
            except ConnectionError as error:
                LOG.warning('ConnectionError: ' + str(error))
                self.ca_cert = False
                if retry > self.request_retries:
                    raise

        if not result:
            LOG.error('cannot get auth_token')
            return False

        if result.status_code != codes.ok:  # pylint: disable-msg=E1101
            exc_cls, exc_msg = self._get_exception(
                headers=result.headers,
                status_code=result.status_code,
                data=result.content)
            raise exc_cls(exc_msg)

        self.auth_token = result.headers['x-rucio-auth-token']
        LOG.debug('got new token')
        return True
Beispiel #54
0
    def __get_token_userpass(self):
        """
        Sends a request to get an auth token from the server and stores it as a class attribute. Uses username/password.

        :returns: True if the token was successfully received. False otherwise.
        """

        headers = {
            'X-Rucio-Account': self.account,
            'X-Rucio-Username': self.creds['username'],
            'X-Rucio-Password': self.creds['password']
        }
        url = build_url(self.auth_host, path='auth/userpass')

        result = None
        for retry in range(self.AUTH_RETRIES + 1):
            try:
                result = self.session.get(url,
                                          headers=headers,
                                          verify=self.ca_cert)
                break
            except ConnectionError as error:
                LOG.warning('ConnectionError: ' + str(error))
                self.ca_cert = False
                if retry > self.request_retries:
                    raise

        if not result or 'result' not in locals():
            LOG.error('cannot get auth_token')
            return False

        if result.status_code != codes.ok:  # pylint: disable-msg=E1101
            exc_cls, exc_msg = self._get_exception(
                headers=result.headers,
                status_code=result.status_code,
                data=result.content)
            raise exc_cls(exc_msg)

        self.auth_token = result.headers['x-rucio-auth-token']
        LOG.debug('got new token')
        return True
Beispiel #55
0
    def update_subscription(self, name, account=None, filter=None, replication_rules=None, comments=None, lifetime=None, retroactive=None, dry_run=None, priority=None):
        """
        Updates a subscription

        :param name: Name of the subscription
        :type:  String
        :param account: Account identifier
        :type account:  String
        :param filter: Dictionary of attributes by which the input data should be filtered
                       **Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
        :type filter:  Dict
        :param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
        :type replication_rules:  Dict
        :param comments: Comments for the subscription
        :type comments:  String
        :param lifetime: Subscription's lifetime (days); False if subscription has no lifetime
        :type lifetime:  Integer or False
        :param retroactive: Flag to know if the subscription should be applied on previous data
        :type retroactive:  Boolean
        :param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
        :type dry_run:  Boolean
        :param priority: The priority of the subscription
        :type priority: Integer
        :raises: exception.NotFound if subscription is not found
        """
        if not account:
            account = self.account
        path = self.SUB_BASEURL + '/' + account + '/' + name
        url = build_url(choice(self.list_hosts), path=path)
        if filter and type(filter) != dict:
            raise TypeError('filter should be a dict')
        if replication_rules and type(replication_rules) != list:
            raise TypeError('replication_rules should be a list')
        data = dumps({'options': {'filter': filter, 'replication_rules': replication_rules, 'comments': comments,
                      'lifetime': lifetime, 'retroactive': retroactive, 'dry_run': dry_run, 'priority': priority}})
        r = self._send_request(url, type='PUT', data=data)
        if r.status_code == codes.created:
            return True
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
            raise exc_cls(exc_msg)
Beispiel #56
0
    def declare_suspicious_file_replicas(self, pfns, reason):
        """
        Declare a list of bad replicas.

        :param pfns: The list of PFNs.
        :param reason: The reason of the loss.
        """
        data = {'reason': reason, 'pfns': pfns}
        url = build_url(self.host,
                        path='/'.join([self.REPLICAS_BASEURL, 'suspicious']))
        headers = {}
        r = self._send_request(url,
                               headers=headers,
                               type='POST',
                               data=dumps(data))
        if r.status_code == codes.created:
            return loads(r.text)
        exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                               status_code=r.status_code,
                                               data=r.content)
        raise exc_cls(exc_msg)
Beispiel #57
0
 def get_did_meta(self, scope, name):
     """
     Get all metadata for a given did
     :param scope: the scope of did
     :param name: the name of the did
     """
     path = '/'.join([
         self.DIDS_BASEURL,
         quote_plus(scope),
         quote_plus(name), 'did_meta'
     ])
     url = build_url(choice(self.list_hosts), path=path)
     r = self._send_request(url, type='GET')
     if r.status_code == codes.ok:
         meta = self._load_json_data(r)
         return next(meta)
     else:
         exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                                status_code=r.status_code,
                                                data=r.content)
         raise exc_cls(exc_msg)
Beispiel #58
0
    def get_dataset_locks_by_rse(self, rse):
        """
        Get all dataset locks of the specified rse.

        :param rse: the rse of the locks to list.
        """

        path = '/'.join([self.LOCKS_BASEURL, rse])
        url = build_url(choice(self.list_hosts),
                        path=path,
                        params={'did_type': 'dataset'})

        result = self._send_request(url)
        if result.status_code == codes.ok:  # pylint: disable-msg=E1101
            locks = self._load_json_data(result)
            return locks
        else:
            exc_cls, exc_msg = self._get_exception(
                headers=result.headers, status_code=result.status_code)

            raise exc_cls(exc_msg)
Beispiel #59
0
    def list_rses(self, rse_expression=None):
        """
        Sends the request to list all rucio locations(RSEs).

        :rse_expression: RSE Expression to use as filter.
        :return:         a list containing the names of all rucio locations.
        """
        if rse_expression:
            path = ['rses', "?expression=" + quote(rse_expression)]
            path = '/'.join(path)
        else:
            path = 'rses/'
        url = build_url(choice(self.list_hosts), path=path)
        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            return self._load_json_data(r)
        else:
            exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                                   status_code=r.status_code,
                                                   data=r.content)
            raise exc_cls(exc_msg)
Beispiel #60
0
    def list_datasets_per_rse(self, rse, filters=None, limit=None):
        """
        List datasets at a RSE.

        :param rse: the rse name.
        :param filters: dictionary of attributes by which the results should be filtered.
        :param limit: limit number.

        :returns: A list of dict dataset replicas.

        """
        url = build_url(self.host,
                        path='/'.join([self.REPLICAS_BASEURL, 'rse', rse]))
        r = self._send_request(url, type='GET')
        if r.status_code == codes.ok:
            return self._load_json_data(r)

        exc_cls, exc_msg = self._get_exception(headers=r.headers,
                                               status_code=r.status_code,
                                               data=r.content)
        raise exc_cls(exc_msg)