Esempio n. 1
0
    def _load_json_data(self, response):
        """
        Helper method to correctly load json data based on the content type of the http response.

        :param response: the response received from the server.
        """
        if 'content-type' in response.headers and response.headers['content-type'] == 'application/x-json-stream':
            for line in response.iter_lines():
                if line:
                    yield parse_response(line)
        elif 'content-type' in response.headers and response.headers['content-type'] == 'application/json':
            yield parse_response(response.text)
        else:  # Exception ?
            yield response.text
Esempio n. 2
0
    def _get_exception(self, headers, status_code=None, data=None):
        """
        Helper method to parse an error string send by the server and transform it into the corresponding rucio exception.

        :param headers: The http response header containing the Rucio exception details.
        :param status_code: The http status code.
        :param data: The data with the ExceptionMessage.

        :return: A rucio exception class and an error string.
        """
        try:
            data = parse_response(data)
        except ValueError:
            data = {}
        if 'ExceptionClass' not in data:
            if 'ExceptionMessage' not in data:
                human_http_code = _codes.get(
                    status_code, None)  # NOQA, pylint: disable-msg=W0612
                return getattr(
                    exception, 'RucioException'
                ), 'no error information passed (http status code: %(status_code)s %(human_http_code)s)' % locals(
                )
            return getattr(exception,
                           'RucioException'), data['ExceptionMessage']

        exc_cls = None
        try:
            exc_cls = getattr(exception, data['ExceptionClass'])
        except AttributeError:
            return getattr(exception,
                           'RucioException'), data['ExceptionMessage']

        return exc_cls, data['ExceptionMessage']
Esempio n. 3
0
File: did.py Progetto: pic-es/rucio
 def POST(self):
     """
     List all meta of a list of data identifiers.
     HTTP Success:
         200 OK
     HTTP Error:
         400 Bad Request
         401 Unauthorized
         404 DataIdentifierNotFound
         500 InternalError
     :returns: A list of dictionaries containing all meta.
     """
     header('Content-Type', 'application/x-json-stream')
     json_data = data()
     try:
         params = parse_response(json_data)
         dids = params['dids']
     except KeyError as error:
         raise generate_http_error(400, 'ValueError', 'Cannot find mandatory parameter : %s' % str(error))
     except ValueError:
         raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
     try:
         for meta in get_metadata_bulk(dids, vo=ctx.env.get('vo')):
             yield render_json(**meta) + '\n'
     except ValueError:
         raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
     except DataIdentifierNotFound as error:
         raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0])
     except RucioException as error:
         raise generate_http_error(500, error.__class__.__name__, error.args[0])
     except Exception as error:
         print(format_exc())
         raise InternalError(error)
Esempio n. 4
0
    def post(self):
        """
        Set a tombstone on a list of replicas.

        .. :quickref: Tombstone; Set a tombstone on a list of replicas.

        :<json string replicas: list fo replicas
        :resheader Content-Type: application/x-json-string
        :status 201: Created.
        :status 401: Invalid auth token.
        :status 404: ReplicaNotFound.
        :status 500: Internal Error.
        """

        json_data = request.data
        replicas = []

        try:
            params = parse_response(json_data)
            if 'replicas' in params:
                replicas = params['replicas']
        except ValueError:
            return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list')

        try:
            for replica in replicas:
                set_tombstone(replica['rse'], replica['scope'], replica['name'], issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
        except ReplicaNotFound as error:
            return generate_http_error_flask(404, 'ReplicaNotFound', error.args[0])
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
        except Exception as error:
            print(format_exc())
            return str(error), 500
        return 'Created', 201
Esempio n. 5
0
    def put(self):
        """
        Update a file replicas state at a given RSE.

        .. :quickref: Replicas; update replicas state.

        :<json string rse: The RSE name.
        :<json list files: list of dicts with 'scope', 'name' and 'state'.
        :status 201: Replica successfully updated.
        :status 400: Cannot decode json parameter list.
        :status 401: Invalid auth token.
        :status 500: Internal Error.
        """
        json_data = request.data
        try:
            parameters = parse_response(json_data)
        except ValueError:
            return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list')

        try:
            update_replicas_states(rse=parameters['rse'], files=parameters['files'], issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
        except AccessDenied as error:
            return generate_http_error_flask(401, 'AccessDenied', error.args[0])
        except UnsupportedOperation as error:
            return generate_http_error_flask(500, 'UnsupportedOperation', error.args[0])
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
        except Exception as error:
            print(format_exc())
            return str(error), 500
        return '', 200
Esempio n. 6
0
    def POST(self):
        """
        Declare a list of suspicious replicas.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 InternalError

        """
        json_data = data()
        pfns = []
        header('Content-Type', 'application/x-json-stream')
        try:
            params = parse_response(json_data)
            if 'pfns' in params:
                pfns = params['pfns']
            if 'reason' in params:
                reason = params['reason']
        except ValueError:
            raise generate_http_error(400, 'ValueError',
                                      'Cannot decode json parameter list')

        not_declared_files = {}
        try:
            not_declared_files = declare_suspicious_file_replicas(
                pfns=pfns, reason=reason, issuer=ctx.env.get('issuer'))
        except ReplicaNotFound, e:
            raise generate_http_error(404, 'ReplicaNotFound', e.args[0][0])
Esempio n. 7
0
    def POST(self):
        """
        List the DIDs associated to a list of replicas.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 InternalError

        :returns: A list of dictionaries containing the mAPPing PFNs to DIDs.
        """
        json_data = data()
        rse, pfns = None, []
        header('Content-Type', 'application/x-json-stream')
        rse = None
        try:
            params = parse_response(json_data)
            if 'pfns' in params:
                pfns = params['pfns']
            if 'rse' in params:
                rse = params['rse']
        except ValueError:
            raise generate_http_error(400, 'ValueError',
                                      'Cannot decode json parameter list')

        try:
            for pfn in get_did_from_pfns(pfns, rse):
                yield dumps(pfn) + '\n'
        except RucioException, e:
            raise generate_http_error(500, e.__class__.__name__, e.args[0][0])
Esempio n. 8
0
    def DELETE(self):
        """
        Delete file replicas at a given RSE.

        HTTP Success:
            200 Ok

        HTTP Error:
            401 Unauthorized
            409 Conflict
            500 Internal Error
        """
        json_data = data()
        try:
            parameters = parse_response(json_data)
        except ValueError:
            raise generate_http_error(400, 'ValueError',
                                      'Cannot decode json parameter list')

        try:
            delete_replicas(rse=parameters['rse'],
                            files=parameters['files'],
                            issuer=ctx.env.get('issuer'),
                            ignore_availability=parameters.get(
                                'ignore_availability', False))
        except AccessDenied, e:
            raise generate_http_error(401, 'AccessDenied', e.args[0][0])
Esempio n. 9
0
    def POST(self):
        """
        Create file replicas at a given RSE.

        HTTP Success:
            201 Created

        HTTP Error:
            401 Unauthorized
            409 Conflict
            500 Internal Error
        """
        json_data = data()
        try:
            parameters = parse_response(json_data)
        except ValueError:
            raise generate_http_error(400, 'ValueError',
                                      'Cannot decode json parameter list')

        try:
            add_replicas(rse=parameters['rse'],
                         files=parameters['files'],
                         issuer=ctx.env.get('issuer'),
                         ignore_availability=parameters.get(
                             'ignore_availability', False))
        except InvalidPath, e:
            raise generate_http_error(400, 'InvalidPath', e.args[0][0])
Esempio n. 10
0
    def POST(self, rse, operation):
        """
        Get URLs for files at a given RSE.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 Internal Error
        """
        header('Content-Type', 'application/json')
        json_data = data()
        try:
            parameters = parse_response(json_data)
        except ValueError:
            raise generate_http_error(400, 'ValueError',
                                      'Cannot decode json parameter list')

        try:
            result = objectstore.get_signed_urls(parameters,
                                                 rse=rse,
                                                 operation=operation)
            for url in result:
                if isinstance(result[url], Exception):
                    raise result[url]
            return render_json(**result)
        except RucioException as error:
            raise generate_http_error(500, error.__class__.__name__,
                                      error.args[0])
        except Exception as error:
            print traceback.format_exc()
            raise InternalError(error)
Esempio n. 11
0
    def POST(self, rse):
        """
        Get files metadata at a given RSE.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 Internal Error
        """
        header('Content-Type', 'application/json')
        json_data = data()
        try:
            parameters = parse_response(json_data)
        except ValueError:
            raise generate_http_error(400, 'ValueError',
                                      'Cannot decode json parameter list')

        try:
            url = parameters['url']
            new_url = parameters['new_url']
            objectstore.rename(url, new_url, rse=rse)
        except RucioException as error:
            raise generate_http_error(500, error.__class__.__name__,
                                      error.args[0])
        except Exception as error:
            print traceback.format_exc()
            raise InternalError(error)

        raise OK()
Esempio n. 12
0
    def POST(self):
        """
        Set a tombstone on a list of replicas.

        HTTP Success:
            201 OK

        HTTP Error:
            401 Unauthorized
            404 ReplicaNotFound
            500 InternalError
        """
        json_data = data()
        replicas = []
        try:
            params = parse_response(json_data)
            if 'replicas' in params:
                replicas = params['replicas']
        except ValueError:
            raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')

        try:
            for replica in replicas:
                set_tombstone(replica['rse'], replica['scope'], replica['name'], issuer=ctx.env.get('issuer'), vo=ctx.env.get('vo'))
        except ReplicaNotFound as error:
            raise generate_http_error(404, 'ReplicaNotFound', error.args[0])
        except RucioException as error:
            raise generate_http_error(500, error.__class__.__name__, error.args[0])
        except Exception as error:
            print(format_exc())
            raise InternalError(error)
        raise Created()
Esempio n. 13
0
    def POST(self):
        """ Import data into Rucio.

        HTTP Success:
            200 OK

        HTTP Error:
            400 Bad request
            401 Unauthorized
            404 Resource not Found
            500 InternalError
        """
        header('Content-Type', 'application/x-json-stream')
        json_data = data()
        try:
            data_to_import = json_data and parse_response(json_data)
        except ValueError:
            raise generate_http_error(
                400, 'ValueError', 'Cannot decode json parameter dictionary')

        try:
            import_data(data=data_to_import,
                        issuer=ctx.env.get('issuer'),
                        vo=ctx.env.get('vo'))
        except RucioException as error:
            raise generate_http_error(500, error.__class__.__name__,
                                      error.args[0])

        raise Created()
Esempio n. 14
0
    def _load_json_data(self, response):
        """
        Helper method to correctly load json data based on the content type of the http response.

        :param response: the response received from the server.
        """
        if 'content-type' in response.headers and response.headers[
                'content-type'] == 'application/x-json-stream':
            for line in response.iter_lines():
                if line:
                    yield parse_response(line)
        elif 'content-type' in response.headers and response.headers[
                'content-type'] == 'application/json':
            yield parse_response(response.text)
        else:  # Exception ?
            yield response.text
Esempio n. 15
0
    def post(self, rse):
        """
        Pass list of urls and return their metadata.

        .. :quickref: ObjectStoreInfo; Get files metadata at a given RSE.

        :param rse: RSE name.
        :<json string urls: A list of URL string.
        :resheader Content-Type: application/json
        :status 200: OK.
        :status 400: Cannot decode json parameter list.
        :status 401: Invalid Auth Token.
        :status 500: Internal Error.
        :returns: Dictonary of metadata.
        """
        json_data = request.data
        try:
            parameters = parse_response(json_data)
        except ValueError:
            return generate_http_error_flask(
                400, 'ValueError', 'Cannot decode json parameter list')

        try:
            result = objectstore.get_metadata(parameters, rse=rse)
            for url in result:
                if isinstance(result[url], Exception):
                    return Response(result[url])
            return Response(render_json(**result),
                            content_type="application/json")
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__,
                                             error.args[0])
        except Exception as error:
            print(traceback.format_exc())
            return error, 500
Esempio n. 16
0
    def POST(self):
        """
        Declare a list of bad replicas.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 InternalError

        """
        json_data = data()
        rse, pfns = None, []
        header('Content-Type', 'application/x-json-stream')
        try:
            params = parse_response(json_data)
            if 'pfns' in params:
                pfns = params['pfns']
            if 'rse' in params:
                rse = params['rse']
        except ValueError:
            raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')

        try:
            declare_bad_file_replicas(rse=rse, pfns=pfns, issuer=ctx.env.get('issuer'))
        except ReplicaNotFound, e:
            raise generate_http_error(404, 'ReplicaNotFound', e.args[0][0])
Esempio n. 17
0
    def PUT(self):
        """
        Update a file replicas state at a given RSE.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 Internal Error
        """
        json_data = data()
        try:
            parameters = parse_response(json_data)
        except ValueError:
            raise generate_http_error(400, 'ValueError',
                                      'Cannot decode json parameter list')

        try:
            update_replicas_states(rse=parameters['rse'],
                                   files=parameters['files'],
                                   issuer=ctx.env.get('issuer'))
        except AccessDenied as error:
            raise generate_http_error(401, 'AccessDenied', error.args[0])
        except UnsupportedOperation as error:
            raise generate_http_error(500, 'UnsupportedOperation',
                                      error.args[0])
        except RucioException as error:
            raise generate_http_error(500, error.__class__.__name__,
                                      error.args[0])
        except Exception as error:
            print(format_exc())
            raise InternalError(error)
        raise OK()
Esempio n. 18
0
    def post(self, rse):
        """
        Rename object.

        .. :quickref: ObjectStoreRename; Rename object.

        :param rse: the RSE name.
        :<json string url: the URL string.
        :<json string new_url: the new URL string.
        :<json string urls: A list of URL string.
        :resheader Content-Type: application/json
        :status 200: OK.
        :status 400: Cannot decode json parameter list.
        :status 401: Invalid Auth Token.
        :status 500: Internal Error.
        """
        json_data = request.data
        try:
            parameters = parse_response(json_data)
        except ValueError:
            return generate_http_error_flask(
                400, 'ValueError', 'Cannot decode json parameter list')

        try:
            url = parameters['url']
            new_url = parameters['new_url']
            objectstore.rename(url, new_url, rse=rse)
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__,
                                             error.args[0])
        except Exception as error:
            print(traceback.format_exc())
            return error, 500

        return "OK", 200
Esempio n. 19
0
 def check_error_api(params, exception_class, exception_message, code):
     headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'}
     response = rest_client.get('/requests/list', query_string=params, headers=headers(auth(auth_token), vohdr(vo), hdrdict(headers_dict)))
     assert response.status_code == code
     body = parse_response(response.get_data(as_text=True))
     assert body['ExceptionClass'] == exception_class
     assert body['ExceptionMessage'] == exception_message
Esempio n. 20
0
    def POST(self):
        """
        List the DIDs associated to a list of replicas.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 InternalError

        :returns: A list of dictionaries containing the mapping PFNs to DIDs.
        """
        json_data = data()
        rse, pfns = None, []
        header('Content-Type', 'application/x-json-stream')
        try:
            params = parse_response(json_data)
            if 'pfns' in params:
                pfns = params['pfns']
            if 'rse' in params:
                rse = params['rse']
        except ValueError:
            raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')

        try:
            for pfn in get_did_from_pfns(pfns, rse):
                yield dumps(pfn) + '\n'
        except RucioException, e:
            raise generate_http_error(500, e.__class__.__name__, e.args[0][0])
Esempio n. 21
0
    def post(self):
        """
        Declare a list of bad PFNs.

        .. :quickref: BadPFNs; Declare bad replicas.

        :<json string pfns: The list of PFNs.
        :<json string reason: The reason of the loss.
        :<json string state: The state is eiher BAD, SUSPICIOUS or TEMPORARY_UNAVAILABLE.
        :<json string expires_at: The expiration date. Only apply to TEMPORARY_UNAVAILABLE.
        :resheader Content-Type: application/x-json-string
        :status 201: Created.
        :status 400: Cannot decode json parameter list.
        :status 401: Invalid auth token.
        :status 404: Replica not found.
        :status 500: Internal Error.
        :returns: A list of not successfully declared files.
        """

        json_data = request.data
        pfns = []
        reason = None
        state = None
        expires_at = None
        try:
            params = parse_response(json_data)
            if 'pfns' in params:
                pfns = params['pfns']
            if 'reason' in params:
                reason = params['reason']
            if 'reason' in params:
                reason = params['reason']
            if 'state' in params:
                reason = params['state']
            if 'expires_at' in params:
                expires_at = datetime.strptime(params['expires_at'],
                                               "%Y-%m-%dT%H:%M:%S.%f")
            add_bad_pfns(pfns=pfns,
                         issuer=request.environ.get('issuer'),
                         state=state,
                         reason=reason,
                         expires_at=expires_at)
        except (ValueError, InvalidType) as error:
            return generate_http_error_flask(400, 'ValueError', error.args[0])
        except AccessDenied as error:
            return generate_http_error_flask(401, 'AccessDenied',
                                             error.args[0])
        except ReplicaNotFound as error:
            return generate_http_error_flask(404, 'ReplicaNotFound',
                                             error.args[0])
        except Duplicate as error:
            return generate_http_error_flask(409, 'Duplicate', error.args[0])
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__,
                                             error.args[0])
        except Exception as error:
            print(format_exc())
            return error, 500
        return 'Created', 201
Esempio n. 22
0
    def test_auth_ssh(self):
        """ MULTI VO (REST): Test ssh authentication to multiple VOs """
        mw = []

        try:
            add_account_identity(PUBLIC_KEY, 'SSH', 'root', '*****@*****.**', 'root', **self.vo)
            add_account_identity(PUBLIC_KEY, 'SSH', 'root', '*****@*****.**', 'root', **self.new_vo)
        except Duplicate:
            pass  # Might already exist, can skip

        headers_tst = {'X-Rucio-Account': 'root'}
        headers_tst.update(self.vo_header)
        res_tst = TestApp(auth_app.wsgifunc(*mw)).get('/ssh_challenge_token', headers=headers_tst, expect_errors=True)
        assert_equal(res_tst.status, 200)
        challenge_tst = str(res_tst.header('X-Rucio-SSH-Challenge-Token'))
        headers_tst.update({'X-Rucio-SSH-Signature': ssh_sign(PRIVATE_KEY, challenge_tst)})
        res_tst = TestApp(auth_app.wsgifunc(*mw)).get('/ssh', headers=headers_tst, expect_errors=True)
        assert_equal(res_tst.status, 200)
        token_tst = str(res_tst.header('X-Rucio-Auth-Token'))

        headers_new = {'X-Rucio-Account': 'root'}
        headers_new.update(self.new_header)
        res_new = TestApp(auth_app.wsgifunc(*mw)).get('/ssh_challenge_token', headers=headers_new, expect_errors=True)
        assert_equal(res_new.status, 200)
        challenge_tst = str(res_new.header('X-Rucio-SSH-Challenge-Token'))
        headers_new.update({'X-Rucio-SSH-Signature': ssh_sign(PRIVATE_KEY, challenge_tst)})
        res_new = TestApp(auth_app.wsgifunc(*mw)).get('/ssh', headers=headers_new, expect_errors=True)
        assert_equal(res_new.status, 200)
        token_new = str(res_new.header('X-Rucio-Auth-Token'))

        headers_tst = {'X-Rucio-Auth-Token': str(token_tst)}
        res_tst = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_tst, expect_errors=True)
        assert_equal(res_tst.status, 200)
        accounts_tst = [parse_response(a)['account'] for a in res_tst.body.decode().split('\n')[:-1]]
        assert_not_equal(len(accounts_tst), 0)
        assert_in(self.account_tst, accounts_tst)
        assert_not_in(self.account_new, accounts_tst)

        headers_new = {'X-Rucio-Auth-Token': str(token_new)}
        res_new = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_new, expect_errors=True)
        assert_equal(res_new.status, 200)
        accounts_new = [parse_response(a)['account'] for a in res_new.body.decode().split('\n')[:-1]]
        assert_not_equal(len(accounts_new), 0)
        assert_in(self.account_new, accounts_new)
        assert_not_in(self.account_tst, accounts_new)
Esempio n. 23
0
 def check_correct_api(params, expected_requests):
     headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'}
     response = rest_client.get('/requests/history/list', query_string=params, headers=headers(auth(auth_token), vohdr(vo), hdrdict(headers_dict)))
     assert response.status_code == 200
     requests = set()
     for request in response.get_data(as_text=True).split('\n')[:-1]:
         request = parse_response(request)
         requests.add((request['state'], request['source_rse_id'], request['dest_rse_id'], request['name']))
     assert requests == expected_requests
Esempio n. 24
0
    def post(self):
        """
        Create file replicas at a given RSE.

        .. :quickref: Replicas; create replicas at RSE

        :<json string rse: The RSE name.
        :<json list files: list of dicts with 'scope', 'name', 'bytes', 'meta' and 'adler32'.
        :<json bool ignore_availability: Flag to ignore the RSE blacklisting.
        :status 201: Replica Successfully created.
        :status 400: Invalid Path.
        :status 401: Invalid auth token.
        :status 404: RSE not found.
        :status 409: Replica already exists.
        :status 409: DID already exists.
        :status 503: Resource Temporary Unavailable.
        """
        json_data = request.data
        try:
            parameters = parse_response(json_data)
        except ValueError:
            return generate_http_error_flask(
                400, 'ValueError', 'Cannot decode json parameter list')

        try:
            add_replicas(rse=parameters['rse'],
                         files=parameters['files'],
                         issuer=request.environ.get('issuer'),
                         vo=request.environ.get('vo'),
                         ignore_availability=parameters.get(
                             'ignore_availability', False))
        except InvalidPath as error:
            return generate_http_error_flask(400, 'InvalidPath', error.args[0])
        except AccessDenied as error:
            return generate_http_error_flask(401, 'AccessDenied',
                                             error.args[0])
        except Duplicate as error:
            return generate_http_error_flask(409, 'Duplicate', error.args[0])
        except DataIdentifierAlreadyExists as error:
            return generate_http_error_flask(409,
                                             'DataIdentifierAlreadyExists',
                                             error.args[0])
        except RSENotFound as error:
            return generate_http_error_flask(404, 'RSENotFound', error.args[0])
        except ResourceTemporaryUnavailable as error:
            return generate_http_error_flask(503,
                                             'ResourceTemporaryUnavailable',
                                             error.args[0])
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__,
                                             error.args[0])
        except Exception as error:
            print(format_exc())
            return error, 500
        return 'Created', 201
Esempio n. 25
0
    def POST(self):
        """
        Declare a list of bad replicas by DID.

        HTTP Success:
            200 OK

        HTTP Error:
            400 BadRequest
            401 Unauthorized
            409 Conflict
            500 InternalError

        """
        json_data = data()
        dids = []
        rse = None
        reason = None
        expires_at = None
        header('Content-Type', 'application/x-json-stream')
        try:
            params = parse_response(json_data)
            if 'dids' in params:
                dids = params['dids']
            if 'rse' in params:
                rse = params['rse']
            if 'reason' in params:
                reason = params['reason']
            state = BadFilesStatus.BAD
            if 'expires_at' in params and params['expires_at']:
                expires_at = datetime.strptime(params['expires_at'],
                                               "%Y-%m-%dT%H:%M:%S.%f")
            not_declared_files = add_bad_dids(dids=dids,
                                              rse=rse,
                                              issuer=ctx.env.get('issuer'),
                                              state=state,
                                              reason=reason,
                                              expires_at=expires_at,
                                              vo=ctx.env.get('vo'))
        except (ValueError, InvalidType) as error:
            raise generate_http_error(400, 'ValueError', error.args[0])
        except AccessDenied as error:
            raise generate_http_error(401, 'AccessDenied', error.args[0])
        except ReplicaNotFound as error:
            raise generate_http_error(404, 'ReplicaNotFound', error.args[0])
        except Duplicate as error:
            raise generate_http_error(409, 'Duplicate', error.args[0])
        except RucioException as error:
            raise generate_http_error(500, error.__class__.__name__,
                                      error.args[0])
        except Exception as error:
            print(format_exc())
            raise InternalError(error)
        raise Created(dumps(not_declared_files))
Esempio n. 26
0
    def post(self):
        """
        List dataset replicas for multiple DIDs.

        .. :quickref: DatasetReplicasBulk; List dataset replicas (bulk).

        :<json list dids: List of DIDs for querying the datasets.
        :resheader Content-Type: application/x-json-stream
        :status 200: OK.
        :status 400: Bad Request.
        :status 401: Invalid auth token.
        :status 406: Not Acceptable.
        :status 500: Internal Error.
        :returns: A dictionary containing all replicas information.
        """

        json_data = request.data
        try:
            params = parse_response(json_data)
            dids = params['dids']
            didslength = len(dids)
        except KeyError as error:
            return generate_http_error_flask(
                400, 'KeyError',
                'Cannot find mandatory parameter : %s' % str(error))
        except ValueError:
            return generate_http_error_flask(
                400, 'ValueError', 'Cannot decode json parameter list')
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__,
                                             error.args[0])
        except Exception as error:
            print(format_exc())
            return error, 500
        if didslength == 0:
            return generate_http_error_flask(400, 'ValueError',
                                             'List of DIDs is empty')
        try:
            data = ""
            for row in list_dataset_replicas_bulk(
                    dids=dids, vo=request.environ.get('vo')):
                data += dumps(row, cls=APIEncoder) + '\n'
            return Response(data, content_type='application/x-json-stream')
        except InvalidObject as error:
            return generate_http_error_flask(
                400, 'InvalidObject',
                'Cannot validate DIDs: %s' % (str(error)))
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__,
                                             error.args[0])
        except Exception as error:
            print(format_exc())
            return error, 500
Esempio n. 27
0
    def POST(self):
        """
        Create file replicas at a given RSE.

        HTTP Success:
            201 Created

        HTTP Error:
            401 Unauthorized
            405 Method Not Allowed
            409 Conflict
            500 Internal Error
            503 Service Unavailable
        """
        json_data = data()
        try:
            parameters = parse_response(json_data)
        except (ValueError, InvalidType):
            raise generate_http_error(400, 'ValueError',
                                      'Cannot decode json parameter list')

        try:
            add_files(lfns=parameters['lfns'],
                      issuer=ctx.env.get('issuer'),
                      ignore_availability=parameters.get(
                          'ignore_availability', False))
        except InvalidPath as error:
            raise generate_http_error(400, 'InvalidPath', error.args[0])
        except AccessDenied as error:
            raise generate_http_error(401, 'AccessDenied', error.args[0])
        except UnsupportedOperation as error:
            raise generate_http_error(405, 'UnsupportedOperation',
                                      error.args[0])
        except Duplicate as error:
            raise generate_http_error(409, 'Duplicate', error.args[0])
        except DataIdentifierAlreadyExists as error:
            raise generate_http_error(409, 'DataIdentifierAlreadyExists',
                                      error.args[0])
        except RSENotFound as error:
            raise generate_http_error(404, 'RSENotFound', error.args[0])
        except DatabaseException as error:
            raise generate_http_error(503, 'DatabaseException', error.args[0])
        except ResourceTemporaryUnavailable as error:
            raise generate_http_error(503, 'ResourceTemporaryUnavailable',
                                      error.args[0])
        except RucioException as error:
            print(format_exc())
            raise generate_http_error(500, error.__class__.__name__,
                                      error.args[0])
        except Exception as error:
            print(format_exc())
            raise InternalError(error)
        raise Created()
Esempio n. 28
0
    def post(self):
        """
        Atomic method used by the RucioFileCatalog plugin in Dirac that :
        - Creates files and their replicas
        - Creates the dataset containing the files and attach the files to the dataset
        - Creates a rule on the dataset with RSE expression ANY and grouping NONE
        - Creates all the container hierarchy containing the dataset

        ..:quickref: AddFiles; Method used by the RucioFileCatalog plugin in Dirac.

        :<json list lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}.
        :<json bool ignore_availability: A boolean to choose if unavailable sites need to be ignored.

        :status 201: Created.
        :status 400: Cannot decode json parameter list.
        :status 401: Invalid auth token.
        :status 404: DID not found.
        :status 405: Unsupported Operation.
        :status 409: Duplicate.
        :status 500: Internal Error.
        :status 503: Temporary error.
        """
        try:
            parameters = parse_response(request.data)
        except (ValueError, InvalidType):
            return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list')

        try:
            add_files(lfns=parameters['lfns'], issuer=request.environ.get('issuer'), ignore_availability=parameters.get('ignore_availability', False))
        except InvalidPath as error:
            return generate_http_error_flask(400, 'InvalidPath', error.args[0])
        except AccessDenied as error:
            return generate_http_error_flask(401, 'AccessDenied', error.args[0])
        except UnsupportedOperation as error:
            return generate_http_error_flask(405, 'UnsupportedOperation', error.args[0])
        except Duplicate as error:
            return generate_http_error_flask(409, 'Duplicate', error.args[0])
        except DataIdentifierAlreadyExists as error:
            return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0])
        except RSENotFound as error:
            return generate_http_error_flask(404, 'RSENotFound', error.args[0])
        except DatabaseException as error:
            return generate_http_error_flask(503, 'DatabaseException', error.args[0])
        except ResourceTemporaryUnavailable as error:
            return generate_http_error_flask(503, 'ResourceTemporaryUnavailable', error.args[0])
        except RucioException as error:
            logging.exception("Internal Error")
            return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
        except Exception as error:
            logging.exception("Internal Error")
            return str(error), 500
        return 'Created', 201
Esempio n. 29
0
    def POST(self):
        """
        Declare a list of bad PFNs.

        HTTP Success:
            200 OK

        HTTP Error:
            400 BadRequest
            401 Unauthorized
            409 Conflict
            500 InternalError

        """
        json_data = data()
        pfns = []
        reason = None
        state = None
        expires_at = None
        header('Content-Type', 'application/x-json-stream')
        try:
            params = parse_response(json_data)
            if 'pfns' in params:
                pfns = params['pfns']
            if 'reason' in params:
                reason = params['reason']
            if 'state' in params:
                state = params['state']
            if 'expires_at' in params:
                expires_at = datetime.strptime(params['expires_at'],
                                               "%Y-%m-%dT%H:%M:%S.%f")
            add_bad_pfns(pfns=pfns,
                         issuer=ctx.env.get('issuer'),
                         state=state,
                         reason=reason,
                         expires_at=expires_at)
        except (ValueError, InvalidType) as error:
            raise generate_http_error(400, 'ValueError', error.args[0])
        except AccessDenied as error:
            raise generate_http_error(401, 'AccessDenied', error.args[0])
        except ReplicaNotFound as error:
            raise generate_http_error(404, 'ReplicaNotFound', error.args[0])
        except Duplicate as error:
            raise generate_http_error(409, 'Duplicate', error.args[0])
        except RucioException as error:
            raise generate_http_error(500, error.__class__.__name__,
                                      error.args[0])
        except Exception as error:
            print(format_exc())
            raise InternalError(error)
        raise Created()
Esempio n. 30
0
    def POST(self):
        """
        List dataset replicas for multiple DIDs.

        HTTP Success:
            200 OK

        HTTP Error:
            400 Bad Request
            401 Unauthorized
            406 Not Acceptable
            500 InternalError

        :returns: A dictionary containing all replicas information.
        """
        header('Content-Type', 'application/x-json-stream')
        json_data = data()
        try:
            params = parse_response(json_data)
            dids = params['dids']
            didslength = len(dids)
        except KeyError as error:
            raise generate_http_error(
                400, 'KeyError',
                'Cannot find mandatory parameter : %s' % str(error))
        except ValueError:
            raise generate_http_error(400, 'ValueError',
                                      'Cannot decode json parameter list')
        except RucioException as error:
            raise generate_http_error(500, error.__class__.__name__,
                                      error.args[0])
        except Exception as error:
            print(format_exc())
            raise InternalError(error)
        if didslength == 0:
            raise generate_http_error(400, 'ValueError',
                                      'List of DIDs is empty')
        try:
            for row in list_dataset_replicas_bulk(dids=dids,
                                                  vo=ctx.env.get('vo')):
                yield dumps(row, cls=APIEncoder) + '\n'
        except InvalidObject as error:
            raise generate_http_error(
                400, 'InvalidObject',
                'Cannot validate DIDs: %s' % (str(error)))
        except RucioException as error:
            raise generate_http_error(500, error.__class__.__name__,
                                      error.args[0])
        except Exception as error:
            print(format_exc())
            raise InternalError(error)
Esempio n. 31
0
    def test_auth_gss(self):
        """ MULTI VO (REST): Test gss authentication to multiple VOs """
        mw = []

        # Can't rely on `requests_kerberos` module being present, so get tokens from API instead
        token_tst = get_auth_token_gss('root', '*****@*****.**', 'unknown', None, **self.vo).token
        token_new = get_auth_token_gss('root', '*****@*****.**', 'unknown', None, **self.new_vo).token

        headers_tst = {'X-Rucio-Auth-Token': str(token_tst)}
        res_tst = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_tst, expect_errors=True)
        assert_equal(res_tst.status, 200)
        accounts_tst = [parse_response(a)['account'] for a in res_tst.body.decode().split('\n')[:-1]]
        assert_not_equal(len(accounts_tst), 0)
        assert_in(self.account_tst, accounts_tst)
        assert_not_in(self.account_new, accounts_tst)

        headers_new = {'X-Rucio-Auth-Token': str(token_new)}
        res_new = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_new, expect_errors=True)
        assert_equal(res_new.status, 200)
        accounts_new = [parse_response(a)['account'] for a in res_new.body.decode().split('\n')[:-1]]
        assert_not_equal(len(accounts_new), 0)
        assert_in(self.account_new, accounts_new)
        assert_not_in(self.account_tst, accounts_new)
Esempio n. 32
0
    def test_auth_x509(self):
        """ MULTI VO (REST): Test X509 authentication to multiple VOs """
        mw = []

        # TestApp doesn't support `cert` argument, so get tokens from API instead
        token_tst = get_auth_token_x509('root', '/CN=Rucio User', 'unknown', None, **self.vo).token
        token_new = get_auth_token_x509('root', '/CN=Rucio User', 'unknown', None, **self.new_vo).token

        headers_tst = {'X-Rucio-Auth-Token': str(token_tst)}
        res_tst = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_tst, expect_errors=True)
        assert_equal(res_tst.status, 200)
        accounts_tst = [parse_response(a)['account'] for a in res_tst.body.decode().split('\n')[:-1]]
        assert_not_equal(len(accounts_tst), 0)
        assert_in(self.account_tst, accounts_tst)
        assert_not_in(self.account_new, accounts_tst)

        headers_new = {'X-Rucio-Auth-Token': str(token_new)}
        res_new = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_new, expect_errors=True)
        assert_equal(res_new.status, 200)
        accounts_new = [parse_response(a)['account'] for a in res_new.body.decode().split('\n')[:-1]]
        assert_not_equal(len(accounts_new), 0)
        assert_in(self.account_new, accounts_new)
        assert_not_in(self.account_tst, accounts_new)
Esempio n. 33
0
    def delete(self):
        """
        Delete file replicas at a given RSE.

        .. :quickref: Replicas; Delete replica at RSE.

        :<json string rse: The RSE name.
        :<json list files: list of dicts with 'scope', 'name'.
        :<json bool ignore_availability: Flag to ignore the RSE blacklisting.
        :status 200: Replica successfully deleted.
        :status 400: Cannot decode json parameter list.
        :status 401: Invalid auth token.
        :status 404: RSE not found.
        :status 404: Replica not found.
        :status 500: Internal Error.
        """
        json_data = request.data
        try:
            parameters = parse_response(json_data)
        except ValueError:
            return generate_http_error_flask(
                400, 'ValueError', 'Cannot decode json parameter list')

        try:
            delete_replicas(rse=parameters['rse'],
                            files=parameters['files'],
                            issuer=request.environ.get('issuer'),
                            vo=request.environ.get('vo'),
                            ignore_availability=parameters.get(
                                'ignore_availability', False))
        except AccessDenied as error:
            return generate_http_error_flask(401, 'AccessDenied',
                                             error.args[0])
        except RSENotFound as error:
            return generate_http_error_flask(404, 'RSENotFound', error.args[0])
        except ResourceTemporaryUnavailable as error:
            return generate_http_error_flask(503,
                                             'ResourceTemporaryUnavailable',
                                             error.args[0])
        except ReplicaNotFound as error:
            return generate_http_error_flask(404, 'ReplicaNotFound',
                                             error.args[0])
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__,
                                             error.args[0])
        except Exception as error:
            print(format_exc())
            return error, 500
        return 'OK', 200
Esempio n. 34
0
    def PUT(self):
        """
        Update a file replicas state at a given RSE.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 Internal Error
        """
        json_data = data()
        try:
            parameters = parse_response(json_data)
        except ValueError:
            raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')

        try:
            update_replicas_states(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer'))
        except AccessDenied, e:
            raise generate_http_error(401, 'AccessDenied', e.args[0][0])
Esempio n. 35
0
    def DELETE(self):
        """
        Delete file replicas at a given RSE.

        HTTP Success:
            200 Ok

        HTTP Error:
            401 Unauthorized
            409 Conflict
            500 Internal Error
        """
        json_data = data()
        try:
            parameters = parse_response(json_data)
        except ValueError:
            raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')

        try:
            delete_replicas(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer'), ignore_availability=parameters.get('ignore_availability', False))
        except AccessDenied, e:
            raise generate_http_error(401, 'AccessDenied', e.args[0][0])
Esempio n. 36
0
    def POST(self):
        """
        List all replicas for data identifiers.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 InternalError

        :returns: A dictionary containing all replicas information.
        :returns: A metalink description of replicas if metalink(4)+xml is specified in Accept:
        """

        metalink = None
        if ctx.env.get('HTTP_ACCEPT') is not None:
            tmp = ctx.env.get('HTTP_ACCEPT').split(',')
            # first check if client accepts metalink
            if 'application/metalink+xml' in tmp:
                metalink = 3
            # but prefer metalink4 if the client has support for it
            # (clients can put both in their ACCEPT header!)
            if 'application/metalink4+xml' in tmp:
                metalink = 4

        dids, schemes, select, unavailable, limit = [], None, None, False, None
        ignore_availability = False
        all_states = False
        json_data = data()
        try:
            params = parse_response(json_data)
            if 'dids' in params:
                dids = params['dids']
            if 'schemes' in params:
                schemes = params['schemes']
            if 'unavailable' in params:
                unavailable = params['unavailable']
                ignore_availability = True
            if 'all_states' in params:
                all_states = params['all_states']
        except ValueError:
            raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')

        if ctx.query:
            params = parse_qs(ctx.query[1:])
            if 'select' in params:
                select = params['select'][0]
            if 'limit' in params:
                limit = params['limit'][0]

        try:
            # first, set the appropriate content type, and stream the header
            if metalink is None:
                header('Content-Type', 'application/x-json-stream')
            elif metalink == 3:
                header('Content-Type', 'application/metalink+xml')
                yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink version="3.0" xmlns="http://www.metalinker.org/">\n<files>\n'
            elif metalink == 4:
                header('Content-Type', 'application/metalink4+xml')
                yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'

            # then, stream the replica information
            for rfile in list_replicas(dids=dids, schemes=schemes, unavailable=unavailable, request_id=ctx.env.get('request_id'), ignore_availability=ignore_availability, all_states=all_states):
                client_ip = ctx.get('ip')
                replicas = []
                dictreplica = {}
                for rse in rfile['rses']:
                    for replica in rfile['rses'][rse]:
                        replicas.append(replica)
                        dictreplica[replica] = rse
                if select == 'geoip':
                    replicas = geoIP_order(dictreplica, client_ip)
                else:
                    replicas = random_order(dictreplica, client_ip)
                if metalink is None:
                    yield dumps(rfile, cls=APIEncoder) + '\n'
                elif metalink == 3:
                    idx = 0
                    yield ' <file name="' + rfile['name'] + '">\n  <resources>\n'
                    for replica in replicas:
                        yield '   <url type="http" preference="' + str(idx) + '">' + replica + '</url>\n'
                        idx += 1
                        if limit and limit == idx:
                            break
                    yield '  </resources>\n </file>\n'
                elif metalink == 4:
                    yield ' <file name="' + rfile['name'] + '">\n'
                    yield '  <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'
                    if rfile['adler32'] is not None:
                        yield '  <hash type="adler32">' + rfile['adler32'] + '</hash>\n'
                    if rfile['md5'] is not None:
                        yield '  <hash type="md5">' + rfile['md5'] + '</hash>\n'
                    yield '  <size>' + str(rfile['bytes']) + '</size>\n'
                    idx = 0
                    for replica in replicas:
                        yield '   <url location="' + str(dictreplica[replica]) + '" priority="' + str(idx + 1) + '">' + replica + '</url>\n'
                        idx += 1
                        if limit and limit == idx:
                            break
                    yield ' </file>\n'

            # don't forget to send the metalink footer
            if metalink:
                if metalink == 3:
                    yield '</files>\n</metalink>\n'
                elif metalink == 4:
                    yield '</metalink>\n'

        except DataIdentifierNotFound, e:
            raise generate_http_error(404, 'DataIdentifierNotFound', e.args[0][0])