示例#1
0
    def get(self, scope_name, rse):
        """
        List request for given DID to a destination RSE.

        .. :quickref: RequestGet; list requests

        :param scope_name: data identifier (scope)/(name).
        :param rse: destination RSE.
        :reqheader Content-Type: application/json
        :status 200: Request found.
        :status 404: Request not found.
        :status 406: Not Acceptable.
        """
        try:
            scope, name = parse_scope_name(scope_name,
                                           flask.request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        try:
            request_data = request.get_request_by_did(
                scope=scope,
                name=name,
                rse=rse,
                issuer=flask.request.environ.get('issuer'),
                vo=flask.request.environ.get('vo'),
            )
            return Response(json.dumps(request_data, cls=APIEncoder),
                            content_type='application/json')
        except RequestNotFound as error:
            return generate_http_error_flask(
                404, error.__class__.__name__,
                f'No request found for DID {scope}:{name} at RSE {rse}')
示例#2
0
    def get(self, scope_name):
        """
        Returns the contents history of a data identifier.

        .. :quickref: AttachementHistory; List the content history of a DID.

        :resheader Content-Type: application/x-json-stream
        :param scope_name: data identifier (scope)/(name).
        :status 200: DID found
        :status 401: Invalid Auth Token
        :status 404: DID not found
        :status 406: Not Acceptable
        :returns: Stream of dictionarys with DIDs
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))

            def generate(vo):
                for did in list_content_history(scope=scope, name=name, vo=vo):
                    yield render_json(**did) + '\n'

            return try_stream(generate(vo=request.environ.get('vo')))
        except ValueError as error:
            return generate_http_error_flask(400, error)
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
示例#3
0
文件: archive.py 项目: stfc/rucio
    def get(self, scope_name):
        """
        List archive content keys.

        .. :quickref: Archive; list archive content keys.

        :param scope_name: data identifier (scope)/(name).
        :resheader Content-Type: application/x-json-stream
        :status 200: OK.
        :status 400: Invalid value.
        :status 406: Not Acceptable.
        :status 500: Internal Error.
        """
        try:
            scope, name = parse_scope_name(scope_name,
                                           request.environ.get('vo'))

            def generate(vo):
                for file in list_archive_content(scope=scope, name=name,
                                                 vo=vo):
                    yield dumps(file) + '\n'

            return try_stream(generate(vo=request.environ.get('vo')))
        except ValueError as error:
            return generate_http_error_flask(400, 'ValueError', error.args[0])
        except Exception as error:
            print(format_exc())
            return str(error), 500
示例#4
0
    def get(self, scope_name):
        """ List all parents of a data identifier.

        .. :quickref: Parents; List parents of DID.

        :resheader Content-Type: application/x-json-stream
        :param scope_name: data identifier (scope)/(name).
        :status 200: DID found
        :status 401: Invalid Auth Token
        :status 404: DID not found
        :status 406: Not Acceptable.
        :returns: A list of dictionary containing all dataset information.
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))

            def generate(vo):
                for dataset in list_parent_dids(scope=scope, name=name, vo=vo):
                    yield render_json(**dataset) + "\n"

            return try_stream(generate(vo=request.environ.get('vo')))
        except ValueError as error:
            return generate_http_error_flask(400, error)
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
示例#5
0
    def get(self, scope_name):
        """ List all replicas of a data identifier.

        .. :quickref: Files; List replicas of DID.

        :resheader Content-Type: application/x-json-stream
        :param scope_name: data identifier (scope)/(name).
        :query long: Flag to trigger long output
        :status 200: DID found
        :status 401: Invalid Auth Token
        :status 404: DID not found
        :status 406: Not Acceptable
        :returns: A dictionary containing all replicas information.
        """
        long = 'long' in request.args

        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))

            def generate(vo):
                for file in list_files(scope=scope, name=name, long=long, vo=vo):
                    yield dumps(file) + '\n'

            return try_stream(generate(vo=request.environ.get('vo')))
        except ValueError as error:
            return generate_http_error_flask(400, error)
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
示例#6
0
    def delete(self, scope_name):
        """
        Mark the input DID as not followed

        .. :quickref: Follow; Unfollow DID.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 InternalError

        :param scope_name: data identifier (scope)/(name).
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        parameters = json_parameters()
        account = param_get(parameters, 'account')

        try:
            remove_did_from_followed(scope=scope, name=name, account=account, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)

        return '', 200
示例#7
0
    def get(self, scope_name):
        """
        List all meta of a data identifier.

        .. :quickref: Meta; List DID metadata.

        :resheader Content-Type: application/json
        :param scope_name: data identifier (scope)/(name).
        :status 200: DID found
        :status 401: Invalid Auth Token
        :status 404: DID not found
        :status 406: Not Acceptable
        :returns: A dictionary containing all meta.
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        try:
            plugin = request.args.get('plugin', default='DID_COLUMN')
            meta = get_metadata(scope=scope, name=name, plugin=plugin, vo=request.environ.get('vo'))
            return Response(render_json(**meta), content_type='application/json')
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
示例#8
0
    def get(self, scope_name):
        """
        Return all associated rules of a file.

        .. :quickref: AssociatedRules; List associated rules of DID.

        :resheader Content-Type: application/x-json-stream
        :param scope_name: data identifier (scope)/(name).
        :status 200: DID found
        :status 401: Invalid Auth Token
        :status 404: DID not found
        :status 406: Not Acceptable
        :returns: List of associated rules.
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))

            def generate(vo):
                for rule in list_associated_replication_rules_for_file(scope=scope, name=name, vo=vo):
                    yield dumps(rule, cls=APIEncoder) + '\n'

            return try_stream(generate(vo=request.environ.get('vo')))
        except ValueError as error:
            return generate_http_error_flask(400, error)
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
示例#9
0
    def delete(self, scope_name):
        """
        Deletes the specified metadata from the DID

        .. :quickref: Meta; Delete DID metadata.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            404 KeyNotFound
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        if 'key' in request.args:
            key = request.args['key']
        else:
            return generate_http_error_flask(404, KeyNotFound.__name__, 'No key provided to remove')

        try:
            delete_metadata(scope=scope, name=name, key=key, vo=request.environ.get('vo'))
        except (KeyNotFound, DataIdentifierNotFound) as error:
            return generate_http_error_flask(404, error)
        except NotImplementedError as error:
            return generate_http_error_flask(409, error, 'Feature not in current database')

        return '', 200
示例#10
0
文件: dids.py 项目: vokac/rucio
    def get(self, scope_name):
        """
        Return all users following a specific DID.

        .. :quickref: Follow; List users following DID.

        :status 200: OK
        :status 400: ValueError
        :status 401: Unauthorized
        :status 404: DataIdentifierNotFound
        :status 406: Not Acceptable
        """
        try:
            scope, name = parse_scope_name(scope_name,
                                           request.environ.get('vo'))

            def generate(vo):
                for user in get_users_following_did(scope=scope,
                                                    name=name,
                                                    vo=vo):
                    yield render_json(**user) + '\n'

            return try_stream(generate(vo=request.environ.get('vo')),
                              content_type='application/json')
        except ValueError as error:
            return generate_http_error_flask(400, error)
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
示例#11
0
文件: replicas.py 项目: vokac/rucio
    def get(self, scope_name):
        """
        List dataset replicas.

        .. :quickref: DatasetReplicas; List dataset replicas.

        :param scope_name: data identifier (scope)/(name).
        :query deep: Flag to ennable lookup at the file level.
        :resheader Content-Type: application/x-json-stream
        :status 200: OK.
        :status 401: Invalid auth token.
        :status 406: Not Acceptable.
        :returns: A dictionary containing all replicas information.
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))

            def generate(_deep, vo):
                for row in list_dataset_replicas(scope=scope, name=name, deep=_deep, vo=vo):
                    yield dumps(row, cls=APIEncoder) + '\n'

            deep = request.args.get('deep', default=False)

            return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
        except ValueError as error:
            return generate_http_error_flask(400, error)
示例#12
0
    def get(self, scope_name, rse):
        """
        List request for given DID to a destination RSE.

        .. :quickref: RequestGet; list requests

        :param scope_name: data identifier (scope)/(name).
        :param rse: destination RSE.
        :reqheader Content-Type: application/json
        :status 200: Request found.
        :status 404: Request not found.
        :status 406: Not Acceptable.
        """
        try:
            scope, name = parse_scope_name(scope_name)
        except ValueError as error:
            return generate_http_error_flask(400, 'ValueError', error.args[0])
        except Exception as error:
            print(format_exc())
            return str(error), 500

        try:
            request_data = request.get_request_by_did(
                scope=scope,
                name=name,
                rse=rse,
                issuer=f_request.environ.get('issuer'),
                vo=f_request.environ.get('vo'))
            return Response(json.dumps(request_data, cls=APIEncoder),
                            content_type='application/json')
        except Exception:
            return generate_http_error_flask(
                404, 'RequestNotFound',
                'No request found for DID %s:%s at RSE %s' %
                (scope, name, rse))
示例#13
0
    def get(self, scope_name):
        """ get locks for a given scope, name.

        :param scope_name: data identifier (scope)/(name).
        :query did_type: The type used to filter, e.g., DATASET, CONTAINER.
        :resheader Content-Type: application/x-json-stream
        :status 200: OK.
        :status 401: Invalid Auth Token.
        :status 406: Not Acceptable.
        :status 500: Internal Error.
        :returns: Line separated list of dictionary with lock information.
        """
        did_type = request.args.get('did_type', None)
        try:
            scope, name = parse_scope_name(scope_name,
                                           request.environ.get('vo'))
            if did_type == 'dataset':

                def generate(vo):
                    for lock in get_dataset_locks(scope, name, vo=vo):
                        yield render_json(**lock) + '\n'

                return try_stream(generate(vo=request.environ.get('vo')))
            else:
                return 'Wrong did_type specified', 500
        except ValueError as error:
            return generate_http_error_flask(400, 'ValueError', error.args[0])
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__,
                                             error.args[0])
        except Exception as error:
            print(format_exc())
            return str(error), 500
示例#14
0
文件: rule.py 项目: ddavila0/rucio
    def get(self, scope_name):
        """ get history for a given DID.

        .. :quickref: RuleHistoryFull; get rule history for DID

        :resheader Content-Type: application/x-json-stream
        :param scope_name: data identifier (scope)/(name).
        :status 200: Rule found
        :status 406: Not Acceptable
        :status 500: Database Exception
        :returns: JSON dict containing informations about the requested user.
        """
        try:
            scope, name = parse_scope_name(scope_name)

            def generate(vo):
                for history in list_replication_rule_full_history(scope,
                                                                  name,
                                                                  vo=vo):
                    yield render_json(**history) + '\n'

            return try_stream(generate(vo=request.environ.get('vo')))
        except ValueError as error:
            return generate_http_error_flask(400, 'ValueError', error.args[0])
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__,
                                             error.args[0])
        except Exception as error:
            print(format_exc())
            return str(error), 500
示例#15
0
    def delete(self, scope_name):
        """
        Detach data identifiers from data identifiers.

        .. :quickref: DIDs; Detach DID from DID.

        :param scope_name: data identifier (scope)/(name).
        :<json dicts data: Must contain key 'dids' with list of dids to detach.
        :status 200: DIDs successfully detached
        :status 401: Invalid Auth Token
        :status 404: DID not found
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        parameters = json_parameters()
        dids = param_get(parameters, 'dids')

        try:
            detach_dids(scope=scope, name=name, dids=dids, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
        except UnsupportedOperation as error:
            return generate_http_error_flask(409, error)
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
        except AccessDenied as error:
            return generate_http_error_flask(401, error)

        return '', 200
示例#16
0
    def post(self, scope_name):
        """
        Mark the input DID as being followed by the given account.

        .. :quickref: Follow; Follow DID.

        HTTP Success:
            201 Created

        HTTP Error:
            401 Unauthorized
            404 Not Found
            500 Internal Error

        :param scope_name: data identifier (scope)/(name).
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        parameters = json_parameters()
        account = param_get(parameters, 'account')

        try:
            add_did_to_followed(scope=scope, name=name, account=account, vo=request.environ.get('vo'))
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
        except AccessDenied as error:
            return generate_http_error_flask(401, error)
示例#17
0
文件: replicas.py 项目: vokac/rucio
    def get(self, scope_name):
        """
        List dataset replicas using the Virtual Placement service.

        NOTICE: This is an RnD function and might change or go away at any time.

        .. :quickref: DatasetReplicas; List dataset replicas with VP.

        :param scope_name: data identifier (scope)/(name).
        :query deep: Flag to ennable lookup at the file level.
        :resheader Content-Type: application/x-json-stream
        :status 200: OK.
        :status 401: Invalid auth token.
        :status 406: Not Acceptable.
        :returns: If VP exists a list of dicts of sites, otherwise nothing
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))

            def generate(_deep, vo):
                for row in list_dataset_replicas_vp(scope=scope, name=name, deep=_deep, vo=vo):
                    yield dumps(row, cls=APIEncoder) + '\n'

            deep = request.args.get('deep', default=False)

            return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
        except ValueError as error:
            return generate_http_error_flask(400, error)
示例#18
0
文件: dids.py 项目: vokac/rucio
    def post(self, scope_name):
        """
        Append data identifiers to data identifiers.

        .. :quickref: Attachment; Append DID to DID.

        **Example request**:

        .. sourcecode:: http

            POST /dids/scope1/datasets1/dids HTTP/1.1
            Host: rucio.cern.ch

            [{"scope": "scope1", "name": "file1"},
             {"scope": "scope1", "name": "file2"},
             {"scope": "scope1", "name": "file3"}]

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 201 Created
            Vary: Accept

        :param scope_name: data identifier (scope)/(name).
        :<json list attachments: List of dicts of DIDs to attach.
        :status 201: DIDs successfully attached
        :status 401: Invalid Auth Token
        :status 404: DID not found
        :status 409: DIDs already attached
        """
        try:
            scope, name = parse_scope_name(scope_name,
                                           request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        attachments = json_parameters()

        try:
            attach_dids(scope=scope,
                        name=name,
                        attachment=attachments,
                        issuer=request.environ.get('issuer'),
                        vo=request.environ.get('vo'))
        except InvalidPath as error:
            return generate_http_error_flask(400, error)
        except (DataIdentifierNotFound, RSENotFound) as error:
            return generate_http_error_flask(404, error)
        except (DuplicateContent, UnsupportedOperation,
                FileAlreadyExists) as error:
            return generate_http_error_flask(409, error)
        except AccessDenied as error:
            return generate_http_error_flask(401, error)

        return 'Created', 201
示例#19
0
    def get(self, scope_name):
        """
        ---
        summary: List
        description: List archive contents.
        tags:
          - Archive
        parameters:
        - name: scope_name
          in: path
          description: The data identifier of the scope.
          schema:
            type: string
          style: simple
        responses:
          201:
            description: OK
            content:
              application/x-json-stream:
                schema:
                  type: array
                  items:
                    type: object
                    properties:
                      scope:
                        description: The scope of the did.
                        type: str
                      name:
                        description: The name of the did.
                        type: str
                      bytes:
                        description: The number of bytes.
                        type: int
                      adler32:
                        description: The adler32 checksum.
                        type: str
                      md5:
                        description: The md5 checksum.
                        type: str
          400:
            description: Invalid value
          406:
            description: Not acceptable
        """
        try:
            scope, name = parse_scope_name(scope_name,
                                           request.environ.get('vo'))

            def generate(vo):
                for file in list_archive_content(scope=scope, name=name,
                                                 vo=vo):
                    yield dumps(file) + '\n'

            return try_stream(generate(vo=request.environ.get('vo')))
        except ValueError as error:
            return generate_http_error_flask(400, error)
示例#20
0
文件: dids.py 项目: vokac/rucio
    def put(self, scope_name):
        """
        Update data identifier status.

        .. :quickref: DIDs; Update DID status.

        .. sourcecode:: http

            PUT /dids/scope1/container1 HTTP/1.1
            Host: rucio.cern.ch

            {"open": False},

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Vary: Accept

        :param scope_name: data identifier (scope)/(name).
        :<json bool open: open or close did
        :status 200: DIDs successfully updated
        :status 401: Invalid Auth Token
        :status 404: DID not found
        :status 409: Wrong status
        """
        try:
            scope, name = parse_scope_name(scope_name,
                                           request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        parameters = json_parameters()

        try:
            set_status(scope=scope,
                       name=name,
                       issuer=request.environ.get('issuer'),
                       vo=request.environ.get('vo'),
                       **parameters)
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
        except (UnsupportedStatus, UnsupportedOperation) as error:
            return generate_http_error_flask(409, error)
        except AccessDenied as error:
            return generate_http_error_flask(401, error)

        return '', 200
示例#21
0
文件: dids.py 项目: vokac/rucio
    def get(self, scope_name):
        """
        Retrieve a single data identifier.

        .. :quickref: DIDs; Retrieve a single DID.

        **Example request**:

        .. sourcecode:: http

            GET /dids/scope1/dataset1?dynamic HTTP/1.1
            Host: rucio.cern.ch

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Vary: Accept
            Content-Type: application/json

            {"scope": "scope1", "did_type": "DATASET", "name": "dataset1",
             "bytes": 234, "length": 3, "account": "jdoe", "open": True,
             "monotonic": False, "expired_at": null}

        :query dynamic: Flag to dynamically calculate size for open DIDs
        :resheader Content-Type: application/json
        :status 200: DID found
        :status 401: Invalid Auth Token
        :status 404: Scope not found
        :status 406: Not Acceptable.
        :returns: Dictionary with DID metadata
        """
        try:
            scope, name = parse_scope_name(scope_name,
                                           request.environ.get('vo'))
            dynamic = 'dynamic' in request.args
            did = get_did(scope=scope,
                          name=name,
                          dynamic=dynamic,
                          vo=request.environ.get('vo'))
            return Response(render_json(**did),
                            content_type='application/json')
        except ValueError as error:
            return generate_http_error_flask(400, error)
        except (ScopeNotFound, DataIdentifierNotFound) as error:
            return generate_http_error_flask(404, error)
示例#22
0
文件: dids.py 项目: vokac/rucio
    def post(self, scope_name, key):
        """
        Add metadata to a data identifier.

        .. :quickref: SingleMeta; Add DID metadata.

        HTTP Success:
            201 Created

        HTTP Error:
            400 Bad Request
            401 Unauthorized
            404 Not Found
            409 Conflict
            500 Internal Error

        :param scope_name: data identifier (scope)/(name).
        :param key: the key.

        """
        try:
            scope, name = parse_scope_name(scope_name,
                                           request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        parameters = json_parameters()
        value = param_get(parameters, 'value')

        try:
            set_metadata(
                scope=scope,
                name=name,
                key=key,
                value=value,
                issuer=request.environ.get('issuer'),
                recursive=param_get(parameters, 'recursive', default=False),
                vo=request.environ.get('vo'),
            )
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
        except Duplicate as error:
            return generate_http_error_flask(409, error)
        except (KeyNotFound, InvalidMetadata, InvalidValueForKey) as error:
            return generate_http_error_flask(400, error)

        return 'Created', 201
示例#23
0
文件: dids.py 项目: vokac/rucio
    def get(self, scope_name):
        """
        Returns the contents of a data identifier.

        .. :quickref: Attachement; Get DID contents.

        **Example request**:

        .. sourcecode:: http

            GET /dids/scope1/dataset1?dynamic HTTP/1.1
            Host: rucio.cern.ch

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Vary: Accept
            Content-Type: application/json

            {"scope": "scope1", "did_type": "DATASET", "name": "dataset1",
             "bytes": 234, "length": 3, "account": "jdoe", "open": True,
             "monotonic": False, "expired_at": null}

        :query dynamic: Flag to dynamically calculate size for open DIDs
        :resheader Content-Type: application/x-json-stream
        :status 200: DID found
        :status 401: Invalid Auth Token
        :status 404: Scope not found
        :status 406: Not Acceptable
        :returns: Dictionary with DID metadata
        """
        try:
            scope, name = parse_scope_name(scope_name,
                                           request.environ.get('vo'))

            def generate(vo):
                for did in list_content(scope=scope, name=name, vo=vo):
                    yield render_json(**did) + '\n'

            return try_stream(generate(vo=request.environ.get('vo')))
        except ValueError as error:
            return generate_http_error_flask(400, error)
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
示例#24
0
文件: dids.py 项目: vokac/rucio
    def post(self, scope_name):
        """
        Add metadata to a data identifier in bulk.

        .. :quickref: Meta; Add DID metadata.

        :param scope_name: data identifier (scope)/(name).
        :status 201: Metadata created.
        :status 400: Invalid input data.
        :status 404: DID not found.
        :status 409: Duplicate.
        :returns: Created
        """
        try:
            scope, name = parse_scope_name(scope_name,
                                           request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        parameters = json_parameters()
        meta = param_get(parameters, 'meta')

        try:
            set_metadata_bulk(
                scope=scope,
                name=name,
                meta=meta,
                issuer=request.environ.get('issuer'),
                recursive=param_get(parameters, 'recursive', default=False),
                vo=request.environ.get('vo'),
            )
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
        except Duplicate as error:
            return generate_http_error_flask(409, error)
        except (KeyNotFound, InvalidMetadata, InvalidValueForKey) as error:
            return generate_http_error_flask(400, error)

        return "Created", 201
示例#25
0
    def get(self, scope_name):
        """
        ---
        summary: Header redirect
        description: Get the header redirect.
        tags:
          - Redirect
        parameters:
        - name: scope_name
          in: path
          description: The data identifier (scope)/(name).
          schema:
            type: string
          style: simple
        - name: ip
          in: query
          description: The client ip.
          schema:
            type: string
          style: simple
          required: false
        - name: fqdn
          in: query
          schema:
            type: string
          style: simple
          required: false
        - name: site
          in: query
          schema:
            type: string
          style: simple
          required: false
        - name: schemes
          in: query
          schema:
            type: array
          style: simple
          required: false
        - name: select
          in: query
          schema:
            type: string
          style: simple
          required: false
        - name: sort
          in: query
          schema:
            type: string
          style: simple
          required: false
        - name: rse
          in: query
          schema:
            type: string
          style: simple
          required: false
        responses:
          303:
            description: OK
            content:
              application/json:
                schema:
                  description: The redirect url.
                  type: string
          401:
            description: Invalid Auth Token
          404:
            description: Rse or did not found
        """
        headers = self.get_headers()

        try:
            scope, name = parse_scope_name(scope_name,
                                           extract_vo(request.headers))
        except ValueError as error:
            return generate_http_error_flask(400, error, headers=headers)

        try:
            client_ip = request.headers.get('X-Forwarded-For',
                                            default=request.remote_addr)

            client_location = {
                'ip': request.args.get('ip', default=client_ip),
                'fqdn': request.args.get('fqdn', default=None),
                'site': request.args.get('site', default=None),
            }
            # use the default HTTP protocols if no scheme is given
            schemes = request.args.getlist('schemes') or [
                'davs', 'https', 's3'
            ]
            sortby = request.args.get('select', default='random')
            sortby = request.args.get('sort', default=sortby)
            rse = request.args.get('rse', default=None)
            site = request.args.get('site', default=None)

            # correctly forward the schemes and select to potential metalink followups
            cleaned_url = request.environ.get('REQUEST_URI').split('?')[0]

            headers.set(
                'Link',
                f'<{cleaned_url}/metalink?schemes={",".join(schemes)}&select={sortby}>; rel=describedby; type="application/metalink+xml"'
            )

            # get vo if given
            vo = extract_vo(request.headers)

            replicas = list(
                list_replicas(dids=[{
                    'scope': scope,
                    'name': name,
                    'type': 'FILE'
                }],
                              schemes=schemes,
                              client_location=client_location,
                              vo=vo))

            selected_url = None
            for r in replicas:
                if r['rses']:
                    dictreplica = {}

                    if rse:
                        if rse in r['rses'] and r['rses'][rse]:
                            selected_url = r['rses'][rse][0]
                        else:
                            return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
                    else:

                        for rep in r['rses']:
                            for replica in r['rses'][rep]:
                                # since this is HTTP-only redirection, and to ensure compatibility with as many http clients as possible
                                # forcibly replacement davs and s3 URLs to https
                                replica = replica.replace(
                                    'davs://',
                                    'https://').replace('s3://', 'https://')
                                dictreplica[replica] = rep

                        if not dictreplica:
                            return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers

                        elif site:
                            rep = site_selector(dictreplica, site, vo)
                            if rep:
                                selected_url = rep[0]
                            else:
                                return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
                        else:
                            rep = sort_replicas(dictreplica,
                                                client_location,
                                                selection=sortby)
                            selected_url = rep[0]

            if selected_url:
                response = redirect(selected_url, code=303)
                response.headers.extend(headers)
                return response

            return 'no redirection possible - file does not exist', 404, headers
        except ReplicaNotFound as error:
            return generate_http_error_flask(404, error, headers=headers)
示例#26
0
文件: replicas.py 项目: vokac/rucio
    def get(self, scope_name):
        """
        List all replicas for data identifiers.

        .. :quickref: Replicas; List replicas for DID.
        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 InternalError

        :reqheader HTTP_ACCEPT: application/metalink4+xml
        :param scope_name: data identifier (scope)/(name).
        :resheader Content-Type: application/x-json-stream
        :resheader Content-Type: application/metalink4+xml
        :status 200: OK.
        :status 401: Invalid auth token.
        :status 404: DID not found.
        :status 406: Not Acceptable.
        :returns: A dictionary containing all replicas information.
        :returns: A metalink description of replicas if metalink(4)+xml is specified in Accept:
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        content_type = request.accept_mimetypes.best_match(['application/x-json-stream', 'application/metalink4+xml'], 'application/x-json-stream')
        metalink = (content_type == 'application/metalink4+xml')
        dids = [{'scope': scope, 'name': name}]
        select, limit = None, None

        client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
        client_location = {'ip': client_ip, 'fqdn': None, 'site': None}

        schemes = request.args.get('schemes', default=None)
        select = request.args.get('select', default=None)
        limit = request.args.get('limit', default=None)
        if limit:
            limit = int(limit)

        # Resolve all reasonable protocols when doing metalink for maximum access possibilities
        if metalink and schemes is None:
            schemes = SUPPORTED_PROTOCOLS

        try:
            def generate(vo):
                # we need to call list_replicas before starting to reply
                # otherwise the exceptions won't be propagated correctly
                first = metalink

                # then, stream the replica information
                for rfile in list_replicas(dids=dids, schemes=schemes, vo=vo):
                    if first and metalink:
                        # first, set the appropriate content type, and stream the header
                        yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
                        first = False

                    replicas = []
                    dictreplica = {}
                    for rse in rfile['rses']:
                        for replica in rfile['rses'][rse]:
                            replicas.append(replica)
                            dictreplica[replica] = rse

                    replicas = sort_replicas(dictreplica, client_location, selection=select)

                    if not metalink:
                        yield dumps(rfile) + '\n'
                    else:
                        yield ' <file name="' + rfile['name'] + '">\n'
                        yield '  <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'

                        if rfile['adler32'] is not None:
                            yield '  <hash type="adler32">' + rfile['adler32'] + '</hash>\n'
                        if rfile['md5'] is not None:
                            yield '  <hash type="md5">' + rfile['md5'] + '</hash>\n'

                        yield '  <size>' + str(rfile['bytes']) + '</size>\n'

                        yield f'  <glfn name="/atlas/rucio/{rfile["scope"]}:{rfile["name"]}">'
                        yield '</glfn>\n'

                        for idx, replica in enumerate(replicas, start=1):
                            yield '   <url location="' + str(dictreplica[replica]) + '" priority="' + str(idx) + '">' + escape(replica) + '</url>\n'
                            if limit and limit == idx:
                                break
                        yield ' </file>\n'

                if metalink:
                    if first:
                        # if still first output, i.e. there were no replicas
                        yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n</metalink>\n'
                    else:
                        # don't forget to send the metalink footer
                        yield '</metalink>\n'

            return try_stream(generate(vo=request.environ.get('vo')), content_type=content_type)
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)
示例#27
0
    def get(self, scope_name):
        """
        Header Redirect

        .. :quickref: HeaderRedirector; Header redirect.

        :param scope_name: data identifier (scope)/(name).
        :resheader Content-Type: application/metalink+xml'.
        :status 303: Redirect.
        :status 401: Invalid Auth Token.
        :status 404: RSE Not Found.
        :status 404: DID Not Found.
        :status 500: Internal Error.
        """
        headers = Headers()
        headers.set('Access-Control-Allow-Origin',
                    request.environ.get('HTTP_ORIGIN'))
        headers.set('Access-Control-Allow-Headers',
                    request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
        headers.set('Access-Control-Allow-Methods', '*')
        headers.set('Access-Control-Allow-Credentials', 'true')

        try:
            scope, name = parse_scope_name(
                scope_name, request.headers.get('X-Rucio-VO', default='def'))
        except ValueError as error:
            return generate_http_error_flask(400,
                                             'ValueError',
                                             error.args[0],
                                             headers=headers)
        except Exception as error:
            logging.exception("Internal Error")
            return str(error), 500, headers

        try:

            # use the default HTTP protocols if no scheme is given
            select, rse, site, schemes = 'random', None, None, [
                'davs', 'http', 'https'
            ]

            client_ip = request.headers.get('X-Forwarded-For',
                                            default=request.remote_addr)

            client_location = {'ip': client_ip, 'fqdn': None, 'site': None}

            if request.query_string:
                query_string = request.query_string.decode(encoding='utf-8')
                params = parse_qs(query_string)
                if 'select' in params:
                    select = params['select'][0]
                if 'sort' in params:
                    select = params['sort'][0]
                if 'rse' in params:
                    rse = params['rse'][0]
                if 'site' in params:
                    site = params['site'][0]
                if 'schemes' in params:
                    schemes = params['schemes'][0]
                else:
                    schemes = ['davs', 'https', 's3']

                if 'ip' in params:
                    client_location['ip'] = params['ip'][0]
                if 'fqdn' in params:
                    client_location['fqdn'] = params['fqdn'][0]
                if 'site' in params:
                    client_location['site'] = params['site'][0]

            # correctly forward the schemes and select to potential metalink followups
            cleaned_url = request.environ.get('REQUEST_URI').split('?')[0]
            if isinstance(schemes, list):
                headers.set(
                    'Link',
                    '<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"'
                    % (cleaned_url, ','.join(schemes), select))
            else:
                headers.set(
                    'Link',
                    '<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"'
                    % (cleaned_url, schemes, select))
                schemes = [schemes]  # list_replicas needs a list

            # get vo if given
            vo = request.headers.get('X-Rucio-VO', default='def')

            replicas = [
                r for r in list_replicas(dids=[{
                    'scope': scope,
                    'name': name,
                    'type': 'FILE'
                }],
                                         schemes=schemes,
                                         client_location=client_location,
                                         vo=vo)
            ]

            selected_url = None
            for r in replicas:
                if r['rses']:
                    dictreplica = {}

                    if rse:
                        if rse in r['rses'] and r['rses'][rse]:
                            selected_url = r['rses'][rse][0]
                        else:
                            return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
                    else:

                        for rep in r['rses']:
                            for replica in r['rses'][rep]:
                                # since this is HTTP-only redirection, and to ensure compatibility with as many http clients as possible
                                # forcibly replacement davs and s3 URLs to https
                                replica = replica.replace(
                                    'davs://',
                                    'https://').replace('s3://', 'https://')
                                dictreplica[replica] = rep

                        if not dictreplica:
                            return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers

                        elif site:
                            rep = site_selector(dictreplica, site, vo)
                            if rep:
                                selected_url = rep[0]
                            else:
                                return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
                        else:
                            rep = sort_replicas(dictreplica,
                                                client_location,
                                                selection=select)
                            selected_url = rep[0]

            if selected_url:
                response = redirect(selected_url, code=303)
                response.headers.extend(headers)
                return response

            return 'no redirection possible - file does not exist', 404, headers
        except ReplicaNotFound as error:
            return generate_http_error_flask(404,
                                             'ReplicaNotFound',
                                             error.args[0],
                                             headers=headers)
        except RucioException as error:
            return generate_http_error_flask(500,
                                             error.__class__.__name__,
                                             error.args[0],
                                             headers=headers)
        except Exception as error:
            logging.exception("Internal Error")
            return str(error), 500, headers
示例#28
0
    def get(self, scope_name):
        """
        Metalink redirect

        .. :quickref: MetaLinkRedirector; Metalink redirect.

        :param scope_name: data identifier (scope)/(name).
        :resheader Content-Type: application/metalink4+xml'.
        :status 200: OK.
        :status 401: Invalid Auth Token.
        :status 404: RSE Not Found.
        :status 404: DID Not Found.
        :status 406: Not Acceptable.
        :status 500: Internal Error.
        :returns: Metalink file
        """
        headers = Headers()
        headers.set('Access-Control-Allow-Origin',
                    request.environ.get('HTTP_ORIGIN'))
        headers.set('Access-Control-Allow-Headers',
                    request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
        headers.set('Access-Control-Allow-Methods', '*')
        headers.set('Access-Control-Allow-Credentials', 'true')

        try:
            scope, name = parse_scope_name(
                scope_name, request.headers.get('X-Rucio-VO', default='def'))
        except ValueError as error:
            return generate_http_error_flask(400,
                                             'ValueError',
                                             error.args[0],
                                             headers=headers)
        except Exception as error:
            logging.exception("Internal Error")
            return str(error), 500, headers

        dids, schemes, sortby = [{
            'scope': scope,
            'name': name
        }], ['http', 'https', 'root', 'gsiftp', 'srm', 'davs'], None

        # set the correct client IP
        client_ip = request.headers.get('X-Forwarded-For',
                                        default=request.remote_addr)

        client_location = {'ip': client_ip, 'fqdn': None, 'site': None}

        if request.query_string:
            query_string = request.query_string.decode(encoding='utf-8')
            params = parse_qs(query_string)
            if 'schemes' in params:
                schemes = params['schemes']
            if 'select' in params:
                sortby = params['select'][0]
            if 'sort' in params:
                sortby = params['sort'][0]

            if 'ip' in params:
                client_location['ip'] = params['ip'][0]
            if 'fqdn' in params:
                client_location['fqdn'] = params['fqdn'][0]
            if 'site' in params:
                client_location['site'] = params['site'][0]

        # get vo if given
        vo = request.headers.get('X-Rucio-VO', default='def')

        try:
            replicas_iter = list_replicas(dids=dids,
                                          schemes=schemes,
                                          client_location=client_location,
                                          vo=vo)
            try:
                first = next(replicas_iter)
            except StopIteration:
                return 'no redirection possible - cannot find the DID', 404

            def generate():
                # first, set the appropriate content type, and stream the header
                yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'

                # iteratively stream the XML per file
                for rfile in itertools.chain((first, ), replicas_iter):
                    replicas = []
                    dictreplica = {}
                    for rse in rfile['rses']:
                        for replica in rfile['rses'][rse]:
                            replicas.append(replica)
                            dictreplica[replica] = rse

                    # stream metadata
                    yield ' <file name="' + rfile['name'] + '">\n'
                    yield '  <identity>' + rfile['scope'] + ':' + rfile[
                        'name'] + '</identity>\n'

                    if rfile['adler32'] is not None:
                        yield '  <hash type="adler32">' + rfile[
                            'adler32'] + '</hash>\n'
                    if rfile['md5'] is not None:
                        yield '  <hash type="md5">' + rfile['md5'] + '</hash>\n'

                    yield '  <size>' + str(rfile['bytes']) + '</size>\n'

                    yield '  <glfn name="/atlas/rucio/%s:%s">' % (
                        rfile['scope'], rfile['name'])
                    yield '</glfn>\n'

                    replicas = sort_replicas(dictreplica,
                                             client_location,
                                             selection=sortby)

                    # stream URLs
                    idx = 1
                    for replica in replicas:
                        yield '  <url location="' + str(
                            dictreplica[replica]) + '" priority="' + str(
                                idx) + '">' + replica + '</url>\n'
                        idx += 1

                    yield ' </file>\n'

                # don't forget to send the metalink footer
                yield '</metalink>\n'

            return try_stream(generate(),
                              content_type='application/metalink4+xml')
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404,
                                             'DataIdentifierNotFound',
                                             error.args[0],
                                             headers=headers)
        except ReplicaNotFound as error:
            return generate_http_error_flask(404,
                                             'ReplicaNotFound',
                                             error.args[0],
                                             headers=headers)
        except RucioException as error:
            return generate_http_error_flask(500,
                                             error.__class__.__name__,
                                             error.args[0],
                                             headers=headers)
        except Exception as error:
            logging.exception("Internal Error")
            return str(error), 500, headers
示例#29
0
    def get(self, scope_name):
        """
        ---
        summary: Metalink redirect
        description: Get Metalink redirect.
        tags:
          - Redirect
        parameters:
        - name: scope_name
          in: path
          description: The data identifier (scope)/(name).
          schema:
            type: string
          style: simple
        - name: ip
          in: query
          description: The client ip.
          schema:
            type: string
          style: simple
          required: false
        - name: fqdn
          in: query
          schema:
            type: string
          style: simple
          required: false
        - name: site
          in: query
          schema:
            type: string
          style: simple
          required: false
        - name: schemes
          in: query
          schema:
            type: array
          style: simple
          required: false
        - name: select
          in: query
          schema:
            type: string
          style: simple
          required: false
        - name: sort
          in: query
          schema:
            type: string
          style: simple
          required: false
        responses:
          200:
            description: OK
            content:
              application/metalink4+xml:
                schema:
                  description: The metalink file.
                  type: string
          401:
            description: Invalid Auth Token
          404:
            description: Rse or did not found
          406:
            description: Not acceptable
        """
        headers = self.get_headers()

        try:
            scope, name = parse_scope_name(scope_name,
                                           extract_vo(request.headers))
        except ValueError as error:
            return generate_http_error_flask(400, error, headers=headers)

        # set the correct client IP
        client_ip = request.headers.get('X-Forwarded-For',
                                        default=request.remote_addr)

        client_location = {
            'ip': request.args.get('ip', default=client_ip),
            'fqdn': request.args.get('fqdn', default=None),
            'site': request.args.get('site', default=None),
        }

        dids = [{'scope': scope, 'name': name}]
        schemes = request.args.getlist('schemes') or [
            'http', 'https', 'root', 'gsiftp', 'srm', 'davs'
        ]
        sortby = request.args.get('select', default=None)
        sortby = request.args.get('sort', default=sortby)

        # get vo if given
        vo = extract_vo(request.headers)

        try:
            replicas_iter = list_replicas(dids=dids,
                                          schemes=schemes,
                                          client_location=client_location,
                                          vo=vo)
            try:
                first = next(replicas_iter)
            except StopIteration:
                return 'no redirection possible - cannot find the DID', 404

            def generate():
                # first, set the appropriate content type, and stream the header
                yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'

                # iteratively stream the XML per file
                for rfile in itertools.chain((first, ), replicas_iter):
                    replicas = []
                    dictreplica = {}
                    for rse in rfile['rses']:
                        for replica in rfile['rses'][rse]:
                            replicas.append(replica)
                            dictreplica[replica] = rse

                    # stream metadata
                    yield ' <file name="' + rfile['name'] + '">\n'
                    yield '  <identity>' + rfile['scope'] + ':' + rfile[
                        'name'] + '</identity>\n'

                    if rfile['adler32'] is not None:
                        yield '  <hash type="adler32">' + rfile[
                            'adler32'] + '</hash>\n'
                    if rfile['md5'] is not None:
                        yield '  <hash type="md5">' + rfile['md5'] + '</hash>\n'

                    yield '  <size>' + str(rfile['bytes']) + '</size>\n'

                    yield f'  <glfn name="/atlas/rucio/{rfile["scope"]}:{rfile["name"]}">'
                    yield '</glfn>\n'

                    replicas = sort_replicas(dictreplica,
                                             client_location,
                                             selection=sortby)

                    # stream URLs
                    idx = 1
                    for replica in replicas:
                        yield '  <url location="' + str(
                            dictreplica[replica]) + '" priority="' + str(
                                idx) + '">' + replica + '</url>\n'
                        idx += 1

                    yield ' </file>\n'

                # don't forget to send the metalink footer
                yield '</metalink>\n'

            return try_stream(generate(),
                              content_type='application/metalink4+xml')
        except (DataIdentifierNotFound, ReplicaNotFound) as error:
            return generate_http_error_flask(404, error, headers=headers)
示例#30
0
    def post(self, scope_name):
        """
        Create a new data identifier.

        .. :quickref: DIDs; Create a new DID.

        .. sourcecode:: http

            POST /dids/scope1/container1 HTTP/1.1
            Host: rucio.cern.ch

            {"type": "CONTAINER", "lifetime": 86400},

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 201 Created
            Vary: Accept

        :reqheader Accept: application/json
        :param scope_name: data identifier (scope)/(name).
        :<json string type: the new DID type
        :<json dict statuses: Dictionary with statuses, e.g. {'monotonic':True}
        :<json dict meta: Dictionary with metadata, e.g. {'length':1234}
        :<json dict rules: Replication rules associated with the did. e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ]
        :<json int lifetime: DID's liftime in seconds.
        :<json list dids: The content.
        :<json string rse: The RSE name when registering replicas.
        :status 201: new DIDs created
        :status 401: Invalid Auth Token
        :status 409: DID already exists
        """
        try:
            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
        except ValueError as error:
            return generate_http_error_flask(400, error)

        parameters = json_parameters()
        type_param = param_get(parameters, 'type')

        try:
            add_did(
                scope=scope,
                name=name,
                type=type_param,
                statuses=param_get(parameters, 'statuses', default={}),
                meta=param_get(parameters, 'meta', default={}),
                rules=param_get(parameters, 'rules', default=[]),
                lifetime=param_get(parameters, 'lifetime', default=None),
                dids=param_get(parameters, 'dids', default=[]),
                rse=param_get(parameters, 'rse', default=None),
                issuer=request.environ.get('issuer'),
                vo=request.environ.get('vo'),
            )
        except (InvalidObject, InvalidPath) as error:
            return generate_http_error_flask(400, error)
        except (DataIdentifierNotFound, ScopeNotFound) as error:
            return generate_http_error_flask(404, error)
        except (DuplicateContent, DataIdentifierAlreadyExists, UnsupportedOperation) as error:
            return generate_http_error_flask(409, error)
        except AccessDenied as error:
            return generate_http_error_flask(401, error)

        return 'Created', 201