def get(self, scope_name, rse):
"""
List request for given DID to a destination RSE.
.. :quickref: RequestGet; list requests
:param scope_name: data identifier (scope)/(name).
:param rse: destination RSE.
:reqheader Content-Type: application/json
:status 200: Request found.
:status 404: Request not found.
:status 406: Not Acceptable.
"""
try:
scope, name = parse_scope_name(scope_name,
flask.request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
try:
request_data = request.get_request_by_did(
scope=scope,
name=name,
rse=rse,
issuer=flask.request.environ.get('issuer'),
vo=flask.request.environ.get('vo'),
)
return Response(json.dumps(request_data, cls=APIEncoder),
content_type='application/json')
except RequestNotFound as error:
return generate_http_error_flask(
404, error.__class__.__name__,
f'No request found for DID {scope}:{name} at RSE {rse}')
def get(self, scope_name):
"""
Returns the contents history of a data identifier.
.. :quickref: AttachementHistory; List the content history of a DID.
:resheader Content-Type: application/x-json-stream
:param scope_name: data identifier (scope)/(name).
:status 200: DID found
:status 401: Invalid Auth Token
:status 404: DID not found
:status 406: Not Acceptable
:returns: Stream of dictionarys with DIDs
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
def generate(vo):
for did in list_content_history(scope=scope, name=name, vo=vo):
yield render_json(**did) + '\n'
return try_stream(generate(vo=request.environ.get('vo')))
except ValueError as error:
return generate_http_error_flask(400, error)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
def get(self, scope_name):
"""
List archive content keys.
.. :quickref: Archive; list archive content keys.
:param scope_name: data identifier (scope)/(name).
:resheader Content-Type: application/x-json-stream
:status 200: OK.
:status 400: Invalid value.
:status 406: Not Acceptable.
:status 500: Internal Error.
"""
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
def generate(vo):
for file in list_archive_content(scope=scope, name=name,
vo=vo):
yield dumps(file) + '\n'
return try_stream(generate(vo=request.environ.get('vo')))
except ValueError as error:
return generate_http_error_flask(400, 'ValueError', error.args[0])
except Exception as error:
print(format_exc())
return str(error), 500
def get(self, scope_name):
""" List all parents of a data identifier.
.. :quickref: Parents; List parents of DID.
:resheader Content-Type: application/x-json-stream
:param scope_name: data identifier (scope)/(name).
:status 200: DID found
:status 401: Invalid Auth Token
:status 404: DID not found
:status 406: Not Acceptable.
:returns: A list of dictionary containing all dataset information.
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
def generate(vo):
for dataset in list_parent_dids(scope=scope, name=name, vo=vo):
yield render_json(**dataset) + "\n"
return try_stream(generate(vo=request.environ.get('vo')))
except ValueError as error:
return generate_http_error_flask(400, error)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
def get(self, scope_name):
""" List all replicas of a data identifier.
.. :quickref: Files; List replicas of DID.
:resheader Content-Type: application/x-json-stream
:param scope_name: data identifier (scope)/(name).
:query long: Flag to trigger long output
:status 200: DID found
:status 401: Invalid Auth Token
:status 404: DID not found
:status 406: Not Acceptable
:returns: A dictionary containing all replicas information.
"""
long = 'long' in request.args
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
def generate(vo):
for file in list_files(scope=scope, name=name, long=long, vo=vo):
yield dumps(file) + '\n'
return try_stream(generate(vo=request.environ.get('vo')))
except ValueError as error:
return generate_http_error_flask(400, error)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
def delete(self, scope_name):
"""
Mark the input DID as not followed
.. :quickref: Follow; Unfollow DID.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
:param scope_name: data identifier (scope)/(name).
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
parameters = json_parameters()
account = param_get(parameters, 'account')
try:
remove_did_from_followed(scope=scope, name=name, account=account, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
return '', 200
def get(self, scope_name):
"""
List all meta of a data identifier.
.. :quickref: Meta; List DID metadata.
:resheader Content-Type: application/json
:param scope_name: data identifier (scope)/(name).
:status 200: DID found
:status 401: Invalid Auth Token
:status 404: DID not found
:status 406: Not Acceptable
:returns: A dictionary containing all meta.
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
try:
plugin = request.args.get('plugin', default='DID_COLUMN')
meta = get_metadata(scope=scope, name=name, plugin=plugin, vo=request.environ.get('vo'))
return Response(render_json(**meta), content_type='application/json')
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
def get(self, scope_name):
"""
Return all associated rules of a file.
.. :quickref: AssociatedRules; List associated rules of DID.
:resheader Content-Type: application/x-json-stream
:param scope_name: data identifier (scope)/(name).
:status 200: DID found
:status 401: Invalid Auth Token
:status 404: DID not found
:status 406: Not Acceptable
:returns: List of associated rules.
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
def generate(vo):
for rule in list_associated_replication_rules_for_file(scope=scope, name=name, vo=vo):
yield dumps(rule, cls=APIEncoder) + '\n'
return try_stream(generate(vo=request.environ.get('vo')))
except ValueError as error:
return generate_http_error_flask(400, error)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
def delete(self, scope_name):
"""
Deletes the specified metadata from the DID
.. :quickref: Meta; Delete DID metadata.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 KeyNotFound
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
if 'key' in request.args:
key = request.args['key']
else:
return generate_http_error_flask(404, KeyNotFound.__name__, 'No key provided to remove')
try:
delete_metadata(scope=scope, name=name, key=key, vo=request.environ.get('vo'))
except (KeyNotFound, DataIdentifierNotFound) as error:
return generate_http_error_flask(404, error)
except NotImplementedError as error:
return generate_http_error_flask(409, error, 'Feature not in current database')
return '', 200
def get(self, scope_name):
"""
Return all users following a specific DID.
.. :quickref: Follow; List users following DID.
:status 200: OK
:status 400: ValueError
:status 401: Unauthorized
:status 404: DataIdentifierNotFound
:status 406: Not Acceptable
"""
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
def generate(vo):
for user in get_users_following_did(scope=scope,
name=name,
vo=vo):
yield render_json(**user) + '\n'
return try_stream(generate(vo=request.environ.get('vo')),
content_type='application/json')
except ValueError as error:
return generate_http_error_flask(400, error)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
def get(self, scope_name):
"""
List dataset replicas.
.. :quickref: DatasetReplicas; List dataset replicas.
:param scope_name: data identifier (scope)/(name).
:query deep: Flag to ennable lookup at the file level.
:resheader Content-Type: application/x-json-stream
:status 200: OK.
:status 401: Invalid auth token.
:status 406: Not Acceptable.
:returns: A dictionary containing all replicas information.
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
def generate(_deep, vo):
for row in list_dataset_replicas(scope=scope, name=name, deep=_deep, vo=vo):
yield dumps(row, cls=APIEncoder) + '\n'
deep = request.args.get('deep', default=False)
return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
except ValueError as error:
return generate_http_error_flask(400, error)
def get(self, scope_name, rse):
"""
List request for given DID to a destination RSE.
.. :quickref: RequestGet; list requests
:param scope_name: data identifier (scope)/(name).
:param rse: destination RSE.
:reqheader Content-Type: application/json
:status 200: Request found.
:status 404: Request not found.
:status 406: Not Acceptable.
"""
try:
scope, name = parse_scope_name(scope_name)
except ValueError as error:
return generate_http_error_flask(400, 'ValueError', error.args[0])
except Exception as error:
print(format_exc())
return str(error), 500
try:
request_data = request.get_request_by_did(
scope=scope,
name=name,
rse=rse,
issuer=f_request.environ.get('issuer'),
vo=f_request.environ.get('vo'))
return Response(json.dumps(request_data, cls=APIEncoder),
content_type='application/json')
except Exception:
return generate_http_error_flask(
404, 'RequestNotFound',
'No request found for DID %s:%s at RSE %s' %
(scope, name, rse))
def get(self, scope_name):
""" get locks for a given scope, name.
:param scope_name: data identifier (scope)/(name).
:query did_type: The type used to filter, e.g., DATASET, CONTAINER.
:resheader Content-Type: application/x-json-stream
:status 200: OK.
:status 401: Invalid Auth Token.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: Line separated list of dictionary with lock information.
"""
did_type = request.args.get('did_type', None)
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
if did_type == 'dataset':
def generate(vo):
for lock in get_dataset_locks(scope, name, vo=vo):
yield render_json(**lock) + '\n'
return try_stream(generate(vo=request.environ.get('vo')))
else:
return 'Wrong did_type specified', 500
except ValueError as error:
return generate_http_error_flask(400, 'ValueError', error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return str(error), 500
def get(self, scope_name):
""" get history for a given DID.
.. :quickref: RuleHistoryFull; get rule history for DID
:resheader Content-Type: application/x-json-stream
:param scope_name: data identifier (scope)/(name).
:status 200: Rule found
:status 406: Not Acceptable
:status 500: Database Exception
:returns: JSON dict containing informations about the requested user.
"""
try:
scope, name = parse_scope_name(scope_name)
def generate(vo):
for history in list_replication_rule_full_history(scope,
name,
vo=vo):
yield render_json(**history) + '\n'
return try_stream(generate(vo=request.environ.get('vo')))
except ValueError as error:
return generate_http_error_flask(400, 'ValueError', error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return str(error), 500
def delete(self, scope_name):
"""
Detach data identifiers from data identifiers.
.. :quickref: DIDs; Detach DID from DID.
:param scope_name: data identifier (scope)/(name).
:<json dicts data: Must contain key 'dids' with list of dids to detach.
:status 200: DIDs successfully detached
:status 401: Invalid Auth Token
:status 404: DID not found
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
parameters = json_parameters()
dids = param_get(parameters, 'dids')
try:
detach_dids(scope=scope, name=name, dids=dids, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
except UnsupportedOperation as error:
return generate_http_error_flask(409, error)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
except AccessDenied as error:
return generate_http_error_flask(401, error)
return '', 200
def post(self, scope_name):
"""
Mark the input DID as being followed by the given account.
.. :quickref: Follow; Follow DID.
HTTP Success:
201 Created
HTTP Error:
401 Unauthorized
404 Not Found
500 Internal Error
:param scope_name: data identifier (scope)/(name).
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
parameters = json_parameters()
account = param_get(parameters, 'account')
try:
add_did_to_followed(scope=scope, name=name, account=account, vo=request.environ.get('vo'))
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
except AccessDenied as error:
return generate_http_error_flask(401, error)
def get(self, scope_name):
"""
List dataset replicas using the Virtual Placement service.
NOTICE: This is an RnD function and might change or go away at any time.
.. :quickref: DatasetReplicas; List dataset replicas with VP.
:param scope_name: data identifier (scope)/(name).
:query deep: Flag to ennable lookup at the file level.
:resheader Content-Type: application/x-json-stream
:status 200: OK.
:status 401: Invalid auth token.
:status 406: Not Acceptable.
:returns: If VP exists a list of dicts of sites, otherwise nothing
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
def generate(_deep, vo):
for row in list_dataset_replicas_vp(scope=scope, name=name, deep=_deep, vo=vo):
yield dumps(row, cls=APIEncoder) + '\n'
deep = request.args.get('deep', default=False)
return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
except ValueError as error:
return generate_http_error_flask(400, error)
def post(self, scope_name):
"""
Append data identifiers to data identifiers.
.. :quickref: Attachment; Append DID to DID.
**Example request**:
.. sourcecode:: http
POST /dids/scope1/datasets1/dids HTTP/1.1
Host: rucio.cern.ch
[{"scope": "scope1", "name": "file1"},
{"scope": "scope1", "name": "file2"},
{"scope": "scope1", "name": "file3"}]
**Example response**:
.. sourcecode:: http
HTTP/1.1 201 Created
Vary: Accept
:param scope_name: data identifier (scope)/(name).
:<json list attachments: List of dicts of DIDs to attach.
:status 201: DIDs successfully attached
:status 401: Invalid Auth Token
:status 404: DID not found
:status 409: DIDs already attached
"""
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
attachments = json_parameters()
try:
attach_dids(scope=scope,
name=name,
attachment=attachments,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'))
except InvalidPath as error:
return generate_http_error_flask(400, error)
except (DataIdentifierNotFound, RSENotFound) as error:
return generate_http_error_flask(404, error)
except (DuplicateContent, UnsupportedOperation,
FileAlreadyExists) as error:
return generate_http_error_flask(409, error)
except AccessDenied as error:
return generate_http_error_flask(401, error)
return 'Created', 201
def get(self, scope_name):
"""
---
summary: List
description: List archive contents.
tags:
- Archive
parameters:
- name: scope_name
in: path
description: The data identifier of the scope.
schema:
type: string
style: simple
responses:
201:
description: OK
content:
application/x-json-stream:
schema:
type: array
items:
type: object
properties:
scope:
description: The scope of the did.
type: str
name:
description: The name of the did.
type: str
bytes:
description: The number of bytes.
type: int
adler32:
description: The adler32 checksum.
type: str
md5:
description: The md5 checksum.
type: str
400:
description: Invalid value
406:
description: Not acceptable
"""
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
def generate(vo):
for file in list_archive_content(scope=scope, name=name,
vo=vo):
yield dumps(file) + '\n'
return try_stream(generate(vo=request.environ.get('vo')))
except ValueError as error:
return generate_http_error_flask(400, error)
def put(self, scope_name):
"""
Update data identifier status.
.. :quickref: DIDs; Update DID status.
.. sourcecode:: http
PUT /dids/scope1/container1 HTTP/1.1
Host: rucio.cern.ch
{"open": False},
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
:param scope_name: data identifier (scope)/(name).
:<json bool open: open or close did
:status 200: DIDs successfully updated
:status 401: Invalid Auth Token
:status 404: DID not found
:status 409: Wrong status
"""
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
parameters = json_parameters()
try:
set_status(scope=scope,
name=name,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'),
**parameters)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
except (UnsupportedStatus, UnsupportedOperation) as error:
return generate_http_error_flask(409, error)
except AccessDenied as error:
return generate_http_error_flask(401, error)
return '', 200
def get(self, scope_name):
"""
Retrieve a single data identifier.
.. :quickref: DIDs; Retrieve a single DID.
**Example request**:
.. sourcecode:: http
GET /dids/scope1/dataset1?dynamic HTTP/1.1
Host: rucio.cern.ch
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
{"scope": "scope1", "did_type": "DATASET", "name": "dataset1",
"bytes": 234, "length": 3, "account": "jdoe", "open": True,
"monotonic": False, "expired_at": null}
:query dynamic: Flag to dynamically calculate size for open DIDs
:resheader Content-Type: application/json
:status 200: DID found
:status 401: Invalid Auth Token
:status 404: Scope not found
:status 406: Not Acceptable.
:returns: Dictionary with DID metadata
"""
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
dynamic = 'dynamic' in request.args
did = get_did(scope=scope,
name=name,
dynamic=dynamic,
vo=request.environ.get('vo'))
return Response(render_json(**did),
content_type='application/json')
except ValueError as error:
return generate_http_error_flask(400, error)
except (ScopeNotFound, DataIdentifierNotFound) as error:
return generate_http_error_flask(404, error)
def post(self, scope_name, key):
"""
Add metadata to a data identifier.
.. :quickref: SingleMeta; Add DID metadata.
HTTP Success:
201 Created
HTTP Error:
400 Bad Request
401 Unauthorized
404 Not Found
409 Conflict
500 Internal Error
:param scope_name: data identifier (scope)/(name).
:param key: the key.
"""
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
parameters = json_parameters()
value = param_get(parameters, 'value')
try:
set_metadata(
scope=scope,
name=name,
key=key,
value=value,
issuer=request.environ.get('issuer'),
recursive=param_get(parameters, 'recursive', default=False),
vo=request.environ.get('vo'),
)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
except Duplicate as error:
return generate_http_error_flask(409, error)
except (KeyNotFound, InvalidMetadata, InvalidValueForKey) as error:
return generate_http_error_flask(400, error)
return 'Created', 201
def get(self, scope_name):
"""
Returns the contents of a data identifier.
.. :quickref: Attachement; Get DID contents.
**Example request**:
.. sourcecode:: http
GET /dids/scope1/dataset1?dynamic HTTP/1.1
Host: rucio.cern.ch
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
{"scope": "scope1", "did_type": "DATASET", "name": "dataset1",
"bytes": 234, "length": 3, "account": "jdoe", "open": True,
"monotonic": False, "expired_at": null}
:query dynamic: Flag to dynamically calculate size for open DIDs
:resheader Content-Type: application/x-json-stream
:status 200: DID found
:status 401: Invalid Auth Token
:status 404: Scope not found
:status 406: Not Acceptable
:returns: Dictionary with DID metadata
"""
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
def generate(vo):
for did in list_content(scope=scope, name=name, vo=vo):
yield render_json(**did) + '\n'
return try_stream(generate(vo=request.environ.get('vo')))
except ValueError as error:
return generate_http_error_flask(400, error)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
def post(self, scope_name):
"""
Add metadata to a data identifier in bulk.
.. :quickref: Meta; Add DID metadata.
:param scope_name: data identifier (scope)/(name).
:status 201: Metadata created.
:status 400: Invalid input data.
:status 404: DID not found.
:status 409: Duplicate.
:returns: Created
"""
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
parameters = json_parameters()
meta = param_get(parameters, 'meta')
try:
set_metadata_bulk(
scope=scope,
name=name,
meta=meta,
issuer=request.environ.get('issuer'),
recursive=param_get(parameters, 'recursive', default=False),
vo=request.environ.get('vo'),
)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
except Duplicate as error:
return generate_http_error_flask(409, error)
except (KeyNotFound, InvalidMetadata, InvalidValueForKey) as error:
return generate_http_error_flask(400, error)
return "Created", 201
def get(self, scope_name):
"""
---
summary: Header redirect
description: Get the header redirect.
tags:
- Redirect
parameters:
- name: scope_name
in: path
description: The data identifier (scope)/(name).
schema:
type: string
style: simple
- name: ip
in: query
description: The client ip.
schema:
type: string
style: simple
required: false
- name: fqdn
in: query
schema:
type: string
style: simple
required: false
- name: site
in: query
schema:
type: string
style: simple
required: false
- name: schemes
in: query
schema:
type: array
style: simple
required: false
- name: select
in: query
schema:
type: string
style: simple
required: false
- name: sort
in: query
schema:
type: string
style: simple
required: false
- name: rse
in: query
schema:
type: string
style: simple
required: false
responses:
303:
description: OK
content:
application/json:
schema:
description: The redirect url.
type: string
401:
description: Invalid Auth Token
404:
description: Rse or did not found
"""
headers = self.get_headers()
try:
scope, name = parse_scope_name(scope_name,
extract_vo(request.headers))
except ValueError as error:
return generate_http_error_flask(400, error, headers=headers)
try:
client_ip = request.headers.get('X-Forwarded-For',
default=request.remote_addr)
client_location = {
'ip': request.args.get('ip', default=client_ip),
'fqdn': request.args.get('fqdn', default=None),
'site': request.args.get('site', default=None),
}
# use the default HTTP protocols if no scheme is given
schemes = request.args.getlist('schemes') or [
'davs', 'https', 's3'
]
sortby = request.args.get('select', default='random')
sortby = request.args.get('sort', default=sortby)
rse = request.args.get('rse', default=None)
site = request.args.get('site', default=None)
# correctly forward the schemes and select to potential metalink followups
cleaned_url = request.environ.get('REQUEST_URI').split('?')[0]
headers.set(
'Link',
f'<{cleaned_url}/metalink?schemes={",".join(schemes)}&select={sortby}>; rel=describedby; type="application/metalink+xml"'
)
# get vo if given
vo = extract_vo(request.headers)
replicas = list(
list_replicas(dids=[{
'scope': scope,
'name': name,
'type': 'FILE'
}],
schemes=schemes,
client_location=client_location,
vo=vo))
selected_url = None
for r in replicas:
if r['rses']:
dictreplica = {}
if rse:
if rse in r['rses'] and r['rses'][rse]:
selected_url = r['rses'][rse][0]
else:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
else:
for rep in r['rses']:
for replica in r['rses'][rep]:
# since this is HTTP-only redirection, and to ensure compatibility with as many http clients as possible
# forcibly replacement davs and s3 URLs to https
replica = replica.replace(
'davs://',
'https://').replace('s3://', 'https://')
dictreplica[replica] = rep
if not dictreplica:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
elif site:
rep = site_selector(dictreplica, site, vo)
if rep:
selected_url = rep[0]
else:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
else:
rep = sort_replicas(dictreplica,
client_location,
selection=sortby)
selected_url = rep[0]
if selected_url:
response = redirect(selected_url, code=303)
response.headers.extend(headers)
return response
return 'no redirection possible - file does not exist', 404, headers
except ReplicaNotFound as error:
return generate_http_error_flask(404, error, headers=headers)
def get(self, scope_name):
"""
List all replicas for data identifiers.
.. :quickref: Replicas; List replicas for DID.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
:reqheader HTTP_ACCEPT: application/metalink4+xml
:param scope_name: data identifier (scope)/(name).
:resheader Content-Type: application/x-json-stream
:resheader Content-Type: application/metalink4+xml
:status 200: OK.
:status 401: Invalid auth token.
:status 404: DID not found.
:status 406: Not Acceptable.
:returns: A dictionary containing all replicas information.
:returns: A metalink description of replicas if metalink(4)+xml is specified in Accept:
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
content_type = request.accept_mimetypes.best_match(['application/x-json-stream', 'application/metalink4+xml'], 'application/x-json-stream')
metalink = (content_type == 'application/metalink4+xml')
dids = [{'scope': scope, 'name': name}]
select, limit = None, None
client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
client_location = {'ip': client_ip, 'fqdn': None, 'site': None}
schemes = request.args.get('schemes', default=None)
select = request.args.get('select', default=None)
limit = request.args.get('limit', default=None)
if limit:
limit = int(limit)
# Resolve all reasonable protocols when doing metalink for maximum access possibilities
if metalink and schemes is None:
schemes = SUPPORTED_PROTOCOLS
try:
def generate(vo):
# we need to call list_replicas before starting to reply
# otherwise the exceptions won't be propagated correctly
first = metalink
# then, stream the replica information
for rfile in list_replicas(dids=dids, schemes=schemes, vo=vo):
if first and metalink:
# first, set the appropriate content type, and stream the header
yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
first = False
replicas = []
dictreplica = {}
for rse in rfile['rses']:
for replica in rfile['rses'][rse]:
replicas.append(replica)
dictreplica[replica] = rse
replicas = sort_replicas(dictreplica, client_location, selection=select)
if not metalink:
yield dumps(rfile) + '\n'
else:
yield ' <file name="' + rfile['name'] + '">\n'
yield ' <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'
if rfile['adler32'] is not None:
yield ' <hash type="adler32">' + rfile['adler32'] + '</hash>\n'
if rfile['md5'] is not None:
yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n'
yield ' <size>' + str(rfile['bytes']) + '</size>\n'
yield f' <glfn name="/atlas/rucio/{rfile["scope"]}:{rfile["name"]}">'
yield '</glfn>\n'
for idx, replica in enumerate(replicas, start=1):
yield ' <url location="' + str(dictreplica[replica]) + '" priority="' + str(idx) + '">' + escape(replica) + '</url>\n'
if limit and limit == idx:
break
yield ' </file>\n'
if metalink:
if first:
# if still first output, i.e. there were no replicas
yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n</metalink>\n'
else:
# don't forget to send the metalink footer
yield '</metalink>\n'
return try_stream(generate(vo=request.environ.get('vo')), content_type=content_type)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, error)
def get(self, scope_name):
"""
Header Redirect
.. :quickref: HeaderRedirector; Header redirect.
:param scope_name: data identifier (scope)/(name).
:resheader Content-Type: application/metalink+xml'.
:status 303: Redirect.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: DID Not Found.
:status 500: Internal Error.
"""
headers = Headers()
headers.set('Access-Control-Allow-Origin',
request.environ.get('HTTP_ORIGIN'))
headers.set('Access-Control-Allow-Headers',
request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
headers.set('Access-Control-Allow-Methods', '*')
headers.set('Access-Control-Allow-Credentials', 'true')
try:
scope, name = parse_scope_name(
scope_name, request.headers.get('X-Rucio-VO', default='def'))
except ValueError as error:
return generate_http_error_flask(400,
'ValueError',
error.args[0],
headers=headers)
except Exception as error:
logging.exception("Internal Error")
return str(error), 500, headers
try:
# use the default HTTP protocols if no scheme is given
select, rse, site, schemes = 'random', None, None, [
'davs', 'http', 'https'
]
client_ip = request.headers.get('X-Forwarded-For',
default=request.remote_addr)
client_location = {'ip': client_ip, 'fqdn': None, 'site': None}
if request.query_string:
query_string = request.query_string.decode(encoding='utf-8')
params = parse_qs(query_string)
if 'select' in params:
select = params['select'][0]
if 'sort' in params:
select = params['sort'][0]
if 'rse' in params:
rse = params['rse'][0]
if 'site' in params:
site = params['site'][0]
if 'schemes' in params:
schemes = params['schemes'][0]
else:
schemes = ['davs', 'https', 's3']
if 'ip' in params:
client_location['ip'] = params['ip'][0]
if 'fqdn' in params:
client_location['fqdn'] = params['fqdn'][0]
if 'site' in params:
client_location['site'] = params['site'][0]
# correctly forward the schemes and select to potential metalink followups
cleaned_url = request.environ.get('REQUEST_URI').split('?')[0]
if isinstance(schemes, list):
headers.set(
'Link',
'<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"'
% (cleaned_url, ','.join(schemes), select))
else:
headers.set(
'Link',
'<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"'
% (cleaned_url, schemes, select))
schemes = [schemes] # list_replicas needs a list
# get vo if given
vo = request.headers.get('X-Rucio-VO', default='def')
replicas = [
r for r in list_replicas(dids=[{
'scope': scope,
'name': name,
'type': 'FILE'
}],
schemes=schemes,
client_location=client_location,
vo=vo)
]
selected_url = None
for r in replicas:
if r['rses']:
dictreplica = {}
if rse:
if rse in r['rses'] and r['rses'][rse]:
selected_url = r['rses'][rse][0]
else:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
else:
for rep in r['rses']:
for replica in r['rses'][rep]:
# since this is HTTP-only redirection, and to ensure compatibility with as many http clients as possible
# forcibly replacement davs and s3 URLs to https
replica = replica.replace(
'davs://',
'https://').replace('s3://', 'https://')
dictreplica[replica] = rep
if not dictreplica:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
elif site:
rep = site_selector(dictreplica, site, vo)
if rep:
selected_url = rep[0]
else:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
else:
rep = sort_replicas(dictreplica,
client_location,
selection=select)
selected_url = rep[0]
if selected_url:
response = redirect(selected_url, code=303)
response.headers.extend(headers)
return response
return 'no redirection possible - file does not exist', 404, headers
except ReplicaNotFound as error:
return generate_http_error_flask(404,
'ReplicaNotFound',
error.args[0],
headers=headers)
except RucioException as error:
return generate_http_error_flask(500,
error.__class__.__name__,
error.args[0],
headers=headers)
except Exception as error:
logging.exception("Internal Error")
return str(error), 500, headers
def get(self, scope_name):
"""
Metalink redirect
.. :quickref: MetaLinkRedirector; Metalink redirect.
:param scope_name: data identifier (scope)/(name).
:resheader Content-Type: application/metalink4+xml'.
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: DID Not Found.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: Metalink file
"""
headers = Headers()
headers.set('Access-Control-Allow-Origin',
request.environ.get('HTTP_ORIGIN'))
headers.set('Access-Control-Allow-Headers',
request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
headers.set('Access-Control-Allow-Methods', '*')
headers.set('Access-Control-Allow-Credentials', 'true')
try:
scope, name = parse_scope_name(
scope_name, request.headers.get('X-Rucio-VO', default='def'))
except ValueError as error:
return generate_http_error_flask(400,
'ValueError',
error.args[0],
headers=headers)
except Exception as error:
logging.exception("Internal Error")
return str(error), 500, headers
dids, schemes, sortby = [{
'scope': scope,
'name': name
}], ['http', 'https', 'root', 'gsiftp', 'srm', 'davs'], None
# set the correct client IP
client_ip = request.headers.get('X-Forwarded-For',
default=request.remote_addr)
client_location = {'ip': client_ip, 'fqdn': None, 'site': None}
if request.query_string:
query_string = request.query_string.decode(encoding='utf-8')
params = parse_qs(query_string)
if 'schemes' in params:
schemes = params['schemes']
if 'select' in params:
sortby = params['select'][0]
if 'sort' in params:
sortby = params['sort'][0]
if 'ip' in params:
client_location['ip'] = params['ip'][0]
if 'fqdn' in params:
client_location['fqdn'] = params['fqdn'][0]
if 'site' in params:
client_location['site'] = params['site'][0]
# get vo if given
vo = request.headers.get('X-Rucio-VO', default='def')
try:
replicas_iter = list_replicas(dids=dids,
schemes=schemes,
client_location=client_location,
vo=vo)
try:
first = next(replicas_iter)
except StopIteration:
return 'no redirection possible - cannot find the DID', 404
def generate():
# first, set the appropriate content type, and stream the header
yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
# iteratively stream the XML per file
for rfile in itertools.chain((first, ), replicas_iter):
replicas = []
dictreplica = {}
for rse in rfile['rses']:
for replica in rfile['rses'][rse]:
replicas.append(replica)
dictreplica[replica] = rse
# stream metadata
yield ' <file name="' + rfile['name'] + '">\n'
yield ' <identity>' + rfile['scope'] + ':' + rfile[
'name'] + '</identity>\n'
if rfile['adler32'] is not None:
yield ' <hash type="adler32">' + rfile[
'adler32'] + '</hash>\n'
if rfile['md5'] is not None:
yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n'
yield ' <size>' + str(rfile['bytes']) + '</size>\n'
yield ' <glfn name="/atlas/rucio/%s:%s">' % (
rfile['scope'], rfile['name'])
yield '</glfn>\n'
replicas = sort_replicas(dictreplica,
client_location,
selection=sortby)
# stream URLs
idx = 1
for replica in replicas:
yield ' <url location="' + str(
dictreplica[replica]) + '" priority="' + str(
idx) + '">' + replica + '</url>\n'
idx += 1
yield ' </file>\n'
# don't forget to send the metalink footer
yield '</metalink>\n'
return try_stream(generate(),
content_type='application/metalink4+xml')
except DataIdentifierNotFound as error:
return generate_http_error_flask(404,
'DataIdentifierNotFound',
error.args[0],
headers=headers)
except ReplicaNotFound as error:
return generate_http_error_flask(404,
'ReplicaNotFound',
error.args[0],
headers=headers)
except RucioException as error:
return generate_http_error_flask(500,
error.__class__.__name__,
error.args[0],
headers=headers)
except Exception as error:
logging.exception("Internal Error")
return str(error), 500, headers
def get(self, scope_name):
"""
---
summary: Metalink redirect
description: Get Metalink redirect.
tags:
- Redirect
parameters:
- name: scope_name
in: path
description: The data identifier (scope)/(name).
schema:
type: string
style: simple
- name: ip
in: query
description: The client ip.
schema:
type: string
style: simple
required: false
- name: fqdn
in: query
schema:
type: string
style: simple
required: false
- name: site
in: query
schema:
type: string
style: simple
required: false
- name: schemes
in: query
schema:
type: array
style: simple
required: false
- name: select
in: query
schema:
type: string
style: simple
required: false
- name: sort
in: query
schema:
type: string
style: simple
required: false
responses:
200:
description: OK
content:
application/metalink4+xml:
schema:
description: The metalink file.
type: string
401:
description: Invalid Auth Token
404:
description: Rse or did not found
406:
description: Not acceptable
"""
headers = self.get_headers()
try:
scope, name = parse_scope_name(scope_name,
extract_vo(request.headers))
except ValueError as error:
return generate_http_error_flask(400, error, headers=headers)
# set the correct client IP
client_ip = request.headers.get('X-Forwarded-For',
default=request.remote_addr)
client_location = {
'ip': request.args.get('ip', default=client_ip),
'fqdn': request.args.get('fqdn', default=None),
'site': request.args.get('site', default=None),
}
dids = [{'scope': scope, 'name': name}]
schemes = request.args.getlist('schemes') or [
'http', 'https', 'root', 'gsiftp', 'srm', 'davs'
]
sortby = request.args.get('select', default=None)
sortby = request.args.get('sort', default=sortby)
# get vo if given
vo = extract_vo(request.headers)
try:
replicas_iter = list_replicas(dids=dids,
schemes=schemes,
client_location=client_location,
vo=vo)
try:
first = next(replicas_iter)
except StopIteration:
return 'no redirection possible - cannot find the DID', 404
def generate():
# first, set the appropriate content type, and stream the header
yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
# iteratively stream the XML per file
for rfile in itertools.chain((first, ), replicas_iter):
replicas = []
dictreplica = {}
for rse in rfile['rses']:
for replica in rfile['rses'][rse]:
replicas.append(replica)
dictreplica[replica] = rse
# stream metadata
yield ' <file name="' + rfile['name'] + '">\n'
yield ' <identity>' + rfile['scope'] + ':' + rfile[
'name'] + '</identity>\n'
if rfile['adler32'] is not None:
yield ' <hash type="adler32">' + rfile[
'adler32'] + '</hash>\n'
if rfile['md5'] is not None:
yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n'
yield ' <size>' + str(rfile['bytes']) + '</size>\n'
yield f' <glfn name="/atlas/rucio/{rfile["scope"]}:{rfile["name"]}">'
yield '</glfn>\n'
replicas = sort_replicas(dictreplica,
client_location,
selection=sortby)
# stream URLs
idx = 1
for replica in replicas:
yield ' <url location="' + str(
dictreplica[replica]) + '" priority="' + str(
idx) + '">' + replica + '</url>\n'
idx += 1
yield ' </file>\n'
# don't forget to send the metalink footer
yield '</metalink>\n'
return try_stream(generate(),
content_type='application/metalink4+xml')
except (DataIdentifierNotFound, ReplicaNotFound) as error:
return generate_http_error_flask(404, error, headers=headers)
def post(self, scope_name):
"""
Create a new data identifier.
.. :quickref: DIDs; Create a new DID.
.. sourcecode:: http
POST /dids/scope1/container1 HTTP/1.1
Host: rucio.cern.ch
{"type": "CONTAINER", "lifetime": 86400},
**Example response**:
.. sourcecode:: http
HTTP/1.1 201 Created
Vary: Accept
:reqheader Accept: application/json
:param scope_name: data identifier (scope)/(name).
:<json string type: the new DID type
:<json dict statuses: Dictionary with statuses, e.g. {'monotonic':True}
:<json dict meta: Dictionary with metadata, e.g. {'length':1234}
:<json dict rules: Replication rules associated with the did. e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ]
:<json int lifetime: DID's liftime in seconds.
:<json list dids: The content.
:<json string rse: The RSE name when registering replicas.
:status 201: new DIDs created
:status 401: Invalid Auth Token
:status 409: DID already exists
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
parameters = json_parameters()
type_param = param_get(parameters, 'type')
try:
add_did(
scope=scope,
name=name,
type=type_param,
statuses=param_get(parameters, 'statuses', default={}),
meta=param_get(parameters, 'meta', default={}),
rules=param_get(parameters, 'rules', default=[]),
lifetime=param_get(parameters, 'lifetime', default=None),
dids=param_get(parameters, 'dids', default=[]),
rse=param_get(parameters, 'rse', default=None),
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'),
)
except (InvalidObject, InvalidPath) as error:
return generate_http_error_flask(400, error)
except (DataIdentifierNotFound, ScopeNotFound) as error:
return generate_http_error_flask(404, error)
except (DuplicateContent, DataIdentifierAlreadyExists, UnsupportedOperation) as error:
return generate_http_error_flask(409, error)
except AccessDenied as error:
return generate_http_error_flask(401, error)
return 'Created', 201