def _load_json_data(self, response):
"""
Helper method to correctly load json data based on the content type of the http response.
:param response: the response received from the server.
"""
if 'content-type' in response.headers and response.headers['content-type'] == 'application/x-json-stream':
for line in response.iter_lines():
if line:
yield parse_response(line)
elif 'content-type' in response.headers and response.headers['content-type'] == 'application/json':
yield parse_response(response.text)
else: # Exception ?
yield response.text
def _get_exception(self, headers, status_code=None, data=None):
"""
Helper method to parse an error string send by the server and transform it into the corresponding rucio exception.
:param headers: The http response header containing the Rucio exception details.
:param status_code: The http status code.
:param data: The data with the ExceptionMessage.
:return: A rucio exception class and an error string.
"""
try:
data = parse_response(data)
except ValueError:
data = {}
if 'ExceptionClass' not in data:
if 'ExceptionMessage' not in data:
human_http_code = _codes.get(
status_code, None) # NOQA, pylint: disable-msg=W0612
return getattr(
exception, 'RucioException'
), 'no error information passed (http status code: %(status_code)s %(human_http_code)s)' % locals(
)
return getattr(exception,
'RucioException'), data['ExceptionMessage']
exc_cls = None
try:
exc_cls = getattr(exception, data['ExceptionClass'])
except AttributeError:
return getattr(exception,
'RucioException'), data['ExceptionMessage']
return exc_cls, data['ExceptionMessage']
def POST(self):
"""
List all meta of a list of data identifiers.
HTTP Success:
200 OK
HTTP Error:
400 Bad Request
401 Unauthorized
404 DataIdentifierNotFound
500 InternalError
:returns: A list of dictionaries containing all meta.
"""
header('Content-Type', 'application/x-json-stream')
json_data = data()
try:
params = parse_response(json_data)
dids = params['dids']
except KeyError as error:
raise generate_http_error(400, 'ValueError', 'Cannot find mandatory parameter : %s' % str(error))
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
try:
for meta in get_metadata_bulk(dids, vo=ctx.env.get('vo')):
yield render_json(**meta) + '\n'
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
except DataIdentifierNotFound as error:
raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__, error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
def post(self):
"""
Set a tombstone on a list of replicas.
.. :quickref: Tombstone; Set a tombstone on a list of replicas.
:<json string replicas: list fo replicas
:resheader Content-Type: application/x-json-string
:status 201: Created.
:status 401: Invalid auth token.
:status 404: ReplicaNotFound.
:status 500: Internal Error.
"""
json_data = request.data
replicas = []
try:
params = parse_response(json_data)
if 'replicas' in params:
replicas = params['replicas']
except ValueError:
return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list')
try:
for replica in replicas:
set_tombstone(replica['rse'], replica['scope'], replica['name'], issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
except ReplicaNotFound as error:
return generate_http_error_flask(404, 'ReplicaNotFound', error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
except Exception as error:
print(format_exc())
return str(error), 500
return 'Created', 201
def put(self):
"""
Update a file replicas state at a given RSE.
.. :quickref: Replicas; update replicas state.
:<json string rse: The RSE name.
:<json list files: list of dicts with 'scope', 'name' and 'state'.
:status 201: Replica successfully updated.
:status 400: Cannot decode json parameter list.
:status 401: Invalid auth token.
:status 500: Internal Error.
"""
json_data = request.data
try:
parameters = parse_response(json_data)
except ValueError:
return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list')
try:
update_replicas_states(rse=parameters['rse'], files=parameters['files'], issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied', error.args[0])
except UnsupportedOperation as error:
return generate_http_error_flask(500, 'UnsupportedOperation', error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
except Exception as error:
print(format_exc())
return str(error), 500
return '', 200
def POST(self):
"""
Declare a list of suspicious replicas.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
"""
json_data = data()
pfns = []
header('Content-Type', 'application/x-json-stream')
try:
params = parse_response(json_data)
if 'pfns' in params:
pfns = params['pfns']
if 'reason' in params:
reason = params['reason']
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
not_declared_files = {}
try:
not_declared_files = declare_suspicious_file_replicas(
pfns=pfns, reason=reason, issuer=ctx.env.get('issuer'))
except ReplicaNotFound, e:
raise generate_http_error(404, 'ReplicaNotFound', e.args[0][0])
def POST(self):
"""
List the DIDs associated to a list of replicas.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
:returns: A list of dictionaries containing the mAPPing PFNs to DIDs.
"""
json_data = data()
rse, pfns = None, []
header('Content-Type', 'application/x-json-stream')
rse = None
try:
params = parse_response(json_data)
if 'pfns' in params:
pfns = params['pfns']
if 'rse' in params:
rse = params['rse']
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
try:
for pfn in get_did_from_pfns(pfns, rse):
yield dumps(pfn) + '\n'
except RucioException, e:
raise generate_http_error(500, e.__class__.__name__, e.args[0][0])
def DELETE(self):
"""
Delete file replicas at a given RSE.
HTTP Success:
200 Ok
HTTP Error:
401 Unauthorized
409 Conflict
500 Internal Error
"""
json_data = data()
try:
parameters = parse_response(json_data)
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
try:
delete_replicas(rse=parameters['rse'],
files=parameters['files'],
issuer=ctx.env.get('issuer'),
ignore_availability=parameters.get(
'ignore_availability', False))
except AccessDenied, e:
raise generate_http_error(401, 'AccessDenied', e.args[0][0])
def POST(self):
"""
Create file replicas at a given RSE.
HTTP Success:
201 Created
HTTP Error:
401 Unauthorized
409 Conflict
500 Internal Error
"""
json_data = data()
try:
parameters = parse_response(json_data)
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
try:
add_replicas(rse=parameters['rse'],
files=parameters['files'],
issuer=ctx.env.get('issuer'),
ignore_availability=parameters.get(
'ignore_availability', False))
except InvalidPath, e:
raise generate_http_error(400, 'InvalidPath', e.args[0][0])
def POST(self, rse, operation):
"""
Get URLs for files at a given RSE.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 Internal Error
"""
header('Content-Type', 'application/json')
json_data = data()
try:
parameters = parse_response(json_data)
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
try:
result = objectstore.get_signed_urls(parameters,
rse=rse,
operation=operation)
for url in result:
if isinstance(result[url], Exception):
raise result[url]
return render_json(**result)
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print traceback.format_exc()
raise InternalError(error)
def POST(self, rse):
"""
Get files metadata at a given RSE.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 Internal Error
"""
header('Content-Type', 'application/json')
json_data = data()
try:
parameters = parse_response(json_data)
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
try:
url = parameters['url']
new_url = parameters['new_url']
objectstore.rename(url, new_url, rse=rse)
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print traceback.format_exc()
raise InternalError(error)
raise OK()
def POST(self):
"""
Set a tombstone on a list of replicas.
HTTP Success:
201 OK
HTTP Error:
401 Unauthorized
404 ReplicaNotFound
500 InternalError
"""
json_data = data()
replicas = []
try:
params = parse_response(json_data)
if 'replicas' in params:
replicas = params['replicas']
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
try:
for replica in replicas:
set_tombstone(replica['rse'], replica['scope'], replica['name'], issuer=ctx.env.get('issuer'), vo=ctx.env.get('vo'))
except ReplicaNotFound as error:
raise generate_http_error(404, 'ReplicaNotFound', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__, error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
raise Created()
def POST(self):
""" Import data into Rucio.
HTTP Success:
200 OK
HTTP Error:
400 Bad request
401 Unauthorized
404 Resource not Found
500 InternalError
"""
header('Content-Type', 'application/x-json-stream')
json_data = data()
try:
data_to_import = json_data and parse_response(json_data)
except ValueError:
raise generate_http_error(
400, 'ValueError', 'Cannot decode json parameter dictionary')
try:
import_data(data=data_to_import,
issuer=ctx.env.get('issuer'),
vo=ctx.env.get('vo'))
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
raise Created()
def _load_json_data(self, response):
"""
Helper method to correctly load json data based on the content type of the http response.
:param response: the response received from the server.
"""
if 'content-type' in response.headers and response.headers[
'content-type'] == 'application/x-json-stream':
for line in response.iter_lines():
if line:
yield parse_response(line)
elif 'content-type' in response.headers and response.headers[
'content-type'] == 'application/json':
yield parse_response(response.text)
else: # Exception ?
yield response.text
def post(self, rse):
"""
Pass list of urls and return their metadata.
.. :quickref: ObjectStoreInfo; Get files metadata at a given RSE.
:param rse: RSE name.
:<json string urls: A list of URL string.
:resheader Content-Type: application/json
:status 200: OK.
:status 400: Cannot decode json parameter list.
:status 401: Invalid Auth Token.
:status 500: Internal Error.
:returns: Dictonary of metadata.
"""
json_data = request.data
try:
parameters = parse_response(json_data)
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
result = objectstore.get_metadata(parameters, rse=rse)
for url in result:
if isinstance(result[url], Exception):
return Response(result[url])
return Response(render_json(**result),
content_type="application/json")
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(traceback.format_exc())
return error, 500
def POST(self):
"""
Declare a list of bad replicas.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
"""
json_data = data()
rse, pfns = None, []
header('Content-Type', 'application/x-json-stream')
try:
params = parse_response(json_data)
if 'pfns' in params:
pfns = params['pfns']
if 'rse' in params:
rse = params['rse']
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
try:
declare_bad_file_replicas(rse=rse, pfns=pfns, issuer=ctx.env.get('issuer'))
except ReplicaNotFound, e:
raise generate_http_error(404, 'ReplicaNotFound', e.args[0][0])
def PUT(self):
"""
Update a file replicas state at a given RSE.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 Internal Error
"""
json_data = data()
try:
parameters = parse_response(json_data)
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
try:
update_replicas_states(rse=parameters['rse'],
files=parameters['files'],
issuer=ctx.env.get('issuer'))
except AccessDenied as error:
raise generate_http_error(401, 'AccessDenied', error.args[0])
except UnsupportedOperation as error:
raise generate_http_error(500, 'UnsupportedOperation',
error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
raise OK()
def post(self, rse):
"""
Rename object.
.. :quickref: ObjectStoreRename; Rename object.
:param rse: the RSE name.
:<json string url: the URL string.
:<json string new_url: the new URL string.
:<json string urls: A list of URL string.
:resheader Content-Type: application/json
:status 200: OK.
:status 400: Cannot decode json parameter list.
:status 401: Invalid Auth Token.
:status 500: Internal Error.
"""
json_data = request.data
try:
parameters = parse_response(json_data)
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
url = parameters['url']
new_url = parameters['new_url']
objectstore.rename(url, new_url, rse=rse)
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(traceback.format_exc())
return error, 500
return "OK", 200
def check_error_api(params, exception_class, exception_message, code):
headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'}
response = rest_client.get('/requests/list', query_string=params, headers=headers(auth(auth_token), vohdr(vo), hdrdict(headers_dict)))
assert response.status_code == code
body = parse_response(response.get_data(as_text=True))
assert body['ExceptionClass'] == exception_class
assert body['ExceptionMessage'] == exception_message
def POST(self):
"""
List the DIDs associated to a list of replicas.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
:returns: A list of dictionaries containing the mapping PFNs to DIDs.
"""
json_data = data()
rse, pfns = None, []
header('Content-Type', 'application/x-json-stream')
try:
params = parse_response(json_data)
if 'pfns' in params:
pfns = params['pfns']
if 'rse' in params:
rse = params['rse']
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
try:
for pfn in get_did_from_pfns(pfns, rse):
yield dumps(pfn) + '\n'
except RucioException, e:
raise generate_http_error(500, e.__class__.__name__, e.args[0][0])
def post(self):
"""
Declare a list of bad PFNs.
.. :quickref: BadPFNs; Declare bad replicas.
:<json string pfns: The list of PFNs.
:<json string reason: The reason of the loss.
:<json string state: The state is eiher BAD, SUSPICIOUS or TEMPORARY_UNAVAILABLE.
:<json string expires_at: The expiration date. Only apply to TEMPORARY_UNAVAILABLE.
:resheader Content-Type: application/x-json-string
:status 201: Created.
:status 400: Cannot decode json parameter list.
:status 401: Invalid auth token.
:status 404: Replica not found.
:status 500: Internal Error.
:returns: A list of not successfully declared files.
"""
json_data = request.data
pfns = []
reason = None
state = None
expires_at = None
try:
params = parse_response(json_data)
if 'pfns' in params:
pfns = params['pfns']
if 'reason' in params:
reason = params['reason']
if 'reason' in params:
reason = params['reason']
if 'state' in params:
reason = params['state']
if 'expires_at' in params:
expires_at = datetime.strptime(params['expires_at'],
"%Y-%m-%dT%H:%M:%S.%f")
add_bad_pfns(pfns=pfns,
issuer=request.environ.get('issuer'),
state=state,
reason=reason,
expires_at=expires_at)
except (ValueError, InvalidType) as error:
return generate_http_error_flask(400, 'ValueError', error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except ReplicaNotFound as error:
return generate_http_error_flask(404, 'ReplicaNotFound',
error.args[0])
except Duplicate as error:
return generate_http_error_flask(409, 'Duplicate', error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
return 'Created', 201
def test_auth_ssh(self):
""" MULTI VO (REST): Test ssh authentication to multiple VOs """
mw = []
try:
add_account_identity(PUBLIC_KEY, 'SSH', 'root', '*****@*****.**', 'root', **self.vo)
add_account_identity(PUBLIC_KEY, 'SSH', 'root', '*****@*****.**', 'root', **self.new_vo)
except Duplicate:
pass # Might already exist, can skip
headers_tst = {'X-Rucio-Account': 'root'}
headers_tst.update(self.vo_header)
res_tst = TestApp(auth_app.wsgifunc(*mw)).get('/ssh_challenge_token', headers=headers_tst, expect_errors=True)
assert_equal(res_tst.status, 200)
challenge_tst = str(res_tst.header('X-Rucio-SSH-Challenge-Token'))
headers_tst.update({'X-Rucio-SSH-Signature': ssh_sign(PRIVATE_KEY, challenge_tst)})
res_tst = TestApp(auth_app.wsgifunc(*mw)).get('/ssh', headers=headers_tst, expect_errors=True)
assert_equal(res_tst.status, 200)
token_tst = str(res_tst.header('X-Rucio-Auth-Token'))
headers_new = {'X-Rucio-Account': 'root'}
headers_new.update(self.new_header)
res_new = TestApp(auth_app.wsgifunc(*mw)).get('/ssh_challenge_token', headers=headers_new, expect_errors=True)
assert_equal(res_new.status, 200)
challenge_tst = str(res_new.header('X-Rucio-SSH-Challenge-Token'))
headers_new.update({'X-Rucio-SSH-Signature': ssh_sign(PRIVATE_KEY, challenge_tst)})
res_new = TestApp(auth_app.wsgifunc(*mw)).get('/ssh', headers=headers_new, expect_errors=True)
assert_equal(res_new.status, 200)
token_new = str(res_new.header('X-Rucio-Auth-Token'))
headers_tst = {'X-Rucio-Auth-Token': str(token_tst)}
res_tst = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_tst, expect_errors=True)
assert_equal(res_tst.status, 200)
accounts_tst = [parse_response(a)['account'] for a in res_tst.body.decode().split('\n')[:-1]]
assert_not_equal(len(accounts_tst), 0)
assert_in(self.account_tst, accounts_tst)
assert_not_in(self.account_new, accounts_tst)
headers_new = {'X-Rucio-Auth-Token': str(token_new)}
res_new = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_new, expect_errors=True)
assert_equal(res_new.status, 200)
accounts_new = [parse_response(a)['account'] for a in res_new.body.decode().split('\n')[:-1]]
assert_not_equal(len(accounts_new), 0)
assert_in(self.account_new, accounts_new)
assert_not_in(self.account_tst, accounts_new)
def check_correct_api(params, expected_requests):
headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'}
response = rest_client.get('/requests/history/list', query_string=params, headers=headers(auth(auth_token), vohdr(vo), hdrdict(headers_dict)))
assert response.status_code == 200
requests = set()
for request in response.get_data(as_text=True).split('\n')[:-1]:
request = parse_response(request)
requests.add((request['state'], request['source_rse_id'], request['dest_rse_id'], request['name']))
assert requests == expected_requests
def post(self):
"""
Create file replicas at a given RSE.
.. :quickref: Replicas; create replicas at RSE
:<json string rse: The RSE name.
:<json list files: list of dicts with 'scope', 'name', 'bytes', 'meta' and 'adler32'.
:<json bool ignore_availability: Flag to ignore the RSE blacklisting.
:status 201: Replica Successfully created.
:status 400: Invalid Path.
:status 401: Invalid auth token.
:status 404: RSE not found.
:status 409: Replica already exists.
:status 409: DID already exists.
:status 503: Resource Temporary Unavailable.
"""
json_data = request.data
try:
parameters = parse_response(json_data)
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
add_replicas(rse=parameters['rse'],
files=parameters['files'],
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'),
ignore_availability=parameters.get(
'ignore_availability', False))
except InvalidPath as error:
return generate_http_error_flask(400, 'InvalidPath', error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except Duplicate as error:
return generate_http_error_flask(409, 'Duplicate', error.args[0])
except DataIdentifierAlreadyExists as error:
return generate_http_error_flask(409,
'DataIdentifierAlreadyExists',
error.args[0])
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except ResourceTemporaryUnavailable as error:
return generate_http_error_flask(503,
'ResourceTemporaryUnavailable',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
return 'Created', 201
def POST(self):
"""
Declare a list of bad replicas by DID.
HTTP Success:
200 OK
HTTP Error:
400 BadRequest
401 Unauthorized
409 Conflict
500 InternalError
"""
json_data = data()
dids = []
rse = None
reason = None
expires_at = None
header('Content-Type', 'application/x-json-stream')
try:
params = parse_response(json_data)
if 'dids' in params:
dids = params['dids']
if 'rse' in params:
rse = params['rse']
if 'reason' in params:
reason = params['reason']
state = BadFilesStatus.BAD
if 'expires_at' in params and params['expires_at']:
expires_at = datetime.strptime(params['expires_at'],
"%Y-%m-%dT%H:%M:%S.%f")
not_declared_files = add_bad_dids(dids=dids,
rse=rse,
issuer=ctx.env.get('issuer'),
state=state,
reason=reason,
expires_at=expires_at,
vo=ctx.env.get('vo'))
except (ValueError, InvalidType) as error:
raise generate_http_error(400, 'ValueError', error.args[0])
except AccessDenied as error:
raise generate_http_error(401, 'AccessDenied', error.args[0])
except ReplicaNotFound as error:
raise generate_http_error(404, 'ReplicaNotFound', error.args[0])
except Duplicate as error:
raise generate_http_error(409, 'Duplicate', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
raise Created(dumps(not_declared_files))
def post(self):
"""
List dataset replicas for multiple DIDs.
.. :quickref: DatasetReplicasBulk; List dataset replicas (bulk).
:<json list dids: List of DIDs for querying the datasets.
:resheader Content-Type: application/x-json-stream
:status 200: OK.
:status 400: Bad Request.
:status 401: Invalid auth token.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: A dictionary containing all replicas information.
"""
json_data = request.data
try:
params = parse_response(json_data)
dids = params['dids']
didslength = len(dids)
except KeyError as error:
return generate_http_error_flask(
400, 'KeyError',
'Cannot find mandatory parameter : %s' % str(error))
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
if didslength == 0:
return generate_http_error_flask(400, 'ValueError',
'List of DIDs is empty')
try:
data = ""
for row in list_dataset_replicas_bulk(
dids=dids, vo=request.environ.get('vo')):
data += dumps(row, cls=APIEncoder) + '\n'
return Response(data, content_type='application/x-json-stream')
except InvalidObject as error:
return generate_http_error_flask(
400, 'InvalidObject',
'Cannot validate DIDs: %s' % (str(error)))
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
def POST(self):
"""
Create file replicas at a given RSE.
HTTP Success:
201 Created
HTTP Error:
401 Unauthorized
405 Method Not Allowed
409 Conflict
500 Internal Error
503 Service Unavailable
"""
json_data = data()
try:
parameters = parse_response(json_data)
except (ValueError, InvalidType):
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
try:
add_files(lfns=parameters['lfns'],
issuer=ctx.env.get('issuer'),
ignore_availability=parameters.get(
'ignore_availability', False))
except InvalidPath as error:
raise generate_http_error(400, 'InvalidPath', error.args[0])
except AccessDenied as error:
raise generate_http_error(401, 'AccessDenied', error.args[0])
except UnsupportedOperation as error:
raise generate_http_error(405, 'UnsupportedOperation',
error.args[0])
except Duplicate as error:
raise generate_http_error(409, 'Duplicate', error.args[0])
except DataIdentifierAlreadyExists as error:
raise generate_http_error(409, 'DataIdentifierAlreadyExists',
error.args[0])
except RSENotFound as error:
raise generate_http_error(404, 'RSENotFound', error.args[0])
except DatabaseException as error:
raise generate_http_error(503, 'DatabaseException', error.args[0])
except ResourceTemporaryUnavailable as error:
raise generate_http_error(503, 'ResourceTemporaryUnavailable',
error.args[0])
except RucioException as error:
print(format_exc())
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
raise Created()
def post(self):
"""
Atomic method used by the RucioFileCatalog plugin in Dirac that :
- Creates files and their replicas
- Creates the dataset containing the files and attach the files to the dataset
- Creates a rule on the dataset with RSE expression ANY and grouping NONE
- Creates all the container hierarchy containing the dataset
..:quickref: AddFiles; Method used by the RucioFileCatalog plugin in Dirac.
:<json list lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}.
:<json bool ignore_availability: A boolean to choose if unavailable sites need to be ignored.
:status 201: Created.
:status 400: Cannot decode json parameter list.
:status 401: Invalid auth token.
:status 404: DID not found.
:status 405: Unsupported Operation.
:status 409: Duplicate.
:status 500: Internal Error.
:status 503: Temporary error.
"""
try:
parameters = parse_response(request.data)
except (ValueError, InvalidType):
return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list')
try:
add_files(lfns=parameters['lfns'], issuer=request.environ.get('issuer'), ignore_availability=parameters.get('ignore_availability', False))
except InvalidPath as error:
return generate_http_error_flask(400, 'InvalidPath', error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied', error.args[0])
except UnsupportedOperation as error:
return generate_http_error_flask(405, 'UnsupportedOperation', error.args[0])
except Duplicate as error:
return generate_http_error_flask(409, 'Duplicate', error.args[0])
except DataIdentifierAlreadyExists as error:
return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0])
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except DatabaseException as error:
return generate_http_error_flask(503, 'DatabaseException', error.args[0])
except ResourceTemporaryUnavailable as error:
return generate_http_error_flask(503, 'ResourceTemporaryUnavailable', error.args[0])
except RucioException as error:
logging.exception("Internal Error")
return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
except Exception as error:
logging.exception("Internal Error")
return str(error), 500
return 'Created', 201
def POST(self):
"""
Declare a list of bad PFNs.
HTTP Success:
200 OK
HTTP Error:
400 BadRequest
401 Unauthorized
409 Conflict
500 InternalError
"""
json_data = data()
pfns = []
reason = None
state = None
expires_at = None
header('Content-Type', 'application/x-json-stream')
try:
params = parse_response(json_data)
if 'pfns' in params:
pfns = params['pfns']
if 'reason' in params:
reason = params['reason']
if 'state' in params:
state = params['state']
if 'expires_at' in params:
expires_at = datetime.strptime(params['expires_at'],
"%Y-%m-%dT%H:%M:%S.%f")
add_bad_pfns(pfns=pfns,
issuer=ctx.env.get('issuer'),
state=state,
reason=reason,
expires_at=expires_at)
except (ValueError, InvalidType) as error:
raise generate_http_error(400, 'ValueError', error.args[0])
except AccessDenied as error:
raise generate_http_error(401, 'AccessDenied', error.args[0])
except ReplicaNotFound as error:
raise generate_http_error(404, 'ReplicaNotFound', error.args[0])
except Duplicate as error:
raise generate_http_error(409, 'Duplicate', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
raise Created()
def POST(self):
"""
List dataset replicas for multiple DIDs.
HTTP Success:
200 OK
HTTP Error:
400 Bad Request
401 Unauthorized
406 Not Acceptable
500 InternalError
:returns: A dictionary containing all replicas information.
"""
header('Content-Type', 'application/x-json-stream')
json_data = data()
try:
params = parse_response(json_data)
dids = params['dids']
didslength = len(dids)
except KeyError as error:
raise generate_http_error(
400, 'KeyError',
'Cannot find mandatory parameter : %s' % str(error))
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
if didslength == 0:
raise generate_http_error(400, 'ValueError',
'List of DIDs is empty')
try:
for row in list_dataset_replicas_bulk(dids=dids,
vo=ctx.env.get('vo')):
yield dumps(row, cls=APIEncoder) + '\n'
except InvalidObject as error:
raise generate_http_error(
400, 'InvalidObject',
'Cannot validate DIDs: %s' % (str(error)))
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
def test_auth_gss(self):
""" MULTI VO (REST): Test gss authentication to multiple VOs """
mw = []
# Can't rely on `requests_kerberos` module being present, so get tokens from API instead
token_tst = get_auth_token_gss('root', '*****@*****.**', 'unknown', None, **self.vo).token
token_new = get_auth_token_gss('root', '*****@*****.**', 'unknown', None, **self.new_vo).token
headers_tst = {'X-Rucio-Auth-Token': str(token_tst)}
res_tst = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_tst, expect_errors=True)
assert_equal(res_tst.status, 200)
accounts_tst = [parse_response(a)['account'] for a in res_tst.body.decode().split('\n')[:-1]]
assert_not_equal(len(accounts_tst), 0)
assert_in(self.account_tst, accounts_tst)
assert_not_in(self.account_new, accounts_tst)
headers_new = {'X-Rucio-Auth-Token': str(token_new)}
res_new = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_new, expect_errors=True)
assert_equal(res_new.status, 200)
accounts_new = [parse_response(a)['account'] for a in res_new.body.decode().split('\n')[:-1]]
assert_not_equal(len(accounts_new), 0)
assert_in(self.account_new, accounts_new)
assert_not_in(self.account_tst, accounts_new)
def test_auth_x509(self):
""" MULTI VO (REST): Test X509 authentication to multiple VOs """
mw = []
# TestApp doesn't support `cert` argument, so get tokens from API instead
token_tst = get_auth_token_x509('root', '/CN=Rucio User', 'unknown', None, **self.vo).token
token_new = get_auth_token_x509('root', '/CN=Rucio User', 'unknown', None, **self.new_vo).token
headers_tst = {'X-Rucio-Auth-Token': str(token_tst)}
res_tst = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_tst, expect_errors=True)
assert_equal(res_tst.status, 200)
accounts_tst = [parse_response(a)['account'] for a in res_tst.body.decode().split('\n')[:-1]]
assert_not_equal(len(accounts_tst), 0)
assert_in(self.account_tst, accounts_tst)
assert_not_in(self.account_new, accounts_tst)
headers_new = {'X-Rucio-Auth-Token': str(token_new)}
res_new = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_new, expect_errors=True)
assert_equal(res_new.status, 200)
accounts_new = [parse_response(a)['account'] for a in res_new.body.decode().split('\n')[:-1]]
assert_not_equal(len(accounts_new), 0)
assert_in(self.account_new, accounts_new)
assert_not_in(self.account_tst, accounts_new)
def delete(self):
"""
Delete file replicas at a given RSE.
.. :quickref: Replicas; Delete replica at RSE.
:<json string rse: The RSE name.
:<json list files: list of dicts with 'scope', 'name'.
:<json bool ignore_availability: Flag to ignore the RSE blacklisting.
:status 200: Replica successfully deleted.
:status 400: Cannot decode json parameter list.
:status 401: Invalid auth token.
:status 404: RSE not found.
:status 404: Replica not found.
:status 500: Internal Error.
"""
json_data = request.data
try:
parameters = parse_response(json_data)
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
delete_replicas(rse=parameters['rse'],
files=parameters['files'],
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'),
ignore_availability=parameters.get(
'ignore_availability', False))
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except ResourceTemporaryUnavailable as error:
return generate_http_error_flask(503,
'ResourceTemporaryUnavailable',
error.args[0])
except ReplicaNotFound as error:
return generate_http_error_flask(404, 'ReplicaNotFound',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
return 'OK', 200
def PUT(self):
"""
Update a file replicas state at a given RSE.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 Internal Error
"""
json_data = data()
try:
parameters = parse_response(json_data)
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
try:
update_replicas_states(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer'))
except AccessDenied, e:
raise generate_http_error(401, 'AccessDenied', e.args[0][0])
def DELETE(self):
"""
Delete file replicas at a given RSE.
HTTP Success:
200 Ok
HTTP Error:
401 Unauthorized
409 Conflict
500 Internal Error
"""
json_data = data()
try:
parameters = parse_response(json_data)
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
try:
delete_replicas(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer'), ignore_availability=parameters.get('ignore_availability', False))
except AccessDenied, e:
raise generate_http_error(401, 'AccessDenied', e.args[0][0])
def POST(self):
"""
List all replicas for data identifiers.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
:returns: A dictionary containing all replicas information.
:returns: A metalink description of replicas if metalink(4)+xml is specified in Accept:
"""
metalink = None
if ctx.env.get('HTTP_ACCEPT') is not None:
tmp = ctx.env.get('HTTP_ACCEPT').split(',')
# first check if client accepts metalink
if 'application/metalink+xml' in tmp:
metalink = 3
# but prefer metalink4 if the client has support for it
# (clients can put both in their ACCEPT header!)
if 'application/metalink4+xml' in tmp:
metalink = 4
dids, schemes, select, unavailable, limit = [], None, None, False, None
ignore_availability = False
all_states = False
json_data = data()
try:
params = parse_response(json_data)
if 'dids' in params:
dids = params['dids']
if 'schemes' in params:
schemes = params['schemes']
if 'unavailable' in params:
unavailable = params['unavailable']
ignore_availability = True
if 'all_states' in params:
all_states = params['all_states']
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
if ctx.query:
params = parse_qs(ctx.query[1:])
if 'select' in params:
select = params['select'][0]
if 'limit' in params:
limit = params['limit'][0]
try:
# first, set the appropriate content type, and stream the header
if metalink is None:
header('Content-Type', 'application/x-json-stream')
elif metalink == 3:
header('Content-Type', 'application/metalink+xml')
yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink version="3.0" xmlns="http://www.metalinker.org/">\n<files>\n'
elif metalink == 4:
header('Content-Type', 'application/metalink4+xml')
yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
# then, stream the replica information
for rfile in list_replicas(dids=dids, schemes=schemes, unavailable=unavailable, request_id=ctx.env.get('request_id'), ignore_availability=ignore_availability, all_states=all_states):
client_ip = ctx.get('ip')
replicas = []
dictreplica = {}
for rse in rfile['rses']:
for replica in rfile['rses'][rse]:
replicas.append(replica)
dictreplica[replica] = rse
if select == 'geoip':
replicas = geoIP_order(dictreplica, client_ip)
else:
replicas = random_order(dictreplica, client_ip)
if metalink is None:
yield dumps(rfile, cls=APIEncoder) + '\n'
elif metalink == 3:
idx = 0
yield ' <file name="' + rfile['name'] + '">\n <resources>\n'
for replica in replicas:
yield ' <url type="http" preference="' + str(idx) + '">' + replica + '</url>\n'
idx += 1
if limit and limit == idx:
break
yield ' </resources>\n </file>\n'
elif metalink == 4:
yield ' <file name="' + rfile['name'] + '">\n'
yield ' <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'
if rfile['adler32'] is not None:
yield ' <hash type="adler32">' + rfile['adler32'] + '</hash>\n'
if rfile['md5'] is not None:
yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n'
yield ' <size>' + str(rfile['bytes']) + '</size>\n'
idx = 0
for replica in replicas:
yield ' <url location="' + str(dictreplica[replica]) + '" priority="' + str(idx + 1) + '">' + replica + '</url>\n'
idx += 1
if limit and limit == idx:
break
yield ' </file>\n'
# don't forget to send the metalink footer
if metalink:
if metalink == 3:
yield '</files>\n</metalink>\n'
elif metalink == 4:
yield '</metalink>\n'
except DataIdentifierNotFound, e:
raise generate_http_error(404, 'DataIdentifierNotFound', e.args[0][0])