def GET(self, rse, scheme):
""" List all references of the provided RSE for the given protocol.
HTTP Success:
200 OK
HTTP Error:
404 Resource not Found
406 Not Acceptable
500 InternalError
:returns: A list with detailed protocol information.
"""
header('Content-Type', 'application/json')
p_list = None
try:
p_list = get_rse_protocols(rse, issuer=ctx.env.get('issuer'), vo=ctx.env.get('vo'))
except RSENotFound as error:
raise generate_http_error(404, 'RSENotFound', error.args[0])
except RSEProtocolNotSupported as error:
raise generate_http_error(404, 'RSEProtocolNotSupported', error.args[0])
except RSEProtocolDomainNotSupported as error:
raise generate_http_error(404, 'RSEProtocolDomainNotSupported', error.args[0])
except Exception as error:
print(error)
print(format_exc())
raise InternalError(error)
return dumps(p_list)
def get(self, rse, scheme):
""" List all references of the provided RSE for the given protocol.
.. :quickref: Protocol; List RSE protocol.
:param rse: The RSE name.
:param scheme: The protocol identifier.
:resheader Content-Type: application/json
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: RSE Protocol Not Supported.
:status 404: RSE Protocol Domain Not Supported.
:status 406: Not Acceptable.
:returns: A list with detailed protocol information.
"""
p_list = None
try:
p_list = get_rse_protocols(rse,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'))
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except RSEProtocolNotSupported as error:
return generate_http_error_flask(404, 'RSEProtocolNotSupported',
error.args[0])
except RSEProtocolDomainNotSupported as error:
return generate_http_error_flask(404,
'RSEProtocolDomainNotSupported',
error.args[0])
except Exception as error:
print(format_exc())
return str(error), 500
return jsonify(p_list)
def GET(self, rse):
""" List all supported protocols of the given RSE.
HTTP Success:
200 OK
HTTP Error:
404 Resource not Found
406 Not Acceptable
500 InternalError
:returns: A list containing all supported protocols and all their attributes.
"""
header('Content-Type', 'application/json')
p_list = None
try:
p_list = get_rse_protocols(rse, issuer=ctx.env.get('issuer'), vo=ctx.env.get('vo'))
except RSEOperationNotSupported as error:
raise generate_http_error(404, 'RSEOperationNotSupported', error.args[0])
except RSENotFound as error:
raise generate_http_error(404, 'RSENotFound', error.args[0])
except RSEProtocolNotSupported as error:
raise generate_http_error(404, 'RSEProtocolNotSupported', error.args[0])
except RSEProtocolDomainNotSupported as error:
raise generate_http_error(404, 'RSEProtocolDomainNotSupported', error.args[0])
except Exception as error:
print(error)
print(format_exc())
raise InternalError(error)
if len(p_list['protocols']):
return dumps(p_list['protocols'])
else:
raise generate_http_error(404, 'RSEProtocolNotSupported', 'No prptocols found for this RSE')
def get(self, rse):
""" List all supported protocols of the given RSE.
.. :quickref: Protocols; List all RSE protocols.
:param rse: The RSE name.
:resheader Content-Type: application/json
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE Operation Not Supported.
:status 404: RSE Not Found.
:status 404: RSE Protocol Domain Not Supported.
:status 404: RSE Protocol Not Supported.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: A list containing all supported protocols and all their attributes.
"""
p_list = None
try:
p_list = get_rse_protocols(rse, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
except RSEOperationNotSupported as error:
return generate_http_error_flask(404, 'RSEOperationNotSupported', error.args[0])
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except RSEProtocolNotSupported as error:
return generate_http_error_flask(404, 'RSEProtocolNotSupported', error.args[0])
except RSEProtocolDomainNotSupported as error:
return generate_http_error_flask(404, 'RSEProtocolDomainNotSupported', error.args[0])
except Exception as error:
logging.exception("Internal Error")
return str(error), 500
if len(p_list['protocols']):
return jsonify(p_list['protocols'])
else:
return generate_http_error_flask(404, 'RSEProtocolNotSupported', 'No protocols found for this RSE')
def get(self, rse, scheme=None):
"""
Return PFNs for a set of LFNs. Formatted as a JSON object where the key is a LFN and the
value is the corresponding PFN.
.. :quickref: Attributes; Translate LFNs to PFNs.
:param rse: The RSE name.
:param scheme: The protocol identifier.
:query lfn: One or moref LFN to translate.
:query scheme: Optional argument to help with the protocol selection (e.g., http / gsiftp / srm)
:query domain: Optional argument used to select the protocol for wan or lan use cases.
:query operation: Optional query argument to select the protoco for read-vs-writes.
:resheader Content-Type: application/json
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: RSE Protocol Not Supported.
:status 404: RSE Protocol Domain Not Supported.
:status 500: Internal Error.
:returns: A list with detailed PFN information.
"""
lfns = []
scheme = request.get('scheme', None)
domain = request.get('domain', 'wan')
operation = request.get('operation', 'write')
p_lfns = request.get('lfn', None)
if p_lfns:
info = p_lfns.split(":", 1)
if len(info) != 2:
return generate_http_error_flask(400, 'InvalidPath',
'LFN in invalid format')
lfn_dict = {'scope': info[0], 'name': info[1]}
lfns.append(lfn_dict)
rse_settings = None
try:
rse_settings = get_rse_protocols(
rse, issuer=request.environ.get('issuer'))
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except RSEProtocolNotSupported as error:
return generate_http_error_flask(404, 'RSEProtocolNotSupported',
error.args[0])
except RSEProtocolDomainNotSupported as error:
return generate_http_error_flask(404,
'RSEProtocolDomainNotSupported',
error.args[0])
except Exception as error:
print(error)
print(format_exc())
return error, 500
pfns = rsemanager.lfns2pfns(rse_settings,
lfns,
operation=operation,
scheme=scheme,
domain=domain)
return Response(dumps(pfns), content_type="application/json")
def get(self, rse, scheme=None):
"""
Return PFNs for a set of LFNs. Formatted as a JSON object where the key is a LFN and the
value is the corresponding PFN.
.. :quickref: Attributes; Translate LFNs to PFNs.
:param rse: The RSE name.
:param scheme: The protocol identifier.
:query lfn: One or moref LFN to translate.
:query scheme: Optional argument to help with the protocol selection (e.g., http / gsiftp / srm)
:query domain: Optional argument used to select the protocol for wan or lan use cases.
:query operation: Optional query argument to select the protoco for read-vs-writes.
:resheader Content-Type: application/json
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: RSE Protocol Not Supported.
:status 404: RSE Protocol Domain Not Supported.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: A list with detailed PFN information.
"""
lfns = []
scheme = None
domain = 'wan'
operation = 'write'
if request.query_string:
for key, val in request.args.items():
if key == 'lfn':
info = val.split(":", 1)
if len(info) != 2:
return generate_http_error_flask(400, 'InvalidPath', 'LFN in invalid format')
lfn_dict = {'scope': info[0], 'name': info[1]}
lfns.append(lfn_dict)
elif key == 'scheme':
scheme = val
elif key == 'domain':
domain = val
elif key == 'operation':
operation = val
rse_settings = None
try:
rse_settings = get_rse_protocols(rse, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except RSEProtocolNotSupported as error:
return generate_http_error_flask(404, 'RSEProtocolNotSupported', error.args[0])
except RSEProtocolDomainNotSupported as error:
return generate_http_error_flask(404, 'RSEProtocolDomainNotSupported', error.args[0])
except Exception as error:
logging.exception("Internal Error")
return str(error), 500
pfns = rsemanager.lfns2pfns(rse_settings, lfns, operation=operation, scheme=scheme, domain=domain)
return jsonify(pfns)
def get(self, rse, scheme=None):
"""
Return PFNs for a set of LFNs. Formatted as a JSON object where the key is a LFN and the
value is the corresponding PFN.
.. :quickref: Attributes; Translate LFNs to PFNs.
:param rse: The RSE name.
:param scheme: The protocol identifier.
:query lfn: One or more LFN to translate.
:query scheme: Optional argument to help with the protocol selection (e.g., http / gsiftp / srm)
:query domain: Optional argument used to select the protocol for wan or lan use cases.
:query operation: Optional query argument to select the protoco for read-vs-writes.
:resheader Content-Type: application/json
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: RSE Protocol Not Supported.
:status 404: RSE Protocol Domain Not Supported.
:status 406: Not Acceptable.
:returns: A list with detailed PFN information.
"""
lfns = request.args.getlist('lfn')
lfns = list(map(lambda lfn: lfn.split(":", 1), lfns))
if any(filter(lambda info: len(info) != 2, lfns)):
invalid_lfns = ', '.join(filter(lambda info: len(info) != 2, lfns))
return generate_http_error_flask(
400, InvalidPath.__name__,
'LFN(s) in invalid format: ' + invalid_lfns)
lfns = list(map(lambda info: {
'scope': info[0],
'name': info[1]
}, lfns))
scheme = request.args.get('scheme', default=None)
domain = request.args.get('domain', default='wan')
operation = request.args.get('operation', default='write')
try:
rse_settings = get_rse_protocols(
rse,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'))
except (RSENotFound, RSEProtocolNotSupported,
RSEProtocolDomainNotSupported) as error:
return generate_http_error_flask(404, error)
pfns = rsemanager.lfns2pfns(rse_settings,
lfns,
operation=operation,
scheme=scheme,
domain=domain)
if not pfns:
return generate_http_error_flask(404, ReplicaNotFound.__name__,
'No replicas found')
return jsonify(pfns)
def GET(self, rse, scheme=None):
"""
Return PFNs for a set of LFNs. Formatted as a JSON object where the key is a LFN and the
value is the corresponding PFN.
- One or more LFN should be passed as the LFN arguments.
- A URL scheme (e.g., http / gsiftp / srm) can be passed to help with protocol selection using the
`scheme` query argument.
- The `domain` query argument is used to select protocol for wan or lan use cases.
- The `operation` query argument is used to select the protocol for read-vs-writes.
The `scheme`, `domain`, and `operation` options help with the selection of the protocol, in case
if that affects the possible PFN generation.
HTTP Success:
200 OK
HTTP Error:
400 LFN parameter(s) malformed
404 Resource not Found
500 InternalError
:returns: A list with detailed PFN information.
"""
header('Content-Type', 'application/json')
lfns = []
scheme = None
domain = 'wan'
operation = 'write'
if ctx.query:
params = parse_qsl(ctx.query[1:])
for key, val in params:
if key == 'lfn':
info = val.split(":", 1)
if len(info) != 2:
raise generate_http_error(400, 'InvalidPath',
'LFN in invalid format')
lfn_dict = {'scope': info[0], 'name': info[1]}
lfns.append(lfn_dict)
elif key == 'scheme':
scheme = val
elif key == 'domain':
domain = val
elif key == 'operation':
operation = val
rse_settings = None
try:
rse_settings = get_rse_protocols(rse, issuer=ctx.env.get('issuer'))
except RSENotFound, error:
raise generate_http_error(404, 'RSENotFound', error[0][0])
def GET(self, rse, scheme):
""" List all references of the provided RSE for the given protocol.
HTTP Success:
200 OK
HTTP Error:
404 Resource not Found
500 InternalError
:returns: A list with detailed protocol information.
"""
header('Content-Type', 'application/json')
p_list = None
try:
p_list = get_rse_protocols(rse, issuer=ctx.env.get('issuer'))
except RSENotFound, error:
raise generate_http_error(404, 'RSENotFound', error[0][0])
def GET(self, rse, scheme):
""" List all references of the provided RSE for the given protocol.
HTTP Success:
200 OK
HTTP Error:
404 Resource not Found
500 InternalError
:returns: A list with detailed protocol information.
"""
header('Content-Type', 'application/json')
p_list = None
try:
p_list = get_rse_protocols(rse, issuer=ctx.env.get('issuer'))
except RSENotFound, e:
raise generate_http_error(404, 'RSENotFound', e[0][0])
def GET(self, rse):
""" List all supported protocols of the given RSE.
HTTP Success:
200 OK
HTTP Error:
404 Resource not Found
500 InternalError
:returns: A list containing all supported protocols and all their attributes.
"""
header('Content-Type', 'application/json')
p_list = None
try:
p_list = get_rse_protocols(rse, issuer=ctx.env.get('issuer'))
except RSEOperationNotSupported, e:
raise generate_http_error(404, 'RSEOperationNotSupported', e[0][0])
def GET(self, rse):
""" List all supported protocols of the given RSE.
HTTP Success:
200 OK
HTTP Error:
404 Resource not Found
500 InternalError
:returns: A list containing all supported protocols and all their attributes.
"""
header('Content-Type', 'application/json')
p_list = None
try:
p_list = get_rse_protocols(rse, issuer=ctx.env.get('issuer'))
except RSEOperationNotSupported, error:
raise generate_http_error(404, 'RSEOperationNotSupported', error[0][0])
def get(self, rse):
""" List all supported protocols of the given RSE.
.. :quickref: Protocols; List all RSE protocols.
:param rse: The RSE name.
:resheader Content-Type: application/json
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE Operation Not Supported.
:status 404: RSE Not Found.
:status 404: RSE Protocol Domain Not Supported.
:status 404: RSE Protocol Not Supported.
:status 500: Internal Error.
:returns: A list containing all supported protocols and all their attributes.
"""
p_list = None
try:
p_list = get_rse_protocols(rse,
issuer=request.environ.get('issuer'))
except RSEOperationNotSupported, error:
return generate_http_error_flask(404, 'RSEOperationNotSupported',
error.args[0])
def test_api_rse(self):
""" RSE (API): Test external representation of RSEs """
out = api_rse.get_rse(self.rse_name, **self.vo)
assert out['rse'] == self.rse_name
assert out['id'] == self.rse_id
out = api_rse.list_rses(**self.new_vo)
out = list(out)
assert 0 != len(out)
rse_ids = [rse['id'] for rse in out]
assert self.rse3_id in rse_ids
assert self.rse4_id in rse_ids
for rse in out:
assert 'rse' in rse
if rse['id'] == self.rse3_id:
assert rse['rse'] == self.rse3_name
elif rse['id'] == self.rse4_id:
assert rse['rse'] == self.rse4_name
key = "KEY_" + generate_uuid()
api_rse.add_rse_attribute(self.rse_name,
key,
1,
issuer='root',
**self.vo)
out = api_rse.get_rses_with_attribute(key)
out = list(out)
assert 0 != len(out)
for rse in out:
assert rse['rse'] == self.rse_name
out = api_rse.get_rse_protocols(self.rse_name,
issuer='root',
**self.vo)
assert out['rse'] == self.rse_name
# add some account and RSE counters
rse_mock = 'MOCK4'
rse_mock_id = get_rse_id(rse_mock, **self.vo)
account_counter.del_counter(rse_id=rse_mock_id, account=self.account)
account_counter.add_counter(rse_id=rse_mock_id, account=self.account)
account_counter.increase(rse_id=rse_mock_id,
account=self.account,
files=1,
bytes=10)
account_counter.update_account_counter(self.account, rse_mock_id)
did = 'file_' + generate_uuid()
add_did(self.scope_name,
did,
'DATASET',
'root',
account=self.account_name,
rse=rse_mock,
**self.vo)
abacus_rse.run(once=True)
out = api_rse.get_rse_usage(rse_mock,
per_account=True,
issuer='root',
**self.vo)
assert rse_mock_id in [o['rse_id'] for o in out]
for usage in out:
if usage['rse_id'] == rse_mock_id:
assert usage['rse'] == rse_mock
accounts = [u['account'] for u in usage['account_usages']]
assert self.account_name in accounts
if self.multi_vo:
assert self.account.internal not in accounts
# clean up files
cleaner.run(once=True)
if self.multi_vo:
reaper.run(once=True,
include_rses='vo=%s&(%s)' % (self.vo['vo'], rse_mock),
greedy=True)
else:
reaper.run(once=True, include_rses=rse_mock, greedy=True)
abacus_rse.run(once=True)
out = api_rse.parse_rse_expression(
'%s|%s' % (self.rse_name, self.rse2_name), **self.vo)
assert self.rse_name in out
assert self.rse2_name in out
assert self.rse_id not in out
assert self.rse2_id not in out