def test_rses_at_different_vos(self):
""" MULTI VO (CLIENT): Test that RSEs from 2nd vo don't interfere """
# Set up RSEs at two VOs
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst = 'TST_%s' % rse_str
new = 'NEW_%s' % rse_str
shr = 'SHR_%s' % rse_str
rse_client.add_rse(tst)
rse_client.add_rse(shr)
add_rse(new, 'root', **self.new_vo)
shr_id_new_original = add_rse(shr, 'root', **self.new_vo) # Accurate rse_id for shared RSE at 'new'
# Check the cached rse-id from each VO does not interfere
shr_id_tst = get_rse_id(shr, **self.vo)
shr_id_new = get_rse_id(shr, **self.new_vo)
assert_equal(shr_id_new, shr_id_new_original)
assert_not_equal(shr_id_new, shr_id_tst)
# Check that when listing RSEs we only get RSEs for our VO
rse_list_tst = [r['rse'] for r in rse_client.list_rses()]
rse_list_new = [r['rse'] for r in list_rses(filters={}, **self.new_vo)]
assert_true(tst in rse_list_tst)
assert_false(new in rse_list_tst)
assert_true(shr in rse_list_tst)
assert_false(tst in rse_list_new)
assert_true(new in rse_list_new)
assert_true(shr in rse_list_new)
# Check the cached attribute-value results do not interfere and only give results from the appropriate VO
attribute_value = generate_uuid()
add_rse_attribute(new, 'test', attribute_value, 'root', **self.new_vo)
rses_tst_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo))
rses_new_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo))
rses_tst_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo))
rses_new_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo))
assert_equal(len(rses_tst_1), 0)
assert_not_equal(len(rses_new_1), 0)
assert_equal(len(rses_tst_2), 0)
assert_not_equal(len(rses_new_2), 0)
# check parse_expression
rses_tst_3 = parse_expression(shr, filter={'vo': self.vo['vo']})
rses_tst_4 = parse_expression(tst, filter={'vo': self.vo['vo']})
rses_new_3 = parse_expression(shr, filter={'vo': self.new_vo['vo']})
with assert_raises(InvalidRSEExpression):
parse_expression(tst, filter={'vo': self.new_vo['vo']})
assert_equal(len(rses_tst_3), 1)
assert_equal(shr_id_tst, rses_tst_3[0]['id'])
assert_equal(len(rses_tst_4), 1)
assert_equal(tst, rses_tst_4[0]['rse'])
assert_equal(len(rses_new_3), 1)
assert_equal(shr_id_new, rses_new_3[0]['id'])
def GET(self):
"""
List requests for a given source and destination RSE or site.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 Request Not Found
406 Not Acceptable
"""
header('Content-Type', 'application/x-json-stream')
params = parse_qs(ctx.query[1:])
src_rse = params.get('src_rse', [None])[0]
dst_rse = params.get('dst_rse', [None])[0]
src_site = params.get('src_site', [None])[0]
dst_site = params.get('dst_site', [None])[0]
request_states = params.get('request_states', [None])[0]
if not request_states:
raise generate_http_error(400, 'MissingParameter',
'Request state is missing')
if src_rse and not dst_rse:
raise generate_http_error(400, 'MissingParameter',
'Destination RSE is missing')
elif dst_rse and not src_rse:
raise generate_http_error(400, 'MissingParameter',
'Source RSE is missing')
elif src_site and not dst_site:
raise generate_http_error(400, 'MissingParameter',
'Destination site is missing')
elif dst_site and not src_site:
raise generate_http_error(400, 'MissingParameter',
'Source site is missing')
try:
states = [
RequestState.from_string(state)
for state in request_states.split(',')
]
except ValueError:
raise generate_http_error(400, 'Invalid',
'Request state value is invalid')
src_rses = []
dst_rses = []
if src_site:
src_rses = get_rses_with_attribute_value(key='site',
value=src_site,
lookup_key='site',
vo=ctx.env.get('vo'))
if not src_rses:
raise generate_http_error(
404, 'NotFound',
'Could not resolve site name %s to RSE' % src_site)
src_rses = [get_rse_name(rse['rse_id']) for rse in src_rses]
dst_rses = get_rses_with_attribute_value(key='site',
value=dst_site,
lookup_key='site',
vo=ctx.env.get('vo'))
if not dst_rses:
raise generate_http_error(
404, 'NotFound',
'Could not resolve site name %s to RSE' % dst_site)
dst_rses = [get_rse_name(rse['rse_id']) for rse in dst_rses]
else:
dst_rses = [dst_rse]
src_rses = [src_rse]
for result in request.list_requests(src_rses,
dst_rses,
states,
issuer=ctx.env.get('issuer'),
vo=ctx.env.get('vo')):
del result['_sa_instance_state']
yield render_json(**result) + '\n'
def get(self):
"""
List requests for a given source and destination RSE or site.
.. :quickref: RequestsGet; list requests
:reqheader Content-Type: application/x-json-stream
:status 200: Request found.
:status 404: Request not found.
:status 406: Not Acceptable.
"""
src_rse = f_request.get('src_rse')
dst_rse = f_request.get('dst_rse')
src_site = f_request.get('src_site')
dst_site = f_request.get('dst_site')
request_states = f_request.get('request_states')
if not request_states:
return generate_http_error_flask(400, 'MissingParameter',
'Request state is missing')
if src_rse and not dst_rse:
return generate_http_error_flask(400, 'MissingParameter',
'Destination RSE is missing')
elif dst_rse and not src_rse:
return generate_http_error_flask(400, 'MissingParameter',
'Source RSE is missing')
elif src_site and not dst_site:
return generate_http_error_flask(400, 'MissingParameter',
'Destination site is missing')
elif dst_site and not src_site:
return generate_http_error_flask(400, 'MissingParameter',
'Source site is missing')
try:
states = [
RequestState.from_string(state)
for state in request_states.split(',')
]
except ValueError:
return generate_http_error_flask(400, 'Invalid',
'Request state value is invalid')
src_rses = []
dst_rses = []
if src_site:
src_rses = get_rses_with_attribute_value(key='site',
value=src_site,
lookup_key='site')
if not src_rses:
return generate_http_error_flask(
404, 'NotFound',
'Could not resolve site name %s to RSE' % src_site)
src_rses = [get_rse_name(rse['rse_id']) for rse in src_rses]
dst_rses = get_rses_with_attribute_value(key='site',
value=dst_site,
lookup_key='site')
if not dst_rses:
return generate_http_error_flask(
404, 'NotFound',
'Could not resolve site name %s to RSE' % dst_site)
dst_rses = [get_rse_name(rse['rse_id']) for rse in dst_rses]
else:
dst_rses = [dst_rse]
src_rses = [src_rse]
results = []
for result in request.list_requests(
src_rses, dst_rses, states,
issuer=f_request.environ.get('issuer')):
result = result.to_dict()
del result['_sa_instance_state']
results.append(result)
return json.dumps(results, cls=APIEncoder)
def get(self):
"""
List requests for a given source and destination RSE or site.
.. :quickref: RequestsGet; list requests
:reqheader Content-Type: application/x-json-stream
:status 200: Request found.
:status 404: Request not found.
:status 406: Not Acceptable.
"""
try:
query_string = f_request.query_string.decode(encoding='utf-8')
params = parse_qs(query_string)
src_rse = params.get('src_rse', [None])[0]
dst_rse = params.get('dst_rse', [None])[0]
src_site = params.get('src_site', [None])[0]
dst_site = params.get('dst_site', [None])[0]
request_states = params.get('request_states', [None])[0]
if not request_states:
return generate_http_error_flask(400, 'MissingParameter',
'Request state is missing')
if src_rse and not dst_rse:
return generate_http_error_flask(400, 'MissingParameter',
'Destination RSE is missing')
elif dst_rse and not src_rse:
return generate_http_error_flask(400, 'MissingParameter',
'Source RSE is missing')
elif src_site and not dst_site:
return generate_http_error_flask(
400, 'MissingParameter', 'Destination site is missing')
elif dst_site and not src_site:
return generate_http_error_flask(400, 'MissingParameter',
'Source site is missing')
try:
states = [
RequestState.from_string(state)
for state in request_states.split(',')
]
except ValueError:
return generate_http_error_flask(
400, 'Invalid', 'Request state value is invalid')
src_rses = []
dst_rses = []
if src_site:
src_rses = get_rses_with_attribute_value(
key='site',
value=src_site,
lookup_key='site',
vo=f_request.environ.get('vo'))
if not src_rses:
return generate_http_error_flask(
404, 'NotFound',
'Could not resolve site name %s to RSE' % src_site)
src_rses = [get_rse_name(rse['rse_id']) for rse in src_rses]
dst_rses = get_rses_with_attribute_value(
key='site',
value=dst_site,
lookup_key='site',
vo=f_request.environ.get('vo'))
if not dst_rses:
return generate_http_error_flask(
404, 'NotFound',
'Could not resolve site name %s to RSE' % dst_site)
dst_rses = [get_rse_name(rse['rse_id']) for rse in dst_rses]
else:
dst_rses = [dst_rse]
src_rses = [src_rse]
def generate(issuer, vo):
for result in request.list_requests(src_rses,
dst_rses,
states,
issuer=issuer,
vo=vo):
del result['_sa_instance_state']
yield render_json(**result) + '\n'
return try_stream(
generate(issuer=f_request.environ.get('issuer'),
vo=f_request.environ.get('vo')))
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return str(error), 500
def get(self):
"""
---
summary: List Historic Requests
description: List historical requests for a given source and destination RSE or site.
tags:
- Requests
parameters:
- name: src_rse
in: query
description: The source rse.
schema:
type: array
items:
type: object
required:
- rse_id
properties:
rse_id:
description: The id of the rse.
type: string
- name: dest_rse
in: query
description: The destination rse.
schema:
type: array
items:
type: object
required:
- rse_id
properties:
rse_id:
description: The id of the rse.
type: string
- name: src_site
in: query
description: The source site.
schema:
type: string
- name: dest_site
in: query
description: The destination site.
schema:
type: string
- name: request_states
in: query
description: The accepted request states. Delimited by comma.
schema:
type: string
- name: offset
in: query
description: The offset of the list.
schema:
type: integer
default: 0
- name: limit
in: query
description: The maximum number of items to return.
schema:
type: integer
default: 100
responses:
200:
description: OK
content:
application/x-json-stream:
schema:
description: All requests matching the arguments. Seperated by a new line character.
type: array
items:
description: A request.
type: object
properties:
id:
description: The id of the request.
type: strig
request_type:
description: The request type.
type: string
enum: ["T", "U", "D", "I", "O"]
scope:
description: The scope of the transfer.
type: string
name:
description: The name of the transfer.
type: string
did_type:
description: The did type.
type: string
dest_rse_id:
description: The destination RSE id.
type: string
source_rse_id:
description: The source RSE id.
type: string
attributes:
description: All attributes associated with the request.
type: string
state:
description: The state of the request.
type: string
enum: ["Q", "G", "S", "F", "D", "L", "N", "O", "A", "M", "U", "W", "P"]
external_id:
description: External id of the request.
type: string
external_host:
description: External host of the request.
type: string
retry_count:
description: The numbers of attempted retires.
type: integer
err_msg:
description: An error message if one occured.
type: string
previous_attempt_id:
description: The id of the previous attempt.
type: string
rule_id:
description: The id of the associated replication rule.
type: string
activity:
description: The activity of the request.
type: string
bytes:
description: The size of the did in bytes.
type: integer
md5:
description: The md5 checksum of the did to transfer.
type: string
adler32:
description: The adler32 checksum of the did to transfer.
type: string
dest_url:
description: The destination url.
type: string
submitted_at:
description: The time the request got submitted.
type: string
started_at:
description: The time the request got started.
type: string
transferred_at:
description: The time the request got transfered.
type: string
estimated_at:
description: The time the request got estimated.
type: string
submitter_id:
description: The id of the submitter.
type: string
estimated_stated_at:
description: The estimation of the started at value.
type: string
estimated_transferred_at:
description: The estimation of the transfered at value.
type: string
staging_started_at:
description: The time the staging got started.
type: string
staging_finished_at:
description: The time the staging got finished.
type: string
account:
description: The account which issued the request.
type: string
requested_at:
description: The time the request got requested.
type: string
priority:
description: The priority of the request.
type: integer
transfertool:
description: The transfertool used.
type: string
source_rse:
description: The name of the source RSE.
type: string
dest_rse:
description: The name of the destination RSE.
type: string
401:
description: Invalid Auth Token
404:
description: Not found
406:
description: Not acceptable
"""
src_rse = flask.request.args.get('src_rse', default=None)
dst_rse = flask.request.args.get('dst_rse', default=None)
src_site = flask.request.args.get('src_site', default=None)
dst_site = flask.request.args.get('dst_site', default=None)
request_states = flask.request.args.get('request_states', default=None)
offset = flask.request.args.get('offset', default=0)
limit = flask.request.args.get('limit', default=100)
if not request_states:
return generate_http_error_flask(400, 'MissingParameter',
'Request state is missing')
if src_rse and not dst_rse:
return generate_http_error_flask(400, 'MissingParameter',
'Destination RSE is missing')
elif dst_rse and not src_rse:
return generate_http_error_flask(400, 'MissingParameter',
'Source RSE is missing')
elif src_site and not dst_site:
return generate_http_error_flask(400, 'MissingParameter',
'Destination site is missing')
elif dst_site and not src_site:
return generate_http_error_flask(400, 'MissingParameter',
'Source site is missing')
try:
states = [
RequestState(state) for state in request_states.split(',')
]
except ValueError:
return generate_http_error_flask(400, 'Invalid',
'Request state value is invalid')
src_rses = []
dst_rses = []
if src_site:
src_rses = get_rses_with_attribute_value(
key='site',
value=src_site,
lookup_key='site',
vo=flask.request.environ.get('vo'))
if not src_rses:
return generate_http_error_flask(
404, 'NotFound',
f'Could not resolve site name {src_site} to RSE')
src_rses = [get_rse_name(rse['rse_id']) for rse in src_rses]
dst_rses = get_rses_with_attribute_value(
key='site',
value=dst_site,
lookup_key='site',
vo=flask.request.environ.get('vo'))
if not dst_rses:
return generate_http_error_flask(
404, 'NotFound',
f'Could not resolve site name {dst_site} to RSE')
dst_rses = [get_rse_name(rse['rse_id']) for rse in dst_rses]
else:
dst_rses = [dst_rse]
src_rses = [src_rse]
def generate(issuer, vo):
for result in request.list_requests_history(src_rses,
dst_rses,
states,
issuer=issuer,
vo=vo,
offset=offset,
limit=limit):
del result['_sa_instance_state']
yield render_json(**result) + '\n'
return try_stream(
generate(issuer=flask.request.environ.get('issuer'),
vo=flask.request.environ.get('vo')))
def get(self):
"""
List requests for a given source and destination RSE or site.
.. :quickref: RequestsGet; list requests
:reqheader Content-Type: application/x-json-stream
:status 200: Request found.
:status 404: Request not found.
:status 406: Not Acceptable.
"""
src_rse = flask.request.args.get('src_rse', default=None)
dst_rse = flask.request.args.get('dst_rse', default=None)
src_site = flask.request.args.get('src_site', default=None)
dst_site = flask.request.args.get('dst_site', default=None)
request_states = flask.request.args.get('request_states', default=None)
if not request_states:
return generate_http_error_flask(400, 'MissingParameter',
'Request state is missing')
if src_rse and not dst_rse:
return generate_http_error_flask(400, 'MissingParameter',
'Destination RSE is missing')
elif dst_rse and not src_rse:
return generate_http_error_flask(400, 'MissingParameter',
'Source RSE is missing')
elif src_site and not dst_site:
return generate_http_error_flask(400, 'MissingParameter',
'Destination site is missing')
elif dst_site and not src_site:
return generate_http_error_flask(400, 'MissingParameter',
'Source site is missing')
try:
states = [
RequestState(state) for state in request_states.split(',')
]
except ValueError:
return generate_http_error_flask(400, 'Invalid',
'Request state value is invalid')
src_rses = []
dst_rses = []
if src_site:
src_rses = get_rses_with_attribute_value(
key='site',
value=src_site,
lookup_key='site',
vo=flask.request.environ.get('vo'))
if not src_rses:
return generate_http_error_flask(
404, 'NotFound',
f'Could not resolve site name {src_site} to RSE')
src_rses = [get_rse_name(rse['rse_id']) for rse in src_rses]
dst_rses = get_rses_with_attribute_value(
key='site',
value=dst_site,
lookup_key='site',
vo=flask.request.environ.get('vo'))
if not dst_rses:
return generate_http_error_flask(
404, 'NotFound',
f'Could not resolve site name {dst_site} to RSE')
dst_rses = [get_rse_name(rse['rse_id']) for rse in dst_rses]
else:
dst_rses = [dst_rse]
src_rses = [src_rse]
def generate(issuer, vo):
for result in request.list_requests(src_rses,
dst_rses,
states,
issuer=issuer,
vo=vo):
del result['_sa_instance_state']
yield render_json(**result) + '\n'
return try_stream(
generate(issuer=flask.request.environ.get('issuer'),
vo=flask.request.environ.get('vo')))