def test_list_requests(self):
""" REQUEST (CORE): list requests """
models.Request(state=constants.RequestState.WAITING, source_rse_id=self.source_rse_id, dest_rse_id=self.dest_rse_id).save(session=self.db_session)
models.Request(state=constants.RequestState.SUBMITTED, source_rse_id=self.source_rse_id2, dest_rse_id=self.dest_rse_id).save(session=self.db_session)
models.Request(state=constants.RequestState.SUBMITTED, source_rse_id=self.source_rse_id, dest_rse_id=self.dest_rse_id2).save(session=self.db_session)
models.Request(state=constants.RequestState.SUBMITTED, source_rse_id=self.source_rse_id, dest_rse_id=self.dest_rse_id).save(session=self.db_session)
models.Request(state=constants.RequestState.SUBMITTED, source_rse_id=self.source_rse_id, dest_rse_id=self.dest_rse_id).save(session=self.db_session)
requests = [request for request in list_requests([self.source_rse_id], [self.dest_rse_id], [constants.RequestState.SUBMITTED], session=self.db_session)]
assert len(requests) == 2
requests = [request for request in list_requests([self.source_rse_id, self.source_rse_id2], [self.dest_rse_id], [constants.RequestState.SUBMITTED], session=self.db_session)]
assert len(requests) == 3
requests = [request for request in list_requests([self.source_rse_id], [self.dest_rse_id], [constants.RequestState.QUEUED], session=self.db_session)]
assert len(requests) == 0
def test_throttler_fifo_release_nothing(self):
""" THROTTLER (CLIENTS): throttler release nothing (fifo). """
# two waiting requests and one active requests but threshold is 1
# more than 80% of the transfer limit are already used -> release nothing
set('throttler', '%s,%s' % (self.user_activity, self.dest_rse), 1, session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.user_activity, state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
name1 = generate_uuid()
name2 = generate_uuid()
add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2020),
'name': name2,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
request = get_request_by_did(self.scope, name1, self.dest_rse_id)
assert_equal(request['state'], constants.RequestState.WAITING)
request2 = get_request_by_did(self.scope, name2, self.dest_rse_id)
assert_equal(request2['state'], constants.RequestState.WAITING)
def __init__(
self,
scope: "InternalScope",
name: str,
dest_rse_id: str,
account: "InternalAccount",
db_session: "Session",
setup_func: "Optional[Callable]" = None,
teardown_func: "Optional[Callable]" = None,
):
self.db_session = db_session
self.setup = setup_func
self.teardown = teardown_func
self.db_object = models.Request(
state=RequestState.PREPARING,
scope=scope,
name=name,
dest_rse_id=dest_rse_id,
account=account,
)
def __init__(
self,
scope,
name,
dest_rse_id,
account,
db_session,
setup_func=None,
teardown_func=None,
):
self.db_session = db_session
self.setup = setup_func
self.teardown = teardown_func
self.db_object = models.Request(
state=RequestState.PREPARING,
scope=scope,
name=name,
dest_rse_id=dest_rse_id,
account=account,
)
def mock_request(db_session, vo, source_rse, dest_rse, file):
account = InternalAccount('root', vo=vo)
add_replicas(rse_id=source_rse['id'],
files=[file],
account=account,
session=db_session)
request = models.Request(state=RequestState.PREPARING,
scope=file['scope'],
name=file['name'],
dest_rse_id=dest_rse['id'],
account=account)
request.save(session=db_session)
db_session.commit()
yield request
request.delete(session=db_session)
delete_replicas(rse_id=source_rse['id'], files=[file], session=db_session)
db_session.commit()
def test_throttler_grouped_fifo_nothing(self):
""" THROTTLER (CLIENTS): throttler release nothing (grouped fifo). """
# four waiting requests and one active requests but threshold is 1
# more than 80% of the transfer limit are already used -> release nothing
set('throttler', '%s,%s' % (self.all_activities, self.dest_rse), 1, session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.user_activity, state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session)
attach_dids(self.scope, dataset_1_name, [{'name': name1, 'scope': self.scope}], self.account, session=self.db_session)
attach_dids(self.scope, dataset_1_name, [{'name': name2, 'scope': self.scope}], self.account, session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'bytes': 1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'bytes': 2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'bytes': 3,
'requested_at': datetime.now().replace(year=2021), # requested after the request below but small enough for max_volume check
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'bytes': 3000,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3000,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id)
assert_equal(request_1['state'], constants.RequestState.WAITING)
request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id)
assert_equal(request_2['state'], constants.RequestState.WAITING)
request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id)
assert_equal(request_3['state'], constants.RequestState.WAITING)
request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id)
assert_equal(request_4['state'], constants.RequestState.WAITING)
def test_list_requests(vo, rest_client, auth_token):
""" REQUEST (REST): list requests """
source_rse = 'MOCK'
source_rse_id = get_rse_id(source_rse, vo=vo)
source_rse2 = 'MOCK2'
source_rse_id2 = get_rse_id(source_rse2, vo=vo)
source_rse3 = 'MOCK5'
source_rse_id3 = get_rse_id(source_rse3, vo=vo)
dest_rse = 'MOCK3'
dest_rse_id = get_rse_id(dest_rse, vo=vo)
dest_rse2 = 'MOCK4'
dest_rse_id2 = get_rse_id(dest_rse2, vo=vo)
db_session = session.get_session()
source_site = 'SITE1'
source_site2 = 'SITE2'
dst_site = 'SITE3'
dst_site2 = 'SITE4'
add_rse_attribute(source_rse_id, 'site', source_site)
add_rse_attribute(source_rse_id2, 'site', source_site2)
add_rse_attribute(source_rse_id3, 'site', source_site)
add_rse_attribute(dest_rse_id, 'site', dst_site)
add_rse_attribute(dest_rse_id2, 'site', dst_site2)
db_session.query(models.Source).delete()
db_session.query(models.Request).delete()
db_session.commit()
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
models.Request(state=constants.RequestState.WAITING, source_rse_id=source_rse_id, dest_rse_id=dest_rse_id, name=name3).save(session=db_session)
models.Request(state=constants.RequestState.SUBMITTED, source_rse_id=source_rse_id2, dest_rse_id=dest_rse_id, name=name1).save(session=db_session)
models.Request(state=constants.RequestState.SUBMITTED, source_rse_id=source_rse_id, dest_rse_id=dest_rse_id2, name=name1).save(session=db_session)
models.Request(state=constants.RequestState.SUBMITTED, source_rse_id=source_rse_id, dest_rse_id=dest_rse_id, name=name1).save(session=db_session)
models.Request(state=constants.RequestState.SUBMITTED, source_rse_id=source_rse_id, dest_rse_id=dest_rse_id, name=name2).save(session=db_session)
models.Request(state=constants.RequestState.SUBMITTED, source_rse_id=source_rse_id3, dest_rse_id=dest_rse_id, name=name2).save(session=db_session)
db_session.commit()
def check_correct_api(params, expected_requests):
headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'}
response = rest_client.get('/requests/list', query_string=params, headers=headers(auth(auth_token), vohdr(vo), hdrdict(headers_dict)))
assert response.status_code == 200
requests = set()
for request in response.get_data(as_text=True).split('\n')[:-1]:
request = parse_response(request)
requests.add((request['state'], request['source_rse_id'], request['dest_rse_id'], request['name']))
assert requests == expected_requests
def check_error_api(params, exception_class, exception_message, code):
headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'}
response = rest_client.get('/requests/list', query_string=params, headers=headers(auth(auth_token), vohdr(vo), hdrdict(headers_dict)))
assert response.status_code == code
body = parse_response(response.get_data(as_text=True))
assert body['ExceptionClass'] == exception_class
assert body['ExceptionMessage'] == exception_message
params = {'src_rse': source_rse, 'dst_rse': dest_rse, 'request_states': 'S'}
expected_requests = set()
expected_requests.add(('SUBMITTED', source_rse_id, dest_rse_id, name1))
expected_requests.add(('SUBMITTED', source_rse_id, dest_rse_id, name2))
check_correct_api(params, expected_requests)
params = {'src_rse': source_rse, 'dst_rse': dest_rse, 'request_states': 'Q'}
expected_requests = set([])
check_correct_api(params, expected_requests)
params = {'src_rse': source_rse2, 'dst_rse': dest_rse, 'request_states': 'S'}
expected_requests = set()
expected_requests.add(('SUBMITTED', source_rse_id2, dest_rse_id, name1))
check_correct_api(params, expected_requests)
params = {'src_rse': source_rse, 'dst_rse': dest_rse2, 'request_states': 'S'}
expected_requests = set()
expected_requests.add(('SUBMITTED', source_rse_id, dest_rse_id2, name1))
check_correct_api(params, expected_requests)
params = {'src_site': source_site, 'dst_site': dst_site, 'request_states': 'S'}
expected_requests = set()
expected_requests.add(('SUBMITTED', source_rse_id, dest_rse_id, name1))
expected_requests.add(('SUBMITTED', source_rse_id, dest_rse_id, name2))
# check correct resolution of site attribute to multiple RSE
expected_requests.add(('SUBMITTED', source_rse_id3, dest_rse_id, name2))
check_correct_api(params, expected_requests)
params = {'src_site': source_site, 'dst_site': dst_site, 'request_states': 'S,W,Q'}
expected_requests = set()
expected_requests.add(('SUBMITTED', source_rse_id, dest_rse_id, name1))
expected_requests.add(('SUBMITTED', source_rse_id, dest_rse_id, name2))
expected_requests.add(('WAITING', source_rse_id, dest_rse_id, name3))
expected_requests.add(('SUBMITTED', source_rse_id3, dest_rse_id, name2))
check_correct_api(params, expected_requests)
params = {'src_site': source_site2, 'dst_site': dst_site, 'request_states': 'S'}
expected_requests = set()
expected_requests.add(('SUBMITTED', source_rse_id2, dest_rse_id, name1))
check_correct_api(params, expected_requests)
params = {'src_site': source_site, 'dst_site': dst_site2, 'request_states': 'S'}
expected_requests = set()
expected_requests.add(('SUBMITTED', source_rse_id, dest_rse_id2, name1))
check_correct_api(params, expected_requests)
params = {'src_site': source_site, 'dst_site': dst_site2, 'request_states': 'S,W,Q'}
expected_requests = set()
expected_requests.add(('SUBMITTED', source_rse_id, dest_rse_id2, name1))
check_correct_api(params, expected_requests)
params = {}
check_error_api(params, 'MissingParameter', 'Request state is missing', 400)
params = {'request_states': 'unkown', 'dst_rse': dest_rse, 'src_rse': source_rse}
check_error_api(params, 'Invalid', 'Request state value is invalid', 400)
params = {'request_states': 'S', 'src_rse': source_rse}
check_error_api(params, 'MissingParameter', 'Destination RSE is missing', 400)
params = {'request_states': 'S', 'dst_rse': source_rse}
check_error_api(params, 'MissingParameter', 'Source RSE is missing', 400)
params = {'request_states': 'S', 'src_rse': source_rse, 'dst_site': 'SITE'}
check_error_api(params, 'MissingParameter', 'Destination RSE is missing', 400)
params = {'request_states': 'S', 'src_site': source_site}
check_error_api(params, 'MissingParameter', 'Destination site is missing', 400)
params = {'request_states': 'S', 'dst_site': dst_site}
check_error_api(params, 'MissingParameter', 'Source site is missing', 400)
params = {'request_states': 'S', 'src_site': source_site, 'dst_site': 'unknown'}
check_error_api(params, 'NotFound', 'Could not resolve site name unknown to RSE', 404)
def test_release_waiting_requests_per_free_volume(self):
""" REQUEST (CORE): release waiting requests that fit grouped in available volume."""
if self.dialect == 'mysql':
return True
# release unattached requests that fit in available volume with respect to already submitted transfers
name1 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
name2 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
name3 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id,
bytes=2,
activity=self.all_activities,
state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
volume = 10
set_rse_transfer_limits(self.dest_rse_id,
'all_activities',
volume=volume,
max_transfers=1,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'User Subscription',
'bytes': 8,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'requested_at': datetime.now().replace(year=2020),
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': 'User Subscription',
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': 'User Subscription',
'bytes': 10,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_per_free_volume(self.dest_rse_id,
volume=volume,
session=self.db_session)
# released because small enough
request = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# still waiting because requested later and to big
request = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
# still waiting because too big
request = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
# release attached requests that fit together with the dataset in available volume with respect to already submitted transfers
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
name2 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
name3 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
name4 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
dataset1_name = generate_uuid()
add_did(self.scope,
dataset1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset1_name, [{
'name': name1,
'scope': self.scope
}, {
'name': name4,
'scope': self.scope
}],
self.account,
session=self.db_session)
dataset2_name = generate_uuid()
add_did(self.scope,
dataset2_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset2_name, [{
'name': name2,
'scope': self.scope
}, {
'name': name3,
'scope': self.scope
}],
self.account,
session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id,
bytes=2,
activity=self.all_activities,
state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
volume = 10
set_rse_transfer_limits(self.dest_rse_id,
'all_activities',
volume=volume,
max_transfers=1,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'User Subscription',
'bytes': 6,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'requested_at': datetime.now().replace(year=2020),
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': 'User Subscription',
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': 'User Subscription',
'bytes': 10,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2030),
'attributes': {
'activity': 'User Subscription',
'bytes': 2,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_per_free_volume(self.dest_rse_id,
volume=volume,
session=self.db_session)
# released because dataset fits in volume
request = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
request = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# waiting because dataset is too big
request = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
request = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
# release requests with no available volume -> release nothing
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
add_replica(self.dest_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
volume = 0
set_rse_transfer_limits(self.dest_rse_id,
'all_activities',
volume=volume,
max_transfers=1,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'User Subscription',
'bytes': 8,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_per_free_volume(self.dest_rse_id,
volume=volume,
session=self.db_session)
# waiting because no available volume
request = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
def test_release_waiting_requests_grouped_fifo(self):
""" REQUEST (CORE): release waiting requests based on grouped FIFO. """
if self.dialect == 'mysql':
return True
# set max_volume to 0 to check first without releasing extra requests
set_rse_transfer_limits(self.dest_rse_id,
self.all_activities,
volume=0,
max_transfers=1,
session=self.db_session)
# one request with an unattached DID -> one request should be released
self.db_session.query(models.Request).delete()
self.db_session.commit()
name = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name,
1,
self.account,
session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
request = get_request_by_did(self.scope,
name,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# one request with an attached DID -> one request should be released
self.db_session.query(models.Request).delete()
self.db_session.commit()
name = generate_uuid()
dataset_name = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name,
1,
self.account,
session=self.db_session)
add_did(self.scope,
dataset_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_name, [{
'name': name,
'scope': self.scope
}],
self.account,
session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name,
'rule_id': generate_uuid(),
'retry_count': 1,
'scope': self.scope,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
request = get_request_by_did(self.scope,
name,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# five requests with different requested_at and multiple attachments per collection -> release only one request -> two requests of one collection should be released
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
name5 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope,
dataset_1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
dataset_2_name = generate_uuid()
add_did(self.scope,
dataset_2_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name5,
1,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name1,
'scope': self.scope
}, {
'name': name2,
'scope': self.scope
}],
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_2_name, [{
'name': name3,
'scope': self.scope
}, {
'name': name4,
'scope': self.scope
}],
self.account,
session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'requested_at': datetime.now().replace(year=2015),
'retry_count': 1,
'scope': self.scope,
'rule_id': generate_uuid(),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'requested_at': datetime.now().replace(year=2010),
'retry_count': 1,
'scope': self.scope,
'rule_id': generate_uuid(),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name5,
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'scope': self.scope,
'rule_id': generate_uuid(),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
request_1 = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
request_2 = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
request_3 = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_3['state'], constants.RequestState.WAITING)
request_4 = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_4['state'], constants.RequestState.WAITING)
request_5 = get_request_by_did(self.scope,
name5,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_5['state'], constants.RequestState.WAITING)
# with maximal volume check -> release one request -> three requests should be released because of attachments and free volume space
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope,
dataset_1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name1,
'scope': self.scope
}],
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name2,
'scope': self.scope
}],
self.account,
session=self.db_session)
set_rse_transfer_limits(self.dest_rse_id,
self.all_activities,
volume=10,
max_transfers=1,
session=self.db_session)
requests = [
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'bytes': 1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
},
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'bytes': 2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 2,
'md5': '',
'adler32': ''
}
},
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'bytes': 3,
'requested_at': datetime.now().replace(
year=2021
), # requested after the request below but small enough for max_volume check
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3,
'md5': '',
'adler32': ''
}
},
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'bytes': 3000,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3000,
'md5': '',
'adler32': ''
}
}
]
queue_requests(requests, session=self.db_session)
amount_updated_requests = release_waiting_requests_grouped_fifo(
self.dest_rse_id, count=1, session=self.db_session)
assert_equal(amount_updated_requests, 3)
# released because it got requested first
request_1 = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
# released because the DID is attached to the same dataset
request_2 = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
# released because of available volume
request_3 = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_3['state'], constants.RequestState.QUEUED)
# still waiting because there is no free volume
request_4 = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_4['state'], constants.RequestState.WAITING)
# with maximal volume check -> release one request -> two requests should be released because of attachments
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope,
dataset_1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name1,
'scope': self.scope
}],
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name2,
'scope': self.scope
}],
self.account,
session=self.db_session)
set_rse_transfer_limits(self.dest_rse_id,
self.all_activities,
volume=5,
max_transfers=1,
session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id,
bytes=2,
activity=self.all_activities,
state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'bytes': 1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'bytes': 2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'bytes': 1,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'bytes': 1,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
# released because it got requested first
request_1 = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
# released because the DID is attached to the same dataset
request_2 = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
# still waiting because there is no free volume after releasing the two requests above
request_3 = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_3['state'], constants.RequestState.WAITING)
request_4 = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_4['state'], constants.RequestState.WAITING)
def test_throttler_fifo_release_all(self):
""" THROTTLER (CLIENTS): throttler release all waiting requests (fifo). """
# no threshold -> release all waiting requests
name1 = generate_uuid()
name2 = generate_uuid()
add_replica(self.source_rse,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse,
self.scope,
name2,
1,
self.account,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2020),
'name': name2,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
request = get_request_by_did(self.scope, name1, self.dest_rse)
assert_equal(request['state'], constants.RequestState.QUEUED)
request2 = get_request_by_did(self.scope, name2, self.dest_rse)
assert_equal(request2['state'], constants.RequestState.QUEUED)
# active transfers + waiting requests are less than the threshold -> release all waiting requests
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
add_replica(self.source_rse,
self.scope,
name1,
1,
self.account,
session=self.db_session)
set('throttler',
'%s,%s' % (self.user_activity, self.dest_rse),
3,
session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id,
activity=self.user_activity,
state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
request = get_request_by_did(self.scope, name1, self.dest_rse)
assert_equal(request['state'], constants.RequestState.QUEUED)
