def test_throttler_fifo_release_nothing(self):
""" THROTTLER (CLIENTS): throttler release nothing (fifo). """
# two waiting requests and one active requests but threshold is 1
# more than 80% of the transfer limit are already used -> release nothing
set('throttler', '%s,%s' % (self.user_activity, self.dest_rse), 1, session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.user_activity, state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
name1 = generate_uuid()
name2 = generate_uuid()
add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2020),
'name': name2,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
request = get_request_by_did(self.scope, name1, self.dest_rse_id)
assert_equal(request['state'], constants.RequestState.WAITING)
request2 = get_request_by_did(self.scope, name2, self.dest_rse_id)
assert_equal(request2['state'], constants.RequestState.WAITING)
def test_throttler_fifo_release_subset(self):
""" THROTTLER (CLIENTS): throttler release subset of waiting requests (fifo). """
# two waiting requests and no active requests but threshold is 1 -> release only 1 request
set('throttler', '%s,%s' % (self.user_activity, self.dest_rse), 1, session=self.db_session)
name1 = generate_uuid()
name2 = generate_uuid()
add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2020),
'name': name2,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
request = get_request_by_did(self.scope, name1, self.dest_rse_id)
assert_equal(request['state'], constants.RequestState.QUEUED)
request2 = get_request_by_did(self.scope, name2, self.dest_rse_id)
assert_equal(request2['state'], constants.RequestState.WAITING)
def queue_requests(requests, issuer, vo='def', session=None):
"""
Submit transfer or deletion requests on destination RSEs for data identifiers.
:param requests: List of dictionaries containing 'scope', 'name', 'dest_rse_id', 'request_type', 'attributes'
:param issuer: Issuing account as a string.
:param vo: The VO to act on.
:param session: The database session in use.
:returns: List of Request-IDs as 32 character hex strings
"""
kwargs = {'requests': requests, 'issuer': issuer}
if not permission.has_permission(issuer=issuer,
vo=vo,
action='queue_requests',
kwargs=kwargs,
session=session):
raise exception.AccessDenied('%(issuer)s can not queue request' %
locals())
for req in requests:
req['scope'] = InternalScope(req['scope'], vo=vo)
if 'account' in req:
req['account'] = InternalAccount(req['account'], vo=vo)
new_requests = request.queue_requests(requests, session=session)
return [api_update_return_dict(r, session=session) for r in new_requests]
def test_release_waiting_requests_all(self):
""" REQUEST (CORE): release all waiting requests. """
name1 = generate_uuid()
name2 = generate_uuid()
add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': 'User Subscription',
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2020),
'name': name2,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': 'User Subscription',
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_all_waiting_requests(self.dest_rse_id, session=self.db_session)
request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
def queue_requests(requests, issuer):
"""
Submit transfer or deletion requests on destination RSEs for data identifiers.
:param requests: List of dictionaries containing 'scope', 'name', 'dest_rse_id', 'request_type', 'attributes'
:param issuer: Issuing account as a string.
:returns: List of Request-IDs as 32 character hex strings
"""
kwargs = {'requests': requests, 'issuer': issuer}
if not permission.has_permission(issuer=issuer, action='queue_requests', kwargs=kwargs):
raise exception.AccessDenied('%(issuer)s can not queue request' % locals())
return request.queue_requests(requests)
def queue_requests(requests, issuer):
"""
Submit transfer or deletion requests on destination RSEs for data identifiers.
:param requests: List of dictionaries containing 'scope', 'name', 'dest_rse_id', 'request_type', 'attributes'
:param issuer: Issuing account as a string.
:returns: List of Request-IDs as 32 character hex strings
"""
kwargs = {'requests': requests, 'issuer': issuer}
if not permission.has_permission(issuer=issuer, action='queue_requests', kwargs=kwargs):
raise exception.AccessDenied('%(issuer)s can not queue request' % locals())
return request.queue_requests(requests)
def __create_missing_replicas_and_requests(
transfer_path: "List[DirectTransferDefinition]",
default_tombstone_delay: int,
logger: "Callable",
session: "Optional[Session]" = None) -> "Tuple[bool, bool]":
"""
Create replicas and requests in the database for the intermediate hops
"""
initial_request_id = transfer_path[-1].rws.request_id
creation_successful = True
must_skip_submission = False
# Iterate the path in reverse order. The last hop is the initial request, so
# next_hop.rws.request_id will always be initialized when handling the current hop.
for i in reversed(range(len(transfer_path))):
hop = transfer_path[i]
rws = hop.rws
if rws.request_id:
continue
tombstone_delay = rws.dest_rse.attributes.get(
'multihop_tombstone_delay', default_tombstone_delay)
try:
tombstone = tombstone_from_delay(tombstone_delay)
except ValueError:
logger(logging.ERROR,
"%s: Cannot parse multihop tombstone delay %s",
initial_request_id, tombstone_delay)
creation_successful = False
break
files = [{
'scope': rws.scope,
'name': rws.name,
'bytes': rws.byte_count,
'adler32': rws.adler32,
'md5': rws.md5,
'tombstone': tombstone,
'state': 'C'
}]
try:
add_replicas(rse_id=rws.dest_rse.id,
files=files,
account=rws.account,
ignore_availability=False,
dataset_meta=None,
session=session)
# Set replica state to Copying in case replica already existed in another state.
# Can happen when a multihop transfer failed previously, and we are re-scheduling it now.
update_replica_state(rse_id=rws.dest_rse.id,
scope=rws.scope,
name=rws.name,
state=ReplicaState.COPYING,
session=session)
except Exception as error:
logger(logging.ERROR, '%s: Problem adding replicas on %s : %s',
initial_request_id, rws.dest_rse, str(error))
rws.attributes['is_intermediate_hop'] = True
# next_hop_request_id and initial_request_id are not used anymore in rucio >=1.28, but are needed
# for running at the same time 1.27 and 1.28 on the same database.
# TODO: remove following two rows
rws.attributes['next_hop_request_id'] = transfer_path[i +
1].rws.request_id
rws.attributes['initial_request_id'] = initial_request_id
rws.attributes['source_replica_expression'] = hop.src.rse.name
req_to_queue = {
'dest_rse_id': rws.dest_rse.id,
'state': RequestState.QUEUED,
'scope': rws.scope,
'name': rws.name,
'rule_id':
'00000000000000000000000000000000', # Dummy Rule ID used for multihop. TODO: Replace with actual rule_id once we can flag intermediate requests
'attributes': rws.attributes,
'request_type': rws.request_type,
'retry_count': rws.retry_count,
'account': rws.account,
'requested_at': datetime.datetime.now()
}
if rws.transfertool:
req_to_queue['transfertool'] = rws.transfertool
new_req = queue_requests(requests=[req_to_queue], session=session)
# If a request already exists, new_req will be an empty list.
if new_req:
db_req = new_req[0]
logger(
logging.DEBUG,
'%s: New request created for the transfer between %s and %s : %s',
initial_request_id, transfer_path[0].src,
transfer_path[-1].dst, db_req['id'])
else:
db_req = request_core.get_request_by_did(rws.scope,
rws.name,
rws.dest_rse.id,
session=session)
# A transfer already exists for part of the path. Just construct the remaining
# path, but don't submit the transfer. We must wait for the existing transfer to be
# completed before continuing.
must_skip_submission = True
logger(logging.DEBUG,
'%s: Reusing intermediate hop between %s and %s : %s',
initial_request_id, transfer_path[0].src,
transfer_path[-1].dst, db_req['id'])
models.TransferHop(
request_id=db_req['id'],
next_hop_request_id=transfer_path[i + 1].rws.request_id,
initial_request_id=initial_request_id,
).save(session=session, flush=False)
rws.request_id = db_req['id']
rws.requested_at = db_req['requested_at']
return creation_successful, must_skip_submission
def test_throttler_grouped_fifo_all(self):
""" THROTTLER (CLIENTS): throttler release all waiting requests (grouped fifo). """
# no threshold -> release all waiting requests
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session)
attach_dids(self.scope, dataset_1_name, [{'name': name1, 'scope': self.scope}], self.account, session=self.db_session)
attach_dids(self.scope, dataset_1_name, [{'name': name2, 'scope': self.scope}], self.account, session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'bytes': 1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'bytes': 2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'bytes': 3,
'requested_at': datetime.now().replace(year=2021), # requested after the request below but small enough for max_volume check
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'bytes': 3000,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3000,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id)
assert_equal(request_3['state'], constants.RequestState.QUEUED)
request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id)
assert_equal(request_4['state'], constants.RequestState.QUEUED)
def test_throttler_grouped_fifo_subset(self):
""" THROTTLER (CLIENTS): throttler release subset of waiting requests (grouped fifo). """
set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=10, max_transfers=1, session=self.db_session)
set('throttler', '%s,%s' % (self.all_activities, self.dest_rse), 1, session=self.db_session) # threshold used by throttler
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session)
attach_dids(self.scope, dataset_1_name, [{'name': name1, 'scope': self.scope}], self.account, session=self.db_session)
attach_dids(self.scope, dataset_1_name, [{'name': name2, 'scope': self.scope}], self.account, session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'bytes': 1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'bytes': 2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'bytes': 3,
'requested_at': datetime.now().replace(year=2021), # requested after the request below but small enough for max_volume check
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'bytes': 3000,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3000,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
# released because it got requested first
request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
# released because the DID is attached to the same dataset
request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
# released because of available volume
request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id)
assert_equal(request_3['state'], constants.RequestState.QUEUED)
# still waiting because there is no free volume
request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id)
assert_equal(request_4['state'], constants.RequestState.WAITING)
def test_throttler_grouped_fifo_nothing(self):
""" THROTTLER (CLIENTS): throttler release nothing (grouped fifo). """
# four waiting requests and one active requests but threshold is 1
# more than 80% of the transfer limit are already used -> release nothing
set('throttler', '%s,%s' % (self.all_activities, self.dest_rse), 1, session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.user_activity, state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session)
attach_dids(self.scope, dataset_1_name, [{'name': name1, 'scope': self.scope}], self.account, session=self.db_session)
attach_dids(self.scope, dataset_1_name, [{'name': name2, 'scope': self.scope}], self.account, session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'bytes': 1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'bytes': 2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'bytes': 3,
'requested_at': datetime.now().replace(year=2021), # requested after the request below but small enough for max_volume check
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'bytes': 3000,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3000,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id)
assert_equal(request_1['state'], constants.RequestState.WAITING)
request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id)
assert_equal(request_2['state'], constants.RequestState.WAITING)
request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id)
assert_equal(request_3['state'], constants.RequestState.WAITING)
request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id)
assert_equal(request_4['state'], constants.RequestState.WAITING)
def test_queue_requests_state(vo, use_preparer):
""" REQUEST (CORE): test queuing requests """
if use_preparer == 'preparer enabled':
use_preparer = True
elif use_preparer == 'preparer disabled':
use_preparer = False
else:
return pytest.xfail(reason=f'unknown test parameter use_preparer={use_preparer}')
db_session = session.get_session()
dest_rse = 'MOCK'
dest_rse2 = 'MOCK2'
source_rse = 'MOCK4'
source_rse2 = 'MOCK5'
dest_rse_id = get_rse_id(dest_rse, vo=vo)
dest_rse_id2 = get_rse_id(dest_rse2, vo=vo)
source_rse_id = get_rse_id(source_rse, vo=vo)
source_rse_id2 = get_rse_id(source_rse2, vo=vo)
scope = InternalScope('mock', vo=vo)
account = InternalAccount('root', vo=vo)
user_activity = 'User Subscription'
config_set('conveyor', 'use_preparer', str(use_preparer))
target_state = RequestState.PREPARING if use_preparer else RequestState.QUEUED
name = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
add_replica(source_rse_id, scope, name, 1, account, session=db_session)
add_replica(source_rse_id2, scope, name2, 1, account, session=db_session)
add_replica(source_rse_id, scope, name3, 1, account, session=db_session)
set_rse_transfer_limits(dest_rse_id, user_activity, max_transfers=1, session=db_session)
set_rse_transfer_limits(dest_rse_id2, user_activity, max_transfers=1, session=db_session)
set_rse_transfer_limits(source_rse_id, user_activity, max_transfers=1, session=db_session)
set_rse_transfer_limits(source_rse_id2, user_activity, max_transfers=1, session=db_session)
requests = [{
'dest_rse_id': dest_rse_id,
'src_rse_id': source_rse_id,
'request_type': RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name,
'scope': scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': user_activity,
'bytes': 10,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': dest_rse_id,
'src_rse_id': source_rse_id2,
'request_type': RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'scope': scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'unknown',
'bytes': 10,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': dest_rse_id2,
'src_rse_id': source_rse_id,
'request_type': RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'scope': scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': user_activity,
'bytes': 10,
'md5': '',
'adler32': ''
}
}]
try:
queue_requests(requests, session=db_session)
request = get_request_by_did(scope, name, dest_rse_id, session=db_session)
assert request['state'] == target_state
request = get_request_by_did(scope, name2, dest_rse_id, session=db_session)
assert request['state'] == target_state
request = get_request_by_did(scope, name3, dest_rse_id2, session=db_session)
assert request['state'] == target_state
finally:
config_remove_option('conveyor', 'use_preparer')
db_session.query(models.Source).delete()
db_session.query(models.Request).delete()
db_session.query(models.RSETransferLimit).delete()
db_session.query(models.Distance).delete()
db_session.commit()
reset_config_table()
def test_release_waiting_requests_fifo(self):
""" REQUEST (CORE): release waiting requests based on FIFO. """
# without account and activity check
# two requests -> release one request -> request with oldest requested_at date should be released
name1 = generate_uuid()
name2 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2020),
'name': name2,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
request = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
request2 = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request2['state'], constants.RequestState.WAITING)
# with activity and account check
# two requests -> release two request -> requests with correct account and activity should be released
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
requests = [
{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'account': self.account,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
},
{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2020),
'name': name2,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'account': self.account,
'attributes': {
'activity': 'ignore',
'bytes': 1,
'md5': '',
'adler32': ''
}
},
{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2020),
'name': name3,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'account': InternalAccount('jdoe'),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
},
{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(
year=2020
), # requested latest but account and activity are correct
'name': name4,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'account': self.account,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}
]
queue_requests(requests, session=self.db_session)
release_waiting_requests_fifo(self.dest_rse_id,
count=2,
account=self.account,
activity=self.user_activity,
session=self.db_session)
request = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
request = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
request = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
request = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
def test_release_waiting_requests_per_free_volume(self):
""" REQUEST (CORE): release waiting requests that fit grouped in available volume."""
if self.dialect == 'mysql':
return True
# release unattached requests that fit in available volume with respect to already submitted transfers
name1 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
name2 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
name3 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id,
bytes=2,
activity=self.all_activities,
state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
volume = 10
set_rse_transfer_limits(self.dest_rse_id,
'all_activities',
volume=volume,
max_transfers=1,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'User Subscription',
'bytes': 8,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'requested_at': datetime.now().replace(year=2020),
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': 'User Subscription',
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': 'User Subscription',
'bytes': 10,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_per_free_volume(self.dest_rse_id,
volume=volume,
session=self.db_session)
# released because small enough
request = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# still waiting because requested later and to big
request = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
# still waiting because too big
request = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
# release attached requests that fit together with the dataset in available volume with respect to already submitted transfers
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
name2 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
name3 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
name4 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
dataset1_name = generate_uuid()
add_did(self.scope,
dataset1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset1_name, [{
'name': name1,
'scope': self.scope
}, {
'name': name4,
'scope': self.scope
}],
self.account,
session=self.db_session)
dataset2_name = generate_uuid()
add_did(self.scope,
dataset2_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset2_name, [{
'name': name2,
'scope': self.scope
}, {
'name': name3,
'scope': self.scope
}],
self.account,
session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id,
bytes=2,
activity=self.all_activities,
state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
volume = 10
set_rse_transfer_limits(self.dest_rse_id,
'all_activities',
volume=volume,
max_transfers=1,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'User Subscription',
'bytes': 6,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'requested_at': datetime.now().replace(year=2020),
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': 'User Subscription',
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': 'User Subscription',
'bytes': 10,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2030),
'attributes': {
'activity': 'User Subscription',
'bytes': 2,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_per_free_volume(self.dest_rse_id,
volume=volume,
session=self.db_session)
# released because dataset fits in volume
request = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
request = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# waiting because dataset is too big
request = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
request = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
# release requests with no available volume -> release nothing
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
add_replica(self.dest_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
volume = 0
set_rse_transfer_limits(self.dest_rse_id,
'all_activities',
volume=volume,
max_transfers=1,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'User Subscription',
'bytes': 8,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_per_free_volume(self.dest_rse_id,
volume=volume,
session=self.db_session)
# waiting because no available volume
request = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
def test_release_waiting_requests_grouped_fifo(self):
""" REQUEST (CORE): release waiting requests based on grouped FIFO. """
if self.dialect == 'mysql':
return True
# set max_volume to 0 to check first without releasing extra requests
set_rse_transfer_limits(self.dest_rse_id,
self.all_activities,
volume=0,
max_transfers=1,
session=self.db_session)
# one request with an unattached DID -> one request should be released
self.db_session.query(models.Request).delete()
self.db_session.commit()
name = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name,
1,
self.account,
session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
request = get_request_by_did(self.scope,
name,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# one request with an attached DID -> one request should be released
self.db_session.query(models.Request).delete()
self.db_session.commit()
name = generate_uuid()
dataset_name = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name,
1,
self.account,
session=self.db_session)
add_did(self.scope,
dataset_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_name, [{
'name': name,
'scope': self.scope
}],
self.account,
session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name,
'rule_id': generate_uuid(),
'retry_count': 1,
'scope': self.scope,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
request = get_request_by_did(self.scope,
name,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# five requests with different requested_at and multiple attachments per collection -> release only one request -> two requests of one collection should be released
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
name5 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope,
dataset_1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
dataset_2_name = generate_uuid()
add_did(self.scope,
dataset_2_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name5,
1,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name1,
'scope': self.scope
}, {
'name': name2,
'scope': self.scope
}],
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_2_name, [{
'name': name3,
'scope': self.scope
}, {
'name': name4,
'scope': self.scope
}],
self.account,
session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'requested_at': datetime.now().replace(year=2015),
'retry_count': 1,
'scope': self.scope,
'rule_id': generate_uuid(),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'requested_at': datetime.now().replace(year=2010),
'retry_count': 1,
'scope': self.scope,
'rule_id': generate_uuid(),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name5,
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'scope': self.scope,
'rule_id': generate_uuid(),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
request_1 = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
request_2 = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
request_3 = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_3['state'], constants.RequestState.WAITING)
request_4 = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_4['state'], constants.RequestState.WAITING)
request_5 = get_request_by_did(self.scope,
name5,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_5['state'], constants.RequestState.WAITING)
# with maximal volume check -> release one request -> three requests should be released because of attachments and free volume space
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope,
dataset_1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name1,
'scope': self.scope
}],
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name2,
'scope': self.scope
}],
self.account,
session=self.db_session)
set_rse_transfer_limits(self.dest_rse_id,
self.all_activities,
volume=10,
max_transfers=1,
session=self.db_session)
requests = [
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'bytes': 1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
},
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'bytes': 2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 2,
'md5': '',
'adler32': ''
}
},
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'bytes': 3,
'requested_at': datetime.now().replace(
year=2021
), # requested after the request below but small enough for max_volume check
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3,
'md5': '',
'adler32': ''
}
},
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'bytes': 3000,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3000,
'md5': '',
'adler32': ''
}
}
]
queue_requests(requests, session=self.db_session)
amount_updated_requests = release_waiting_requests_grouped_fifo(
self.dest_rse_id, count=1, session=self.db_session)
assert_equal(amount_updated_requests, 3)
# released because it got requested first
request_1 = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
# released because the DID is attached to the same dataset
request_2 = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
# released because of available volume
request_3 = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_3['state'], constants.RequestState.QUEUED)
# still waiting because there is no free volume
request_4 = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_4['state'], constants.RequestState.WAITING)
# with maximal volume check -> release one request -> two requests should be released because of attachments
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope,
dataset_1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name1,
'scope': self.scope
}],
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name2,
'scope': self.scope
}],
self.account,
session=self.db_session)
set_rse_transfer_limits(self.dest_rse_id,
self.all_activities,
volume=5,
max_transfers=1,
session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id,
bytes=2,
activity=self.all_activities,
state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'bytes': 1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'bytes': 2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'bytes': 1,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'bytes': 1,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
# released because it got requested first
request_1 = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
# released because the DID is attached to the same dataset
request_2 = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
# still waiting because there is no free volume after releasing the two requests above
request_3 = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_3['state'], constants.RequestState.WAITING)
request_4 = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_4['state'], constants.RequestState.WAITING)
def test_throttler_fifo_release_all(self):
""" THROTTLER (CLIENTS): throttler release all waiting requests (fifo). """
# no threshold -> release all waiting requests
name1 = generate_uuid()
name2 = generate_uuid()
add_replica(self.source_rse,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse,
self.scope,
name2,
1,
self.account,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2020),
'name': name2,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
request = get_request_by_did(self.scope, name1, self.dest_rse)
assert_equal(request['state'], constants.RequestState.QUEUED)
request2 = get_request_by_did(self.scope, name2, self.dest_rse)
assert_equal(request2['state'], constants.RequestState.QUEUED)
# active transfers + waiting requests are less than the threshold -> release all waiting requests
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
add_replica(self.source_rse,
self.scope,
name1,
1,
self.account,
session=self.db_session)
set('throttler',
'%s,%s' % (self.user_activity, self.dest_rse),
3,
session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id,
activity=self.user_activity,
state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'account': self.account,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
request = get_request_by_did(self.scope, name1, self.dest_rse)
assert_equal(request['state'], constants.RequestState.QUEUED)
