def setUpClass(cls):
cls.db_session = session.get_session()
cls.dest_rse = 'MOCK'
cls.source_rse = 'MOCK4'
cls.dest_rse_id = get_rse_id(cls.dest_rse)
cls.source_rse_id = get_rse_id(cls.source_rse)
cls.scope = InternalScope('mock')
cls.account = InternalAccount('root')
cls.user_activity = 'User Subscription'
cls.all_activities = 'all_activities'
set_rse_transfer_limits(cls.dest_rse_id, cls.user_activity, max_transfers=1, session=cls.db_session)
set('throttler_release_strategy', 'dest_%s' % cls.dest_rse_id, 'fifo', session=cls.db_session)
def dest_throttler(db_session, mock_request):
config.set('throttler', 'mode', 'DEST_PER_ACT', session=db_session)
set_rse_transfer_limits(mock_request.dest_rse_id,
activity=mock_request.activity,
max_transfers=1,
strategy='fifo',
session=db_session)
db_session.commit()
yield
db_session.query(models.RSETransferLimit).filter_by(
rse_id=mock_request.dest_rse_id).delete()
config.remove_option('throttler', 'mode', session=db_session)
db_session.commit()
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': 'tst'}
else:
cls.vo = {}
cls.db_session = session.get_session()
cls.dest_rse = 'MOCK'
cls.source_rse = 'MOCK4'
cls.dest_rse_id = get_rse_id(cls.dest_rse, **cls.vo)
cls.source_rse_id = get_rse_id(cls.source_rse, **cls.vo)
cls.scope = InternalScope('mock', **cls.vo)
cls.account = InternalAccount('root', **cls.vo)
cls.user_activity = 'User Subscription'
cls.all_activities = 'all_activities'
set_rse_transfer_limits(cls.dest_rse_id, cls.user_activity, max_transfers=1, session=cls.db_session)
set('throttler_release_strategy', 'dest_%s' % cls.dest_rse_id, 'fifo', session=cls.db_session)
def setUp(self):
self.db_session.query(models.Request).delete()
self.db_session.commit()
# set transfer limits to put requests in waiting state
set_rse_transfer_limits(self.dest_rse_id, self.user_activity, max_transfers=1, session=self.db_session)
set_rse_transfer_limits(self.dest_rse_id, self.all_activities, max_transfers=1, session=self.db_session)
set_rse_transfer_limits(self.dest_rse_id, 'ignore', max_transfers=1, session=self.db_session)
def __release_per_activity(stats, direction, rse_name, rse_id, logger, session):
"""
Release requests per activity.
:param stats: Request statistics
:param direction: String whether request statistics are based on source or destination RSEs.
:param rse_name: RSE name.
:param rse_id: RSE id.
"""
for activity in stats['activities']:
threshold = stats['activities'][activity]['threshold']
transfer = stats['activities'][activity]['transfer']
waiting = stats['activities'][activity]['waiting']
if waiting:
logger(logging.DEBUG, "Request status for %s at %s: %s" % (activity, rse_name,
stats['activities'][activity]))
if threshold is None:
logger(logging.DEBUG, "Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse_id %s" % (threshold, activity, rse_id))
delete_rse_transfer_limits(rse_id, activity=activity, session=session)
release_all_waiting_requests(rse_id, activity=activity, direction=direction, session=session)
record_counter('daemons.conveyor.throttler.delete_rse_transfer_limits.{activity}.{rse}', labels={'activity': activity, 'rse': rse_name})
elif transfer + waiting > threshold:
logger(logging.DEBUG, "Throttler set limits for activity %s, rse %s" % (activity, rse_name))
set_rse_transfer_limits(rse_id, activity=activity, max_transfers=threshold, transfers=transfer, waitings=waiting, session=session)
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', threshold, labels={'activity': activity, 'rse': rse_name, 'limit_attr': 'max_transfers'})
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', transfer, labels={'activity': activity, 'rse': rse_name, 'limit_attr': 'transfers'})
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', waiting, labels={'activity': activity, 'rse': rse_name, 'limit_attr': 'waiting'})
if transfer < 0.8 * threshold:
# release requests on account
nr_accounts = len(stats['activities'][activity]['accounts'])
if nr_accounts < 1:
nr_accounts = 1
to_release = threshold - transfer
threshold_per_account = math.ceil(threshold / nr_accounts)
to_release_per_account = math.ceil(to_release / nr_accounts)
accounts = stats['activities'][activity]['accounts']
for account in accounts:
if nr_accounts == 1:
logger(logging.DEBUG, "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (to_release, activity, rse_name, account))
release_waiting_requests_fifo(rse_id, activity=activity, account=account, count=to_release, direction=direction, session=session)
record_gauge('daemons.conveyor.throttler.release_waiting_requests.{activity}.{rse}.{account}', to_release, labels={'activity': activity, 'rse': rse_name, 'account': account})
elif accounts[account]['transfer'] > threshold_per_account:
logger(logging.DEBUG, "Throttler will not release %s waiting requests for activity %s, rse %s, account %s: It queued more transfers than its share " %
(accounts[account]['waiting'], activity, rse_name, account))
nr_accounts -= 1
to_release_per_account = math.ceil(to_release / nr_accounts)
elif accounts[account]['waiting'] < to_release_per_account:
logger(logging.DEBUG, "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (accounts[account]['waiting'], activity, rse_name, account))
release_waiting_requests_fifo(rse_id, activity=activity, account=account, count=accounts[account]['waiting'], direction=direction, session=session)
record_gauge('daemons.conveyor.throttler.release_waiting_requests.{activity}.{rse}.{account}', accounts[account]['waiting'], labels={'activity': activity, 'rse': rse_name, 'account': account})
to_release = to_release - accounts[account]['waiting']
nr_accounts -= 1
to_release_per_account = math.ceil(to_release / nr_accounts)
else:
logger(logging.DEBUG, "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (to_release_per_account, activity, rse_name, account))
release_waiting_requests_fifo(rse_id, activity=activity, account=account, count=to_release_per_account, direction=direction, session=session)
record_gauge('daemons.conveyor.throttler.release_waiting_requests.{activity}.{rse}.{account}', to_release_per_account, labels={'activity': activity, 'rse': rse_name, 'account': account})
to_release = to_release - to_release_per_account
nr_accounts -= 1
else:
logger(logging.DEBUG, "Throttler has done nothing for activity %s on rse %s (transfer > 0.8 * threshold)" % (activity, rse_name))
elif waiting > 0:
logger(logging.DEBUG, "Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse %s" % (threshold, activity, rse_name))
delete_rse_transfer_limits(rse_id, activity=activity, session=session)
release_all_waiting_requests(rse_id, activity=activity, direction=direction, session=session)
record_counter('daemons.conveyor.throttler.delete_rse_transfer_limits.{activity}.{rse}', labels={'activity': activity, 'rse': rse_name})
def import_data(data, session=None):
"""
Import data to add and update records in Rucio.
:param data: data to be imported as dictionary.
:param session: database session in use.
"""
# RSEs
rses = data.get('rses')
if rses:
for rse in rses:
protocols = rse.get('protocols')
if protocols:
protocols = protocols.get('protocols')
del rse['protocols']
rse_name = rse['rse']
del rse['rse']
if not rse_module.rse_exists(rse_name, session=session):
rse_module.add_rse(rse_name,
deterministic=rse.get('deterministic'),
volatile=rse.get('volatile'),
city=rse.get('city'),
region_code=rse.get('region_code'),
country_name=rse.get('country_name'),
staging_area=rse.get('staging_area'),
continent=rse.get('continent'),
time_zone=rse.get('time_zone'),
ISP=rse.get('ISP'),
rse_type=rse.get('rse_type'),
latitude=rse.get('latitude'),
longitude=rse.get('longitude'),
ASN=rse.get('ASN'),
availability=rse.get('availability'),
session=session)
else:
rse_module.update_rse(rse_name, rse, session=session)
# Protocols
if protocols:
old_protocols = rse_module.get_rse_protocols(rse=rse_name,
session=session)
for protocol in protocols:
scheme = protocol.get('scheme')
hostname = protocol.get('hostname')
port = protocol.get('port')
intersection = [
old_protocol
for old_protocol in old_protocols['protocols']
if old_protocol['scheme'] == scheme
and old_protocol['hostname'] == hostname
and old_protocol['port'] == port
]
if intersection:
del protocol['scheme']
del protocol['hostname']
del protocol['port']
rse_module.update_protocols(rse=rse_name,
scheme=scheme,
data=protocol,
hostname=hostname,
port=port,
session=session)
else:
rse_module.add_protocol(rse=rse_name,
parameter=protocol,
session=session)
# Limits
limits = rse.get('limits')
if limits:
old_limits = rse_module.get_rse_limits(rse=rse_name,
session=session)
for limit in limits:
if limit in old_limits:
rse_module.delete_rse_limit(rse=rse_name,
name=limit,
session=session)
rse_module.set_rse_limits(rse=rse_name,
name=limit,
value=limits[limit],
session=session)
# Transfer limits
transfer_limits = rse.get('transfer_limits')
if transfer_limits:
for limit in transfer_limits:
old_transfer_limits = rse_module.get_rse_transfer_limits(
rse=rse_name, activity=limit, session=session)
if limit in old_transfer_limits:
rse_module.delete_rse_transfer_limits(rse=rse_name,
activity=limit,
session=session)
max_transfers = transfer_limits[limit].items(
)[0][1]['max_transfers']
rse_module.set_rse_transfer_limits(
rse=rse_name,
activity=limit,
max_transfers=max_transfers,
session=session)
# Attributes
attributes = rse.get('attributes')
if attributes:
old_attributes = rse_module.list_rse_attributes(
rse=rse_name, session=session)
for attr in attributes:
if attr in old_attributes:
rse_module.del_rse_attribute(rse=rse_name,
key=attr,
session=session)
rse_module.add_rse_attribute(rse=rse_name,
key=attr,
value=attributes[attr],
session=session)
# Distances
distances = data.get('distances')
if distances:
for src_rse_name in distances:
src = rse_module.get_rse_id(src_rse_name, session=session)
for dest_rse_name in distances[src_rse_name]:
dest = rse_module.get_rse_id(dest_rse_name, session=session)
distance = distances[src_rse_name][dest_rse_name]
del distance['src_rse_id']
del distance['dest_rse_id']
old_distance = distance_module.get_distances(src_rse_id=src,
dest_rse_id=dest,
session=session)
if old_distance:
distance_module.update_distances(src_rse_id=src,
dest_rse_id=dest,
parameters=distance,
session=session)
else:
distance_module.add_distance(
src_rse_id=src,
dest_rse_id=dest,
ranking=distance.get('ranking'),
agis_distance=distance.get('agis_distance'),
geoip_distance=distance.get('geoip_distance'),
active=distance.get('active'),
submitted=distance.get('submitted'),
transfer_speed=distance.get('transfer_speed'),
finished=distance.get('finished'),
failed=distance.get('failed'),
session=session)
def test_throttler_grouped_fifo_subset(self):
""" THROTTLER (CLIENTS): throttler release subset of waiting requests (grouped fifo). """
set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=10, max_transfers=1, session=self.db_session)
set('throttler', '%s,%s' % (self.all_activities, self.dest_rse), 1, session=self.db_session) # threshold used by throttler
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session)
attach_dids(self.scope, dataset_1_name, [{'name': name1, 'scope': self.scope}], self.account, session=self.db_session)
attach_dids(self.scope, dataset_1_name, [{'name': name2, 'scope': self.scope}], self.account, session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'bytes': 1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'bytes': 2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'bytes': 3,
'requested_at': datetime.now().replace(year=2021), # requested after the request below but small enough for max_volume check
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'bytes': 3000,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3000,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
self.db_session.commit()
throttler.run(once=True, sleep_time=1)
# released because it got requested first
request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
# released because the DID is attached to the same dataset
request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
# released because of available volume
request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id)
assert_equal(request_3['state'], constants.RequestState.QUEUED)
# still waiting because there is no free volume
request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id)
assert_equal(request_4['state'], constants.RequestState.WAITING)
def test_queue_requests_state(vo, use_preparer):
""" REQUEST (CORE): test queuing requests """
if use_preparer == 'preparer enabled':
use_preparer = True
elif use_preparer == 'preparer disabled':
use_preparer = False
else:
return pytest.xfail(reason=f'unknown test parameter use_preparer={use_preparer}')
db_session = session.get_session()
dest_rse = 'MOCK'
dest_rse2 = 'MOCK2'
source_rse = 'MOCK4'
source_rse2 = 'MOCK5'
dest_rse_id = get_rse_id(dest_rse, vo=vo)
dest_rse_id2 = get_rse_id(dest_rse2, vo=vo)
source_rse_id = get_rse_id(source_rse, vo=vo)
source_rse_id2 = get_rse_id(source_rse2, vo=vo)
scope = InternalScope('mock', vo=vo)
account = InternalAccount('root', vo=vo)
user_activity = 'User Subscription'
config_set('conveyor', 'use_preparer', str(use_preparer))
target_state = RequestState.PREPARING if use_preparer else RequestState.QUEUED
name = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
add_replica(source_rse_id, scope, name, 1, account, session=db_session)
add_replica(source_rse_id2, scope, name2, 1, account, session=db_session)
add_replica(source_rse_id, scope, name3, 1, account, session=db_session)
set_rse_transfer_limits(dest_rse_id, user_activity, max_transfers=1, session=db_session)
set_rse_transfer_limits(dest_rse_id2, user_activity, max_transfers=1, session=db_session)
set_rse_transfer_limits(source_rse_id, user_activity, max_transfers=1, session=db_session)
set_rse_transfer_limits(source_rse_id2, user_activity, max_transfers=1, session=db_session)
requests = [{
'dest_rse_id': dest_rse_id,
'src_rse_id': source_rse_id,
'request_type': RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name,
'scope': scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': user_activity,
'bytes': 10,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': dest_rse_id,
'src_rse_id': source_rse_id2,
'request_type': RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'scope': scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'unknown',
'bytes': 10,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': dest_rse_id2,
'src_rse_id': source_rse_id,
'request_type': RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'scope': scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': user_activity,
'bytes': 10,
'md5': '',
'adler32': ''
}
}]
try:
queue_requests(requests, session=db_session)
request = get_request_by_did(scope, name, dest_rse_id, session=db_session)
assert request['state'] == target_state
request = get_request_by_did(scope, name2, dest_rse_id, session=db_session)
assert request['state'] == target_state
request = get_request_by_did(scope, name3, dest_rse_id2, session=db_session)
assert request['state'] == target_state
finally:
config_remove_option('conveyor', 'use_preparer')
db_session.query(models.Source).delete()
db_session.query(models.Request).delete()
db_session.query(models.RSETransferLimit).delete()
db_session.query(models.Distance).delete()
db_session.commit()
reset_config_table()
def __schedule_requests():
"""
Schedule requests
"""
try:
logging.info("Throttler retrieve requests statistics")
results = get_stats_by_activity_dest_state(state=[
RequestState.QUEUED, RequestState.SUBMITTING,
RequestState.SUBMITTED, RequestState.WAITING
])
result_dict = {}
for activity, dest_rse_id, account, state, rse, counter in results:
threshold = get_config_limit(activity, dest_rse_id)
if threshold or (counter and (state == RequestState.WAITING)):
if activity not in result_dict:
result_dict[activity] = {}
if dest_rse_id not in result_dict[activity]:
result_dict[activity][dest_rse_id] = {
'waiting': 0,
'transfer': 0,
'threshold': threshold,
'accounts': {},
'rse': rse
}
if account not in result_dict[activity][dest_rse_id][
'accounts']:
result_dict[activity][dest_rse_id]['accounts'][account] = {
'waiting': 0,
'transfer': 0
}
if state == RequestState.WAITING:
result_dict[activity][dest_rse_id]['accounts'][account][
'waiting'] += counter
result_dict[activity][dest_rse_id]['waiting'] += counter
else:
result_dict[activity][dest_rse_id]['accounts'][account][
'transfer'] += counter
result_dict[activity][dest_rse_id]['transfer'] += counter
for activity in result_dict:
for dest_rse_id in result_dict[activity]:
threshold = result_dict[activity][dest_rse_id]['threshold']
transfer = result_dict[activity][dest_rse_id]['transfer']
waiting = result_dict[activity][dest_rse_id]['waiting']
rse_name = result_dict[activity][dest_rse_id]['rse']
if waiting:
logging.debug("Request status for %s at %s: %s" %
(activity, rse_name,
result_dict[activity][dest_rse_id]))
if threshold is None:
logging.debug(
"Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse_id %s"
% (threshold, activity, dest_rse_id))
delete_rse_transfer_limits(rse=None,
activity=activity,
rse_id=dest_rse_id)
release_waiting_requests(rse=None,
activity=activity,
rse_id=dest_rse_id)
record_counter(
'daemons.conveyor.throttler.delete_rse_transfer_limits.%s.%s'
% (activity, rse_name))
elif transfer + waiting > threshold:
logging.debug(
"Throttler set limits for activity %s, rse %s" %
(activity, rse_name))
set_rse_transfer_limits(rse=None,
activity=activity,
rse_id=dest_rse_id,
max_transfers=threshold,
transfers=transfer,
waitings=waiting)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.max_transfers'
% (activity, rse_name), threshold)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.transfers'
% (activity, rse_name), transfer)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.waitings'
% (activity, rse_name), waiting)
if transfer < 0.8 * threshold:
# release requests on account
nr_accounts = len(
result_dict[activity][dest_rse_id]['accounts'])
if nr_accounts < 1:
nr_accounts = 1
to_release = threshold - transfer
threshold_per_account = math.ceil(threshold /
nr_accounts)
to_release_per_account = math.ceil(to_release /
nr_accounts)
accounts = result_dict[activity][dest_rse_id][
'accounts']
for account in accounts:
if nr_accounts == 1:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
%
(to_release, activity, rse_name, account))
release_waiting_requests(rse=None,
activity=activity,
rse_id=dest_rse_id,
account=account,
count=to_release)
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
to_release)
elif accounts[account][
'transfer'] > threshold_per_account:
logging.debug(
"Throttler will not release %s waiting requests for activity %s, rse %s, account %s: It queued more transfers than its share "
% (accounts[account]['waiting'], activity,
rse_name, account))
nr_accounts -= 1
to_release_per_account = math.ceil(to_release /
nr_accounts)
elif accounts[account][
'waiting'] < to_release_per_account:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
% (accounts[account]['waiting'], activity,
rse_name, account))
release_waiting_requests(
rse=None,
activity=activity,
rse_id=dest_rse_id,
account=account,
count=accounts[account]['waiting'])
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
accounts[account]['waiting'])
to_release = to_release - accounts[account][
'waiting']
nr_accounts -= 1
to_release_per_account = math.ceil(to_release /
nr_accounts)
else:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
% (to_release_per_account, activity,
rse_name, account))
release_waiting_requests(
rse=None,
activity=activity,
rse_id=dest_rse_id,
account=account,
count=to_release_per_account)
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
to_release_per_account)
to_release = to_release - to_release_per_account
nr_accounts -= 1
else:
logging.debug(
"Throttler has done nothing for activity %s on rse %s (transfer > 0.8 * threshold)"
% (activity, rse_name))
elif waiting > 0:
logging.debug(
"Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse %s"
% (threshold, activity, rse_name))
delete_rse_transfer_limits(rse=None,
activity=activity,
rse_id=dest_rse_id)
release_waiting_requests(rse=None,
activity=activity,
rse_id=dest_rse_id)
record_counter(
'daemons.conveyor.throttler.delete_rse_transfer_limits.%s.%s'
% (activity, rse_name))
except:
logging.critical("Failed to schedule requests, error: %s" %
(traceback.format_exc()))
def test_release_waiting_requests_per_free_volume(self):
""" REQUEST (CORE): release waiting requests that fit grouped in available volume."""
if self.dialect == 'mysql':
return True
# release unattached requests that fit in available volume with respect to already submitted transfers
name1 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
name2 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
name3 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id,
bytes=2,
activity=self.all_activities,
state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
volume = 10
set_rse_transfer_limits(self.dest_rse_id,
'all_activities',
volume=volume,
max_transfers=1,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'User Subscription',
'bytes': 8,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'requested_at': datetime.now().replace(year=2020),
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': 'User Subscription',
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': 'User Subscription',
'bytes': 10,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_per_free_volume(self.dest_rse_id,
volume=volume,
session=self.db_session)
# released because small enough
request = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# still waiting because requested later and to big
request = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
# still waiting because too big
request = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
# release attached requests that fit together with the dataset in available volume with respect to already submitted transfers
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
name2 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
name3 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
name4 = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
dataset1_name = generate_uuid()
add_did(self.scope,
dataset1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset1_name, [{
'name': name1,
'scope': self.scope
}, {
'name': name4,
'scope': self.scope
}],
self.account,
session=self.db_session)
dataset2_name = generate_uuid()
add_did(self.scope,
dataset2_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset2_name, [{
'name': name2,
'scope': self.scope
}, {
'name': name3,
'scope': self.scope
}],
self.account,
session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id,
bytes=2,
activity=self.all_activities,
state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
volume = 10
set_rse_transfer_limits(self.dest_rse_id,
'all_activities',
volume=volume,
max_transfers=1,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'User Subscription',
'bytes': 6,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'requested_at': datetime.now().replace(year=2020),
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': 'User Subscription',
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': 'User Subscription',
'bytes': 10,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2030),
'attributes': {
'activity': 'User Subscription',
'bytes': 2,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_per_free_volume(self.dest_rse_id,
volume=volume,
session=self.db_session)
# released because dataset fits in volume
request = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
request = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# waiting because dataset is too big
request = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
request = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
# release requests with no available volume -> release nothing
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
add_replica(self.dest_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
volume = 0
set_rse_transfer_limits(self.dest_rse_id,
'all_activities',
volume=volume,
max_transfers=1,
session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2015),
'attributes': {
'activity': 'User Subscription',
'bytes': 8,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_per_free_volume(self.dest_rse_id,
volume=volume,
session=self.db_session)
# waiting because no available volume
request = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.WAITING)
def test_release_waiting_requests_grouped_fifo(self):
""" REQUEST (CORE): release waiting requests based on grouped FIFO. """
if self.dialect == 'mysql':
return True
# set max_volume to 0 to check first without releasing extra requests
set_rse_transfer_limits(self.dest_rse_id,
self.all_activities,
volume=0,
max_transfers=1,
session=self.db_session)
# one request with an unattached DID -> one request should be released
self.db_session.query(models.Request).delete()
self.db_session.commit()
name = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name,
1,
self.account,
session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
request = get_request_by_did(self.scope,
name,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# one request with an attached DID -> one request should be released
self.db_session.query(models.Request).delete()
self.db_session.commit()
name = generate_uuid()
dataset_name = generate_uuid()
add_replica(self.source_rse_id,
self.scope,
name,
1,
self.account,
session=self.db_session)
add_did(self.scope,
dataset_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_name, [{
'name': name,
'scope': self.scope
}],
self.account,
session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name,
'rule_id': generate_uuid(),
'retry_count': 1,
'scope': self.scope,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
request = get_request_by_did(self.scope,
name,
self.dest_rse_id,
session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
# five requests with different requested_at and multiple attachments per collection -> release only one request -> two requests of one collection should be released
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
name5 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope,
dataset_1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
dataset_2_name = generate_uuid()
add_did(self.scope,
dataset_2_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name5,
1,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name1,
'scope': self.scope
}, {
'name': name2,
'scope': self.scope
}],
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_2_name, [{
'name': name3,
'scope': self.scope
}, {
'name': name4,
'scope': self.scope
}],
self.account,
session=self.db_session)
requests = [{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'requested_at': datetime.now().replace(year=2015),
'retry_count': 1,
'scope': self.scope,
'rule_id': generate_uuid(),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'requested_at': datetime.now().replace(year=2010),
'retry_count': 1,
'scope': self.scope,
'rule_id': generate_uuid(),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name5,
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'scope': self.scope,
'rule_id': generate_uuid(),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
request_1 = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
request_2 = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
request_3 = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_3['state'], constants.RequestState.WAITING)
request_4 = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_4['state'], constants.RequestState.WAITING)
request_5 = get_request_by_did(self.scope,
name5,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_5['state'], constants.RequestState.WAITING)
# with maximal volume check -> release one request -> three requests should be released because of attachments and free volume space
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope,
dataset_1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name1,
'scope': self.scope
}],
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name2,
'scope': self.scope
}],
self.account,
session=self.db_session)
set_rse_transfer_limits(self.dest_rse_id,
self.all_activities,
volume=10,
max_transfers=1,
session=self.db_session)
requests = [
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'bytes': 1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
},
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'bytes': 2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 2,
'md5': '',
'adler32': ''
}
},
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'bytes': 3,
'requested_at': datetime.now().replace(
year=2021
), # requested after the request below but small enough for max_volume check
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3,
'md5': '',
'adler32': ''
}
},
{
'source_rse_id': self.source_rse_id,
'dest_rse_id': self.dest_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'bytes': 3000,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 3000,
'md5': '',
'adler32': ''
}
}
]
queue_requests(requests, session=self.db_session)
amount_updated_requests = release_waiting_requests_grouped_fifo(
self.dest_rse_id, count=1, session=self.db_session)
assert_equal(amount_updated_requests, 3)
# released because it got requested first
request_1 = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
# released because the DID is attached to the same dataset
request_2 = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
# released because of available volume
request_3 = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_3['state'], constants.RequestState.QUEUED)
# still waiting because there is no free volume
request_4 = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_4['state'], constants.RequestState.WAITING)
# with maximal volume check -> release one request -> two requests should be released because of attachments
self.db_session.query(models.Request).delete()
self.db_session.commit()
name1 = generate_uuid()
name2 = generate_uuid()
name3 = generate_uuid()
name4 = generate_uuid()
dataset_1_name = generate_uuid()
add_did(self.scope,
dataset_1_name,
constants.DIDType.DATASET,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name1,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name2,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name3,
1,
self.account,
session=self.db_session)
add_replica(self.source_rse_id,
self.scope,
name4,
1,
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name1,
'scope': self.scope
}],
self.account,
session=self.db_session)
attach_dids(self.scope,
dataset_1_name, [{
'name': name2,
'scope': self.scope
}],
self.account,
session=self.db_session)
set_rse_transfer_limits(self.dest_rse_id,
self.all_activities,
volume=5,
max_transfers=1,
session=self.db_session)
request = models.Request(dest_rse_id=self.dest_rse_id,
bytes=2,
activity=self.all_activities,
state=constants.RequestState.SUBMITTED)
request.save(session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'bytes': 1,
'scope': self.scope,
'retry_count': 1,
'rule_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2000),
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name2,
'bytes': 2,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 2,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name3,
'bytes': 1,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name4,
'bytes': 1,
'requested_at': datetime.now().replace(year=2020),
'rule_id': generate_uuid(),
'scope': self.scope,
'retry_count': 1,
'attributes': {
'activity': self.user_activity,
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_waiting_requests_grouped_fifo(self.dest_rse_id,
count=1,
session=self.db_session)
# released because it got requested first
request_1 = get_request_by_did(self.scope,
name1,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_1['state'], constants.RequestState.QUEUED)
# released because the DID is attached to the same dataset
request_2 = get_request_by_did(self.scope,
name2,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_2['state'], constants.RequestState.QUEUED)
# still waiting because there is no free volume after releasing the two requests above
request_3 = get_request_by_did(self.scope,
name3,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_3['state'], constants.RequestState.WAITING)
request_4 = get_request_by_did(self.scope,
name4,
self.dest_rse_id,
session=self.db_session)
assert_equal(request_4['state'], constants.RequestState.WAITING)
def setup(self):
# New RSE
self.new_rse = rse_name_generator()
# RSE 1 that already exists
self.old_rse_1 = rse_name_generator()
add_rse(self.old_rse_1, availability=1)
add_protocol(
self.old_rse_1, {
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'impl': 'impl'
})
self.old_rse_id_1 = get_rse_id(self.old_rse_1)
set_rse_limits(rse=self.old_rse_1, name='limit1', value='10')
set_rse_transfer_limits(rse=self.old_rse_1,
activity='activity1',
max_transfers=10)
add_rse_attribute(rse=self.old_rse_1, key='attr1', value='test10')
# RSE 2 that already exists
self.old_rse_2 = rse_name_generator()
add_rse(self.old_rse_2)
self.old_rse_id_2 = get_rse_id(self.old_rse_2)
# RSE 3 that already exists
self.old_rse_3 = rse_name_generator()
add_rse(self.old_rse_3)
self.old_rse_id_3 = get_rse_id(self.old_rse_3)
# Distance that already exists
add_distance(self.old_rse_id_1, self.old_rse_id_2)
self.data1 = {
'rses': [{
'rse': self.new_rse,
'rse_type': 'TAPE',
'availability': 5,
'city': 'NewCity',
'protocols': {
'protocols': [{
'scheme': 'scheme',
'hostname': 'hostname',
'port': 1000,
'impl': 'impl'
}]
},
'limits': {
'limit1': 0
},
'transfer_limits': {
'activity1': {
'unknown_rse_id': {
'max_transfers': 1
}
}
},
'attributes': {
'attr1': 'test'
}
}, {
'rse': self.old_rse_1,
'protocols': {
'protocols': [{
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'prefix': 'prefix',
'impl': 'impl1'
}, {
'scheme': 'scheme2',
'hostname': 'hostname2',
'port': 1001,
'impl': 'impl'
}]
},
'limits': {
'limit1': 0,
'limit2': 2
},
'transfer_limits': {
'activity1': {
self.old_rse_id_1: {
'max_transfers': 1
}
},
'activity2': {
self.old_rse_id_1: {
'max_transfers': 2
}
}
},
'attributes': {
'attr1': 'test1',
'attr2': 'test2'
}
}],
'distances': {
self.old_rse_1: {
self.old_rse_2: {
'src_rse_id': self.old_rse_id_1,
'dest_rse_id': self.old_rse_id_2,
'ranking': 10
},
self.old_rse_3: {
'src_rse_id': self.old_rse_id_1,
'dest_rse_id': self.old_rse_id_3,
'ranking': 4
}
}
}
}
self.data2 = {'rses': [{'rse': self.new_rse}]}
self.data3 = {'distances': {}}
