def __release_all_activities(stats, direction, rse_name, rse_id, logger, session):
"""
Release requests if activities should be ignored.
:param stats: Request statistics
:param direction: String whether request statistics are based on source or destination RSEs.
:param rse_name: RSE name.
:param rse_id: RSE id.
"""
threshold = stats['threshold']
transfer = stats['transfer']
waiting = stats['waiting']
strategy = stats['strategy']
if threshold is not None and transfer + waiting > threshold:
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', threshold, labels={'activity': 'all_activities', 'rse': rse_name, 'limit_attr': 'max_transfers'})
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', transfer, labels={'activity': 'all_activities', 'rse': rse_name, 'limit_attr': 'transfers'})
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', waiting, labels={'activity': 'all_activities', 'rse': rse_name, 'limit_attr': 'waiting'})
if transfer < 0.8 * threshold:
to_be_released = threshold - transfer
if strategy == 'grouped_fifo':
deadline = stats.get('deadline')
volume = stats.get('volume')
release_waiting_requests_grouped_fifo(rse_id, count=to_be_released, direction=direction, volume=volume, deadline=deadline, session=session)
elif strategy == 'fifo':
release_waiting_requests_fifo(rse_id, count=to_be_released, direction=direction, session=session)
else:
logger(logging.DEBUG, "Throttler has done nothing on rse %s (transfer > 0.8 * threshold)" % rse_name)
elif waiting > 0 or not threshold:
logger(logging.DEBUG, "Throttler remove limits(threshold: %s) and release all waiting requests, rse %s" % (threshold, rse_name))
delete_rse_transfer_limits(rse_id, activity='all_activities', session=session)
release_all_waiting_requests(rse_id, direction=direction, session=session)
record_counter('daemons.conveyor.throttler.delete_rse_transfer_limits.{activity}.{rse}', labels={'activity': 'all_activities', 'rse': rse_name})
def test_release_waiting_requests_all(self):
""" REQUEST (CORE): release all waiting requests. """
name1 = generate_uuid()
name2 = generate_uuid()
add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session)
add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session)
requests = [{
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': name1,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now().replace(year=2018),
'attributes': {
'activity': 'User Subscription',
'bytes': 1,
'md5': '',
'adler32': ''
}
}, {
'dest_rse_id': self.dest_rse_id,
'source_rse_id': self.source_rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'requested_at': datetime.now().replace(year=2020),
'name': name2,
'scope': self.scope,
'rule_id': generate_uuid(),
'retry_count': 1,
'attributes': {
'activity': 'User Subscription',
'bytes': 1,
'md5': '',
'adler32': ''
}
}]
queue_requests(requests, session=self.db_session)
release_all_waiting_requests(self.dest_rse_id, session=self.db_session)
request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session)
assert_equal(request['state'], constants.RequestState.QUEUED)
def __release_per_activity(stats, direction, rse_name, rse_id, logger, session):
"""
Release requests per activity.
:param stats: Request statistics
:param direction: String whether request statistics are based on source or destination RSEs.
:param rse_name: RSE name.
:param rse_id: RSE id.
"""
for activity in stats['activities']:
threshold = stats['activities'][activity]['threshold']
transfer = stats['activities'][activity]['transfer']
waiting = stats['activities'][activity]['waiting']
if waiting:
logger(logging.DEBUG, "Request status for %s at %s: %s" % (activity, rse_name,
stats['activities'][activity]))
if threshold is None:
logger(logging.DEBUG, "Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse_id %s" % (threshold, activity, rse_id))
delete_rse_transfer_limits(rse_id, activity=activity, session=session)
release_all_waiting_requests(rse_id, activity=activity, direction=direction, session=session)
record_counter('daemons.conveyor.throttler.delete_rse_transfer_limits.{activity}.{rse}', labels={'activity': activity, 'rse': rse_name})
elif transfer + waiting > threshold:
logger(logging.DEBUG, "Throttler set limits for activity %s, rse %s" % (activity, rse_name))
set_rse_transfer_limits(rse_id, activity=activity, max_transfers=threshold, transfers=transfer, waitings=waiting, session=session)
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', threshold, labels={'activity': activity, 'rse': rse_name, 'limit_attr': 'max_transfers'})
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', transfer, labels={'activity': activity, 'rse': rse_name, 'limit_attr': 'transfers'})
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', waiting, labels={'activity': activity, 'rse': rse_name, 'limit_attr': 'waiting'})
if transfer < 0.8 * threshold:
# release requests on account
nr_accounts = len(stats['activities'][activity]['accounts'])
if nr_accounts < 1:
nr_accounts = 1
to_release = threshold - transfer
threshold_per_account = math.ceil(threshold / nr_accounts)
to_release_per_account = math.ceil(to_release / nr_accounts)
accounts = stats['activities'][activity]['accounts']
for account in accounts:
if nr_accounts == 1:
logger(logging.DEBUG, "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (to_release, activity, rse_name, account))
release_waiting_requests_fifo(rse_id, activity=activity, account=account, count=to_release, direction=direction, session=session)
record_gauge('daemons.conveyor.throttler.release_waiting_requests.{activity}.{rse}.{account}', to_release, labels={'activity': activity, 'rse': rse_name, 'account': account})
elif accounts[account]['transfer'] > threshold_per_account:
logger(logging.DEBUG, "Throttler will not release %s waiting requests for activity %s, rse %s, account %s: It queued more transfers than its share " %
(accounts[account]['waiting'], activity, rse_name, account))
nr_accounts -= 1
to_release_per_account = math.ceil(to_release / nr_accounts)
elif accounts[account]['waiting'] < to_release_per_account:
logger(logging.DEBUG, "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (accounts[account]['waiting'], activity, rse_name, account))
release_waiting_requests_fifo(rse_id, activity=activity, account=account, count=accounts[account]['waiting'], direction=direction, session=session)
record_gauge('daemons.conveyor.throttler.release_waiting_requests.{activity}.{rse}.{account}', accounts[account]['waiting'], labels={'activity': activity, 'rse': rse_name, 'account': account})
to_release = to_release - accounts[account]['waiting']
nr_accounts -= 1
to_release_per_account = math.ceil(to_release / nr_accounts)
else:
logger(logging.DEBUG, "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (to_release_per_account, activity, rse_name, account))
release_waiting_requests_fifo(rse_id, activity=activity, account=account, count=to_release_per_account, direction=direction, session=session)
record_gauge('daemons.conveyor.throttler.release_waiting_requests.{activity}.{rse}.{account}', to_release_per_account, labels={'activity': activity, 'rse': rse_name, 'account': account})
to_release = to_release - to_release_per_account
nr_accounts -= 1
else:
logger(logging.DEBUG, "Throttler has done nothing for activity %s on rse %s (transfer > 0.8 * threshold)" % (activity, rse_name))
elif waiting > 0:
logger(logging.DEBUG, "Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse %s" % (threshold, activity, rse_name))
delete_rse_transfer_limits(rse_id, activity=activity, session=session)
release_all_waiting_requests(rse_id, activity=activity, direction=direction, session=session)
record_counter('daemons.conveyor.throttler.delete_rse_transfer_limits.{activity}.{rse}', labels={'activity': activity, 'rse': rse_name})
def __schedule_requests():
"""
Schedule requests
"""
try:
logging.info("Throttler retrieve requests statistics")
results = get_stats_by_activity_dest_state(state=[
RequestState.QUEUED, RequestState.SUBMITTING,
RequestState.SUBMITTED, RequestState.WAITING
])
result_dict = {}
for activity, dest_rse_id, account, state, rse, counter in results:
threshold = get_config_limit(activity, dest_rse_id)
if threshold or (counter and (state == RequestState.WAITING)):
if dest_rse_id not in result_dict:
result_dict[dest_rse_id] = {
'waiting':
0,
'transfer':
0,
'threshold':
get_config_limit('all_activities', dest_rse_id),
'rse':
rse,
'activities': {}
}
if activity not in result_dict[dest_rse_id]['activities']:
result_dict[dest_rse_id]['activities'][activity] = {
'waiting': 0,
'transfer': 0,
'threshold': threshold,
'accounts': {}
}
if account not in result_dict[dest_rse_id]['activities'][
activity]['accounts']:
result_dict[dest_rse_id]['activities'][activity][
'accounts'][account] = {
'waiting': 0,
'transfer': 0
}
if state == RequestState.WAITING:
result_dict[dest_rse_id]['activities'][activity][
'accounts'][account]['waiting'] += counter
result_dict[dest_rse_id]['activities'][activity][
'waiting'] += counter
result_dict[dest_rse_id]['waiting'] += counter
else:
result_dict[dest_rse_id]['activities'][activity][
'accounts'][account]['transfer'] += counter
result_dict[dest_rse_id]['activities'][activity][
'transfer'] += counter
result_dict[dest_rse_id]['transfer'] += counter
for dest_rse_id in result_dict:
dest_rse_release_strategy = config_core.get(
'throttler_release_strategy',
'dest_%s' % dest_rse_id,
default='fifo',
use_cache=False)
rse_name = result_dict[dest_rse_id]['rse']
availability = get_rse(dest_rse_id).availability
if availability & 2: # dest_rse is not blacklisted for write
if dest_rse_release_strategy == 'grouped_fifo':
threshold = result_dict[dest_rse_id]['threshold']
transfer = result_dict[dest_rse_id]['transfer']
waiting = result_dict[dest_rse_id]['waiting']
if threshold and transfer + waiting > threshold:
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.max_transfers'
% (rse_name), threshold)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.transfers'
% (rse_name), transfer)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.waitings'
% (rse_name), waiting)
if transfer < 0.8 * threshold:
to_be_released = threshold - transfer
release_waiting_requests_grouped_fifo(
rse_id=dest_rse_id, count=to_be_released)
else:
logging.debug(
"Throttler has done nothing on rse %s (transfer > 0.8 * threshold)"
% rse_name)
elif waiting > 0 or not threshold:
logging.debug(
"Throttler remove limits(threshold: %s) and release all waiting requests, rse %s"
% (threshold, rse_name))
delete_rse_transfer_limits(rse_id=dest_rse_id,
activity=activity)
release_all_waiting_requests(rse_id=dest_rse_id)
record_counter(
'daemons.conveyor.throttler.delete_rse_transfer_limits.%s'
% (rse_name))
elif dest_rse_release_strategy == 'fifo':
for activity in result_dict[dest_rse_id]['activities']:
threshold = result_dict[dest_rse_id]['activities'][
activity]['threshold']
transfer = result_dict[dest_rse_id]['activities'][
activity]['transfer']
waiting = result_dict[dest_rse_id]['activities'][
activity]['waiting']
if waiting:
logging.debug(
"Request status for %s at %s: %s" %
(activity, rse_name, result_dict[dest_rse_id]
['activities'][activity]))
if threshold is None:
logging.debug(
"Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse_id %s"
% (threshold, activity, dest_rse_id))
delete_rse_transfer_limits(rse_id=dest_rse_id,
activity=activity)
release_all_waiting_requests(rse_id=dest_rse_id,
activity=activity)
record_counter(
'daemons.conveyor.throttler.delete_rse_transfer_limits.%s.%s'
% (activity, rse_name))
elif transfer + waiting > threshold:
logging.debug(
"Throttler set limits for activity %s, rse %s"
% (activity, rse_name))
set_rse_transfer_limits(rse_id=dest_rse_id,
activity=activity,
max_transfers=threshold,
transfers=transfer,
waitings=waiting)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.max_transfers'
% (activity, rse_name), threshold)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.transfers'
% (activity, rse_name), transfer)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.waitings'
% (activity, rse_name), waiting)
if transfer < 0.8 * threshold:
# release requests on account
nr_accounts = len(
result_dict[dest_rse_id]['activities']
[activity]['accounts'])
if nr_accounts < 1:
nr_accounts = 1
to_release = threshold - transfer
threshold_per_account = math.ceil(threshold /
nr_accounts)
to_release_per_account = math.ceil(to_release /
nr_accounts)
accounts = result_dict[dest_rse_id][
'activities'][activity]['accounts']
for account in accounts:
if nr_accounts == 1:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
% (to_release, activity, rse_name,
account))
release_waiting_requests_fifo(
rse_id=dest_rse_id,
activity=activity,
account=account,
count=to_release)
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
to_release)
elif accounts[account][
'transfer'] > threshold_per_account:
logging.debug(
"Throttler will not release %s waiting requests for activity %s, rse %s, account %s: It queued more transfers than its share "
% (accounts[account]['waiting'],
activity, rse_name, account))
nr_accounts -= 1
to_release_per_account = math.ceil(
to_release / nr_accounts)
elif accounts[account][
'waiting'] < to_release_per_account:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
% (accounts[account]['waiting'],
activity, rse_name, account))
release_waiting_requests_fifo(
rse_id=dest_rse_id,
activity=activity,
account=account,
count=accounts[account]['waiting'])
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
accounts[account]['waiting'])
to_release = to_release - accounts[
account]['waiting']
nr_accounts -= 1
to_release_per_account = math.ceil(
to_release / nr_accounts)
else:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
% (to_release_per_account,
activity, rse_name, account))
release_waiting_requests_fifo(
rse_id=dest_rse_id,
activity=activity,
account=account,
count=to_release_per_account)
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
to_release_per_account)
to_release = to_release - to_release_per_account
nr_accounts -= 1
else:
logging.debug(
"Throttler has done nothing for activity %s on rse %s (transfer > 0.8 * threshold)"
% (activity, rse_name))
elif waiting > 0:
logging.debug(
"Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse %s"
% (threshold, activity, rse_name))
delete_rse_transfer_limits(rse_id=dest_rse_id,
activity=activity)
release_all_waiting_requests(rse_id=dest_rse_id,
activity=activity)
record_counter(
'daemons.conveyor.throttler.delete_rse_transfer_limits.%s.%s'
% (activity, rse_name))
except Exception:
logging.critical("Failed to schedule requests, error: %s" %
(traceback.format_exc()))
