def __release_all_activities(stats, direction, rse_name, rse_id, logger, session):
"""
Release requests if activities should be ignored.
:param stats: Request statistics
:param direction: String whether request statistics are based on source or destination RSEs.
:param rse_name: RSE name.
:param rse_id: RSE id.
"""
threshold = stats['threshold']
transfer = stats['transfer']
waiting = stats['waiting']
strategy = stats['strategy']
if threshold is not None and transfer + waiting > threshold:
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', threshold, labels={'activity': 'all_activities', 'rse': rse_name, 'limit_attr': 'max_transfers'})
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', transfer, labels={'activity': 'all_activities', 'rse': rse_name, 'limit_attr': 'transfers'})
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', waiting, labels={'activity': 'all_activities', 'rse': rse_name, 'limit_attr': 'waiting'})
if transfer < 0.8 * threshold:
to_be_released = threshold - transfer
if strategy == 'grouped_fifo':
deadline = stats.get('deadline')
volume = stats.get('volume')
release_waiting_requests_grouped_fifo(rse_id, count=to_be_released, direction=direction, volume=volume, deadline=deadline, session=session)
elif strategy == 'fifo':
release_waiting_requests_fifo(rse_id, count=to_be_released, direction=direction, session=session)
else:
logger(logging.DEBUG, "Throttler has done nothing on rse %s (transfer > 0.8 * threshold)" % rse_name)
elif waiting > 0 or not threshold:
logger(logging.DEBUG, "Throttler remove limits(threshold: %s) and release all waiting requests, rse %s" % (threshold, rse_name))
delete_rse_transfer_limits(rse_id, activity='all_activities', session=session)
release_all_waiting_requests(rse_id, direction=direction, session=session)
record_counter('daemons.conveyor.throttler.delete_rse_transfer_limits.{activity}.{rse}', labels={'activity': 'all_activities', 'rse': rse_name})
def __release_per_activity(stats, direction, rse_name, rse_id, logger, session):
"""
Release requests per activity.
:param stats: Request statistics
:param direction: String whether request statistics are based on source or destination RSEs.
:param rse_name: RSE name.
:param rse_id: RSE id.
"""
for activity in stats['activities']:
threshold = stats['activities'][activity]['threshold']
transfer = stats['activities'][activity]['transfer']
waiting = stats['activities'][activity]['waiting']
if waiting:
logger(logging.DEBUG, "Request status for %s at %s: %s" % (activity, rse_name,
stats['activities'][activity]))
if threshold is None:
logger(logging.DEBUG, "Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse_id %s" % (threshold, activity, rse_id))
delete_rse_transfer_limits(rse_id, activity=activity, session=session)
release_all_waiting_requests(rse_id, activity=activity, direction=direction, session=session)
record_counter('daemons.conveyor.throttler.delete_rse_transfer_limits.{activity}.{rse}', labels={'activity': activity, 'rse': rse_name})
elif transfer + waiting > threshold:
logger(logging.DEBUG, "Throttler set limits for activity %s, rse %s" % (activity, rse_name))
set_rse_transfer_limits(rse_id, activity=activity, max_transfers=threshold, transfers=transfer, waitings=waiting, session=session)
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', threshold, labels={'activity': activity, 'rse': rse_name, 'limit_attr': 'max_transfers'})
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', transfer, labels={'activity': activity, 'rse': rse_name, 'limit_attr': 'transfers'})
record_gauge('daemons.conveyor.throttler.set_rse_transfer_limits.{activity}.{rse}.{limit_attr}', waiting, labels={'activity': activity, 'rse': rse_name, 'limit_attr': 'waiting'})
if transfer < 0.8 * threshold:
# release requests on account
nr_accounts = len(stats['activities'][activity]['accounts'])
if nr_accounts < 1:
nr_accounts = 1
to_release = threshold - transfer
threshold_per_account = math.ceil(threshold / nr_accounts)
to_release_per_account = math.ceil(to_release / nr_accounts)
accounts = stats['activities'][activity]['accounts']
for account in accounts:
if nr_accounts == 1:
logger(logging.DEBUG, "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (to_release, activity, rse_name, account))
release_waiting_requests_fifo(rse_id, activity=activity, account=account, count=to_release, direction=direction, session=session)
record_gauge('daemons.conveyor.throttler.release_waiting_requests.{activity}.{rse}.{account}', to_release, labels={'activity': activity, 'rse': rse_name, 'account': account})
elif accounts[account]['transfer'] > threshold_per_account:
logger(logging.DEBUG, "Throttler will not release %s waiting requests for activity %s, rse %s, account %s: It queued more transfers than its share " %
(accounts[account]['waiting'], activity, rse_name, account))
nr_accounts -= 1
to_release_per_account = math.ceil(to_release / nr_accounts)
elif accounts[account]['waiting'] < to_release_per_account:
logger(logging.DEBUG, "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (accounts[account]['waiting'], activity, rse_name, account))
release_waiting_requests_fifo(rse_id, activity=activity, account=account, count=accounts[account]['waiting'], direction=direction, session=session)
record_gauge('daemons.conveyor.throttler.release_waiting_requests.{activity}.{rse}.{account}', accounts[account]['waiting'], labels={'activity': activity, 'rse': rse_name, 'account': account})
to_release = to_release - accounts[account]['waiting']
nr_accounts -= 1
to_release_per_account = math.ceil(to_release / nr_accounts)
else:
logger(logging.DEBUG, "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (to_release_per_account, activity, rse_name, account))
release_waiting_requests_fifo(rse_id, activity=activity, account=account, count=to_release_per_account, direction=direction, session=session)
record_gauge('daemons.conveyor.throttler.release_waiting_requests.{activity}.{rse}.{account}', to_release_per_account, labels={'activity': activity, 'rse': rse_name, 'account': account})
to_release = to_release - to_release_per_account
nr_accounts -= 1
else:
logger(logging.DEBUG, "Throttler has done nothing for activity %s on rse %s (transfer > 0.8 * threshold)" % (activity, rse_name))
elif waiting > 0:
logger(logging.DEBUG, "Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse %s" % (threshold, activity, rse_name))
delete_rse_transfer_limits(rse_id, activity=activity, session=session)
release_all_waiting_requests(rse_id, activity=activity, direction=direction, session=session)
record_counter('daemons.conveyor.throttler.delete_rse_transfer_limits.{activity}.{rse}', labels={'activity': activity, 'rse': rse_name})
def import_data(data, session=None):
"""
Import data to add and update records in Rucio.
:param data: data to be imported as dictionary.
:param session: database session in use.
"""
# RSEs
rses = data.get('rses')
if rses:
for rse in rses:
protocols = rse.get('protocols')
if protocols:
protocols = protocols.get('protocols')
del rse['protocols']
rse_name = rse['rse']
del rse['rse']
if not rse_module.rse_exists(rse_name, session=session):
rse_module.add_rse(rse_name,
deterministic=rse.get('deterministic'),
volatile=rse.get('volatile'),
city=rse.get('city'),
region_code=rse.get('region_code'),
country_name=rse.get('country_name'),
staging_area=rse.get('staging_area'),
continent=rse.get('continent'),
time_zone=rse.get('time_zone'),
ISP=rse.get('ISP'),
rse_type=rse.get('rse_type'),
latitude=rse.get('latitude'),
longitude=rse.get('longitude'),
ASN=rse.get('ASN'),
availability=rse.get('availability'),
session=session)
else:
rse_module.update_rse(rse_name, rse, session=session)
# Protocols
if protocols:
old_protocols = rse_module.get_rse_protocols(rse=rse_name,
session=session)
for protocol in protocols:
scheme = protocol.get('scheme')
hostname = protocol.get('hostname')
port = protocol.get('port')
intersection = [
old_protocol
for old_protocol in old_protocols['protocols']
if old_protocol['scheme'] == scheme
and old_protocol['hostname'] == hostname
and old_protocol['port'] == port
]
if intersection:
del protocol['scheme']
del protocol['hostname']
del protocol['port']
rse_module.update_protocols(rse=rse_name,
scheme=scheme,
data=protocol,
hostname=hostname,
port=port,
session=session)
else:
rse_module.add_protocol(rse=rse_name,
parameter=protocol,
session=session)
# Limits
limits = rse.get('limits')
if limits:
old_limits = rse_module.get_rse_limits(rse=rse_name,
session=session)
for limit in limits:
if limit in old_limits:
rse_module.delete_rse_limit(rse=rse_name,
name=limit,
session=session)
rse_module.set_rse_limits(rse=rse_name,
name=limit,
value=limits[limit],
session=session)
# Transfer limits
transfer_limits = rse.get('transfer_limits')
if transfer_limits:
for limit in transfer_limits:
old_transfer_limits = rse_module.get_rse_transfer_limits(
rse=rse_name, activity=limit, session=session)
if limit in old_transfer_limits:
rse_module.delete_rse_transfer_limits(rse=rse_name,
activity=limit,
session=session)
max_transfers = transfer_limits[limit].items(
)[0][1]['max_transfers']
rse_module.set_rse_transfer_limits(
rse=rse_name,
activity=limit,
max_transfers=max_transfers,
session=session)
# Attributes
attributes = rse.get('attributes')
if attributes:
old_attributes = rse_module.list_rse_attributes(
rse=rse_name, session=session)
for attr in attributes:
if attr in old_attributes:
rse_module.del_rse_attribute(rse=rse_name,
key=attr,
session=session)
rse_module.add_rse_attribute(rse=rse_name,
key=attr,
value=attributes[attr],
session=session)
# Distances
distances = data.get('distances')
if distances:
for src_rse_name in distances:
src = rse_module.get_rse_id(src_rse_name, session=session)
for dest_rse_name in distances[src_rse_name]:
dest = rse_module.get_rse_id(dest_rse_name, session=session)
distance = distances[src_rse_name][dest_rse_name]
del distance['src_rse_id']
del distance['dest_rse_id']
old_distance = distance_module.get_distances(src_rse_id=src,
dest_rse_id=dest,
session=session)
if old_distance:
distance_module.update_distances(src_rse_id=src,
dest_rse_id=dest,
parameters=distance,
session=session)
else:
distance_module.add_distance(
src_rse_id=src,
dest_rse_id=dest,
ranking=distance.get('ranking'),
agis_distance=distance.get('agis_distance'),
geoip_distance=distance.get('geoip_distance'),
active=distance.get('active'),
submitted=distance.get('submitted'),
transfer_speed=distance.get('transfer_speed'),
finished=distance.get('finished'),
failed=distance.get('failed'),
session=session)
def __schedule_requests():
"""
Schedule requests
"""
try:
logging.info("Throttler retrieve requests statistics")
results = get_stats_by_activity_dest_state(state=[
RequestState.QUEUED, RequestState.SUBMITTING,
RequestState.SUBMITTED, RequestState.WAITING
])
result_dict = {}
for activity, dest_rse_id, account, state, rse, counter in results:
threshold = get_config_limit(activity, dest_rse_id)
if threshold or (counter and (state == RequestState.WAITING)):
if activity not in result_dict:
result_dict[activity] = {}
if dest_rse_id not in result_dict[activity]:
result_dict[activity][dest_rse_id] = {
'waiting': 0,
'transfer': 0,
'threshold': threshold,
'accounts': {},
'rse': rse
}
if account not in result_dict[activity][dest_rse_id][
'accounts']:
result_dict[activity][dest_rse_id]['accounts'][account] = {
'waiting': 0,
'transfer': 0
}
if state == RequestState.WAITING:
result_dict[activity][dest_rse_id]['accounts'][account][
'waiting'] += counter
result_dict[activity][dest_rse_id]['waiting'] += counter
else:
result_dict[activity][dest_rse_id]['accounts'][account][
'transfer'] += counter
result_dict[activity][dest_rse_id]['transfer'] += counter
for activity in result_dict:
for dest_rse_id in result_dict[activity]:
threshold = result_dict[activity][dest_rse_id]['threshold']
transfer = result_dict[activity][dest_rse_id]['transfer']
waiting = result_dict[activity][dest_rse_id]['waiting']
rse_name = result_dict[activity][dest_rse_id]['rse']
if waiting:
logging.debug("Request status for %s at %s: %s" %
(activity, rse_name,
result_dict[activity][dest_rse_id]))
if threshold is None:
logging.debug(
"Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse_id %s"
% (threshold, activity, dest_rse_id))
delete_rse_transfer_limits(rse=None,
activity=activity,
rse_id=dest_rse_id)
release_waiting_requests(rse=None,
activity=activity,
rse_id=dest_rse_id)
record_counter(
'daemons.conveyor.throttler.delete_rse_transfer_limits.%s.%s'
% (activity, rse_name))
elif transfer + waiting > threshold:
logging.debug(
"Throttler set limits for activity %s, rse %s" %
(activity, rse_name))
set_rse_transfer_limits(rse=None,
activity=activity,
rse_id=dest_rse_id,
max_transfers=threshold,
transfers=transfer,
waitings=waiting)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.max_transfers'
% (activity, rse_name), threshold)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.transfers'
% (activity, rse_name), transfer)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.waitings'
% (activity, rse_name), waiting)
if transfer < 0.8 * threshold:
# release requests on account
nr_accounts = len(
result_dict[activity][dest_rse_id]['accounts'])
if nr_accounts < 1:
nr_accounts = 1
to_release = threshold - transfer
threshold_per_account = math.ceil(threshold /
nr_accounts)
to_release_per_account = math.ceil(to_release /
nr_accounts)
accounts = result_dict[activity][dest_rse_id][
'accounts']
for account in accounts:
if nr_accounts == 1:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
%
(to_release, activity, rse_name, account))
release_waiting_requests(rse=None,
activity=activity,
rse_id=dest_rse_id,
account=account,
count=to_release)
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
to_release)
elif accounts[account][
'transfer'] > threshold_per_account:
logging.debug(
"Throttler will not release %s waiting requests for activity %s, rse %s, account %s: It queued more transfers than its share "
% (accounts[account]['waiting'], activity,
rse_name, account))
nr_accounts -= 1
to_release_per_account = math.ceil(to_release /
nr_accounts)
elif accounts[account][
'waiting'] < to_release_per_account:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
% (accounts[account]['waiting'], activity,
rse_name, account))
release_waiting_requests(
rse=None,
activity=activity,
rse_id=dest_rse_id,
account=account,
count=accounts[account]['waiting'])
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
accounts[account]['waiting'])
to_release = to_release - accounts[account][
'waiting']
nr_accounts -= 1
to_release_per_account = math.ceil(to_release /
nr_accounts)
else:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
% (to_release_per_account, activity,
rse_name, account))
release_waiting_requests(
rse=None,
activity=activity,
rse_id=dest_rse_id,
account=account,
count=to_release_per_account)
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
to_release_per_account)
to_release = to_release - to_release_per_account
nr_accounts -= 1
else:
logging.debug(
"Throttler has done nothing for activity %s on rse %s (transfer > 0.8 * threshold)"
% (activity, rse_name))
elif waiting > 0:
logging.debug(
"Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse %s"
% (threshold, activity, rse_name))
delete_rse_transfer_limits(rse=None,
activity=activity,
rse_id=dest_rse_id)
release_waiting_requests(rse=None,
activity=activity,
rse_id=dest_rse_id)
record_counter(
'daemons.conveyor.throttler.delete_rse_transfer_limits.%s.%s'
% (activity, rse_name))
except:
logging.critical("Failed to schedule requests, error: %s" %
(traceback.format_exc()))
