def import_distances(distances, vo='def', session=None):
for src_rse_name in distances:
src = rse_module.get_rse_id(rse=src_rse_name, vo=vo, session=session)
for dest_rse_name in distances[src_rse_name]:
dest = rse_module.get_rse_id(rse=dest_rse_name,
vo=vo,
session=session)
distance = distances[src_rse_name][dest_rse_name]
if 'src_rse_id' in distance:
del distance['src_rse_id']
if 'dest_rse_id' in distance:
del distance['dest_rse_id']
old_distance = distance_module.get_distances(src_rse_id=src,
dest_rse_id=dest,
session=session)
if old_distance:
distance_module.update_distances(src_rse_id=src,
dest_rse_id=dest,
parameters=distance,
session=session)
else:
distance_module.add_distance(
src_rse_id=src,
dest_rse_id=dest,
ranking=distance.get('ranking'),
agis_distance=distance.get('agis_distance'),
geoip_distance=distance.get('geoip_distance'),
active=distance.get('active'),
submitted=distance.get('submitted'),
transfer_speed=distance.get('transfer_speed'),
finished=distance.get('finished'),
failed=distance.get('failed'),
session=session)
def update_distance(source,
destination,
parameters,
issuer,
vo='def',
session=None):
"""
Update distances with the given RSE ids.
:param source: The source RSE.
:param destination: The destination RSE.
:param parameters: A dictionnary with property
:param session: The database session to use.
:param issuer: The issuer account.
:param vo: The VO to act on.
:param session: The database session in use.
"""
kwargs = {'source': source, 'destination': destination}
if not permission.has_permission(issuer=issuer,
vo=vo,
action='update_distance',
kwargs=kwargs,
session=session):
raise exception.AccessDenied(
'Account %s can not update RSE distances' % (issuer))
if 'distance' in parameters:
parameters['agis_distance'] = parameters['distance']
parameters.pop('distance', None)
return distance_module.update_distances(
src_rse_id=rse_module.get_rse_id(source, vo=vo, session=session),
dest_rse_id=rse_module.get_rse_id(destination, vo=vo, session=session),
parameters=parameters,
session=session)
def import_data(data, session=None):
"""
Import data to add and update records in Rucio.
:param data: data to be imported as dictionary.
:param session: database session in use.
"""
# RSEs
rses = data.get('rses')
if rses:
for rse in rses:
protocols = rse.get('protocols')
if protocols:
protocols = protocols.get('protocols')
del rse['protocols']
rse_name = rse['rse']
del rse['rse']
if not rse_module.rse_exists(rse_name, session=session):
rse_module.add_rse(rse_name,
deterministic=rse.get('deterministic'),
volatile=rse.get('volatile'),
city=rse.get('city'),
region_code=rse.get('region_code'),
country_name=rse.get('country_name'),
staging_area=rse.get('staging_area'),
continent=rse.get('continent'),
time_zone=rse.get('time_zone'),
ISP=rse.get('ISP'),
rse_type=rse.get('rse_type'),
latitude=rse.get('latitude'),
longitude=rse.get('longitude'),
ASN=rse.get('ASN'),
availability=rse.get('availability'),
session=session)
else:
rse_module.update_rse(rse_name, rse, session=session)
# Protocols
if protocols:
old_protocols = rse_module.get_rse_protocols(rse=rse_name,
session=session)
for protocol in protocols:
scheme = protocol.get('scheme')
hostname = protocol.get('hostname')
port = protocol.get('port')
intersection = [
old_protocol
for old_protocol in old_protocols['protocols']
if old_protocol['scheme'] == scheme
and old_protocol['hostname'] == hostname
and old_protocol['port'] == port
]
if intersection:
del protocol['scheme']
del protocol['hostname']
del protocol['port']
rse_module.update_protocols(rse=rse_name,
scheme=scheme,
data=protocol,
hostname=hostname,
port=port,
session=session)
else:
rse_module.add_protocol(rse=rse_name,
parameter=protocol,
session=session)
# Limits
limits = rse.get('limits')
if limits:
old_limits = rse_module.get_rse_limits(rse=rse_name,
session=session)
for limit in limits:
if limit in old_limits:
rse_module.delete_rse_limit(rse=rse_name,
name=limit,
session=session)
rse_module.set_rse_limits(rse=rse_name,
name=limit,
value=limits[limit],
session=session)
# Transfer limits
transfer_limits = rse.get('transfer_limits')
if transfer_limits:
for limit in transfer_limits:
old_transfer_limits = rse_module.get_rse_transfer_limits(
rse=rse_name, activity=limit, session=session)
if limit in old_transfer_limits:
rse_module.delete_rse_transfer_limits(rse=rse_name,
activity=limit,
session=session)
max_transfers = transfer_limits[limit].items(
)[0][1]['max_transfers']
rse_module.set_rse_transfer_limits(
rse=rse_name,
activity=limit,
max_transfers=max_transfers,
session=session)
# Attributes
attributes = rse.get('attributes')
if attributes:
old_attributes = rse_module.list_rse_attributes(
rse=rse_name, session=session)
for attr in attributes:
if attr in old_attributes:
rse_module.del_rse_attribute(rse=rse_name,
key=attr,
session=session)
rse_module.add_rse_attribute(rse=rse_name,
key=attr,
value=attributes[attr],
session=session)
# Distances
distances = data.get('distances')
if distances:
for src_rse_name in distances:
src = rse_module.get_rse_id(src_rse_name, session=session)
for dest_rse_name in distances[src_rse_name]:
dest = rse_module.get_rse_id(dest_rse_name, session=session)
distance = distances[src_rse_name][dest_rse_name]
del distance['src_rse_id']
del distance['dest_rse_id']
old_distance = distance_module.get_distances(src_rse_id=src,
dest_rse_id=dest,
session=session)
if old_distance:
distance_module.update_distances(src_rse_id=src,
dest_rse_id=dest,
parameters=distance,
session=session)
else:
distance_module.add_distance(
src_rse_id=src,
dest_rse_id=dest,
ranking=distance.get('ranking'),
agis_distance=distance.get('agis_distance'),
geoip_distance=distance.get('geoip_distance'),
active=distance.get('active'),
submitted=distance.get('submitted'),
transfer_speed=distance.get('transfer_speed'),
finished=distance.get('finished'),
failed=distance.get('failed'),
session=session)
def test_disk_vs_tape_priority(rse_factory, root_account, mock_scope):
tape1_rse_name, tape1_rse_id = rse_factory.make_posix_rse(rse_type=RSEType.TAPE)
tape2_rse_name, tape2_rse_id = rse_factory.make_posix_rse(rse_type=RSEType.TAPE)
disk1_rse_name, disk1_rse_id = rse_factory.make_posix_rse(rse_type=RSEType.DISK)
disk2_rse_name, disk2_rse_id = rse_factory.make_posix_rse(rse_type=RSEType.DISK)
dst_rse_name, dst_rse_id = rse_factory.make_posix_rse()
source_rses = [tape1_rse_id, tape2_rse_id, disk1_rse_id, disk2_rse_id]
all_rses = source_rses + [dst_rse_id]
# add same file to all source RSEs
file = {'scope': mock_scope, 'name': 'lfn.' + generate_uuid(), 'type': 'FILE', 'bytes': 1, 'adler32': 'beefdead'}
did = {'scope': file['scope'], 'name': file['name']}
for rse_id in source_rses:
add_replicas(rse_id=rse_id, files=[file], account=root_account)
rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
@transactional_session
def __fake_source_ranking(source_rse_id, new_ranking, session=None):
rowcount = session.query(models.Source).filter(models.Source.rse_id == source_rse_id).update({'ranking': new_ranking})
if not rowcount:
models.Source(request_id=request['id'],
scope=request['scope'],
name=request['name'],
rse_id=source_rse_id,
dest_rse_id=request['dest_rse_id'],
ranking=new_ranking,
bytes=request['bytes'],
url=None,
is_using=False). \
save(session=session, flush=False)
# Init all distances to the same value
for rse_id in source_rses:
add_distance(rse_id, dst_rse_id, ranking=10)
# On equal priority and distance, disk should be preferred over tape. Both disk sources will be returned
transfers, _reqs_no_source, _reqs_scheme_mismatch, _reqs_only_tape_source = get_transfer_requests_and_source_replicas(rses=all_rses)
assert len(transfers) == 1
transfer = next(iter(transfers.values()))
assert len(transfer['sources']) == 2
assert transfer['sources'][0][0] in (disk1_rse_name, disk2_rse_name)
# Change the rating of the disk RSEs. Tape RSEs must now be preferred.
# Multiple tape sources are not allowed. Only one tape RSE source must be returned.
__fake_source_ranking(disk1_rse_id, -1)
__fake_source_ranking(disk2_rse_id, -1)
transfers, _reqs_no_source, _reqs_scheme_mismatch, _reqs_only_tape_source = get_transfer_requests_and_source_replicas(rses=all_rses)
assert len(transfers) == 1
transfer = next(iter(transfers.values()))
assert len(transfer['sources']) == 1
assert transfer['sources'][0][0] in (tape1_rse_name, tape2_rse_name)
# On equal source ranking, but different distance; the smaller distance is preferred
update_distances(tape1_rse_id, dst_rse_id, parameters={'ranking': 15})
transfers, _reqs_no_source, _reqs_scheme_mismatch, _reqs_only_tape_source = get_transfer_requests_and_source_replicas(rses=all_rses)
assert len(transfers) == 1
transfer = next(iter(transfers.values()))
assert len(transfer['sources']) == 1
assert transfer['sources'][0][0] == tape2_rse_name
# On different source ranking, the bigger ranking is preferred
__fake_source_ranking(tape2_rse_id, -1)
transfers, _reqs_no_source, _reqs_scheme_mismatch, _reqs_only_tape_source = get_transfer_requests_and_source_replicas(rses=all_rses)
assert len(transfers) == 1
transfer = next(iter(transfers.values()))
assert len(transfer['sources']) == 1
assert transfer['sources'][0][0] == tape1_rse_name
