def import_distances(distances, vo='def', session=None): for src_rse_name in distances: src = rse_module.get_rse_id(rse=src_rse_name, vo=vo, session=session) for dest_rse_name in distances[src_rse_name]: dest = rse_module.get_rse_id(rse=dest_rse_name, vo=vo, session=session) distance = distances[src_rse_name][dest_rse_name] if 'src_rse_id' in distance: del distance['src_rse_id'] if 'dest_rse_id' in distance: del distance['dest_rse_id'] old_distance = distance_module.get_distances(src_rse_id=src, dest_rse_id=dest, session=session) if old_distance: distance_module.update_distances(src_rse_id=src, dest_rse_id=dest, parameters=distance, session=session) else: distance_module.add_distance( src_rse_id=src, dest_rse_id=dest, ranking=distance.get('ranking'), agis_distance=distance.get('agis_distance'), geoip_distance=distance.get('geoip_distance'), active=distance.get('active'), submitted=distance.get('submitted'), transfer_speed=distance.get('transfer_speed'), finished=distance.get('finished'), failed=distance.get('failed'), session=session)
def update_distance(source, destination, parameters, issuer, vo='def', session=None): """ Update distances with the given RSE ids. :param source: The source RSE. :param destination: The destination RSE. :param parameters: A dictionnary with property :param session: The database session to use. :param issuer: The issuer account. :param vo: The VO to act on. :param session: The database session in use. """ kwargs = {'source': source, 'destination': destination} if not permission.has_permission(issuer=issuer, vo=vo, action='update_distance', kwargs=kwargs, session=session): raise exception.AccessDenied( 'Account %s can not update RSE distances' % (issuer)) if 'distance' in parameters: parameters['agis_distance'] = parameters['distance'] parameters.pop('distance', None) return distance_module.update_distances( src_rse_id=rse_module.get_rse_id(source, vo=vo, session=session), dest_rse_id=rse_module.get_rse_id(destination, vo=vo, session=session), parameters=parameters, session=session)
def import_data(data, session=None): """ Import data to add and update records in Rucio. :param data: data to be imported as dictionary. :param session: database session in use. """ # RSEs rses = data.get('rses') if rses: for rse in rses: protocols = rse.get('protocols') if protocols: protocols = protocols.get('protocols') del rse['protocols'] rse_name = rse['rse'] del rse['rse'] if not rse_module.rse_exists(rse_name, session=session): rse_module.add_rse(rse_name, deterministic=rse.get('deterministic'), volatile=rse.get('volatile'), city=rse.get('city'), region_code=rse.get('region_code'), country_name=rse.get('country_name'), staging_area=rse.get('staging_area'), continent=rse.get('continent'), time_zone=rse.get('time_zone'), ISP=rse.get('ISP'), rse_type=rse.get('rse_type'), latitude=rse.get('latitude'), longitude=rse.get('longitude'), ASN=rse.get('ASN'), availability=rse.get('availability'), session=session) else: rse_module.update_rse(rse_name, rse, session=session) # Protocols if protocols: old_protocols = rse_module.get_rse_protocols(rse=rse_name, session=session) for protocol in protocols: scheme = protocol.get('scheme') hostname = protocol.get('hostname') port = protocol.get('port') intersection = [ old_protocol for old_protocol in old_protocols['protocols'] if old_protocol['scheme'] == scheme and old_protocol['hostname'] == hostname and old_protocol['port'] == port ] if intersection: del protocol['scheme'] del protocol['hostname'] del protocol['port'] rse_module.update_protocols(rse=rse_name, scheme=scheme, data=protocol, hostname=hostname, port=port, session=session) else: rse_module.add_protocol(rse=rse_name, parameter=protocol, session=session) # Limits limits = rse.get('limits') if limits: old_limits = rse_module.get_rse_limits(rse=rse_name, session=session) for limit in limits: if limit in old_limits: rse_module.delete_rse_limit(rse=rse_name, name=limit, session=session) rse_module.set_rse_limits(rse=rse_name, name=limit, value=limits[limit], session=session) # Transfer limits transfer_limits = rse.get('transfer_limits') if transfer_limits: for limit in transfer_limits: old_transfer_limits = rse_module.get_rse_transfer_limits( rse=rse_name, activity=limit, session=session) if limit in old_transfer_limits: rse_module.delete_rse_transfer_limits(rse=rse_name, activity=limit, session=session) max_transfers = transfer_limits[limit].items( )[0][1]['max_transfers'] rse_module.set_rse_transfer_limits( rse=rse_name, activity=limit, max_transfers=max_transfers, session=session) # Attributes attributes = rse.get('attributes') if attributes: old_attributes = rse_module.list_rse_attributes( rse=rse_name, session=session) for attr in attributes: if attr in old_attributes: rse_module.del_rse_attribute(rse=rse_name, key=attr, session=session) rse_module.add_rse_attribute(rse=rse_name, key=attr, value=attributes[attr], session=session) # Distances distances = data.get('distances') if distances: for src_rse_name in distances: src = rse_module.get_rse_id(src_rse_name, session=session) for dest_rse_name in distances[src_rse_name]: dest = rse_module.get_rse_id(dest_rse_name, session=session) distance = distances[src_rse_name][dest_rse_name] del distance['src_rse_id'] del distance['dest_rse_id'] old_distance = distance_module.get_distances(src_rse_id=src, dest_rse_id=dest, session=session) if old_distance: distance_module.update_distances(src_rse_id=src, dest_rse_id=dest, parameters=distance, session=session) else: distance_module.add_distance( src_rse_id=src, dest_rse_id=dest, ranking=distance.get('ranking'), agis_distance=distance.get('agis_distance'), geoip_distance=distance.get('geoip_distance'), active=distance.get('active'), submitted=distance.get('submitted'), transfer_speed=distance.get('transfer_speed'), finished=distance.get('finished'), failed=distance.get('failed'), session=session)
def test_disk_vs_tape_priority(rse_factory, root_account, mock_scope): tape1_rse_name, tape1_rse_id = rse_factory.make_posix_rse(rse_type=RSEType.TAPE) tape2_rse_name, tape2_rse_id = rse_factory.make_posix_rse(rse_type=RSEType.TAPE) disk1_rse_name, disk1_rse_id = rse_factory.make_posix_rse(rse_type=RSEType.DISK) disk2_rse_name, disk2_rse_id = rse_factory.make_posix_rse(rse_type=RSEType.DISK) dst_rse_name, dst_rse_id = rse_factory.make_posix_rse() source_rses = [tape1_rse_id, tape2_rse_id, disk1_rse_id, disk2_rse_id] all_rses = source_rses + [dst_rse_id] # add same file to all source RSEs file = {'scope': mock_scope, 'name': 'lfn.' + generate_uuid(), 'type': 'FILE', 'bytes': 1, 'adler32': 'beefdead'} did = {'scope': file['scope'], 'name': file['name']} for rse_id in source_rses: add_replicas(rse_id=rse_id, files=[file], account=root_account) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) request = request_core.get_request_by_did(rse_id=dst_rse_id, **did) @transactional_session def __fake_source_ranking(source_rse_id, new_ranking, session=None): rowcount = session.query(models.Source).filter(models.Source.rse_id == source_rse_id).update({'ranking': new_ranking}) if not rowcount: models.Source(request_id=request['id'], scope=request['scope'], name=request['name'], rse_id=source_rse_id, dest_rse_id=request['dest_rse_id'], ranking=new_ranking, bytes=request['bytes'], url=None, is_using=False). \ save(session=session, flush=False) # Init all distances to the same value for rse_id in source_rses: add_distance(rse_id, dst_rse_id, ranking=10) # On equal priority and distance, disk should be preferred over tape. Both disk sources will be returned transfers, _reqs_no_source, _reqs_scheme_mismatch, _reqs_only_tape_source = get_transfer_requests_and_source_replicas(rses=all_rses) assert len(transfers) == 1 transfer = next(iter(transfers.values())) assert len(transfer['sources']) == 2 assert transfer['sources'][0][0] in (disk1_rse_name, disk2_rse_name) # Change the rating of the disk RSEs. Tape RSEs must now be preferred. # Multiple tape sources are not allowed. Only one tape RSE source must be returned. __fake_source_ranking(disk1_rse_id, -1) __fake_source_ranking(disk2_rse_id, -1) transfers, _reqs_no_source, _reqs_scheme_mismatch, _reqs_only_tape_source = get_transfer_requests_and_source_replicas(rses=all_rses) assert len(transfers) == 1 transfer = next(iter(transfers.values())) assert len(transfer['sources']) == 1 assert transfer['sources'][0][0] in (tape1_rse_name, tape2_rse_name) # On equal source ranking, but different distance; the smaller distance is preferred update_distances(tape1_rse_id, dst_rse_id, parameters={'ranking': 15}) transfers, _reqs_no_source, _reqs_scheme_mismatch, _reqs_only_tape_source = get_transfer_requests_and_source_replicas(rses=all_rses) assert len(transfers) == 1 transfer = next(iter(transfers.values())) assert len(transfer['sources']) == 1 assert transfer['sources'][0][0] == tape2_rse_name # On different source ranking, the bigger ranking is preferred __fake_source_ranking(tape2_rse_id, -1) transfers, _reqs_no_source, _reqs_scheme_mismatch, _reqs_only_tape_source = get_transfer_requests_and_source_replicas(rses=all_rses) assert len(transfers) == 1 transfer = next(iter(transfers.values())) assert len(transfer['sources']) == 1 assert transfer['sources'][0][0] == tape1_rse_name