def test_importer_core(self):
""" IMPORTER (CORE): test import. """
import_data(data=deepcopy(self.data1))
# RSE that had not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
new_rse_id = get_rse_id(rse=self.new_rse)
attributes = list_rse_attributes(rse_id=new_rse_id)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse_id=new_rse_id)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already exists
check_rse(self.old_rse_1, self.data1['rses'])
# one protocol should be created, one should be updated
check_protocols(self.old_rse_1, self.data1['rses'])
# one protocol should be removed as it is not specified in the import data
protocols = get_rse_protocols(self.old_rse_id_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in protocols['protocols']]
assert_true({
'hostename': 'hostname3',
'port': 1000,
'scheme': 'scheme3'
} not in protocols)
attributes = list_rse_attributes(rse_id=self.old_rse_id_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse_id=self.old_rse_id_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
self.check_accounts(self.data1['accounts'])
# RSE 4 should be flagged as deleted as it is missing in the import data
with assert_raises(RSENotFound):
get_rse(rse_id=self.old_rse_id_4)
import_data(data=self.data2)
import_data(data=self.data3)
def test_importer_client(self):
""" IMPORTER (CLIENT): test import. """
import_client = ImportClient()
import_client.import_data(data=deepcopy(self.data1))
# RSE that had not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
new_rse_id = get_rse_id(rse=self.new_rse)
protocols = get_rse_protocols(self.old_rse_id_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in protocols['protocols']]
assert_true({
'hostename': 'hostname3',
'port': 1000,
'scheme': 'scheme3'
} not in protocols)
attributes = list_rse_attributes(rse_id=new_rse_id)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse_id=new_rse_id)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already exists
check_rse(self.old_rse_1, self.data1['rses'])
check_protocols(self.old_rse_1, self.data1['rses'])
attributes = list_rse_attributes(rse_id=self.old_rse_id_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse_id=self.old_rse_id_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
self.check_accounts(self.data1['accounts'])
with assert_raises(RSENotFound):
get_rse(rse_id=self.old_rse_id_4)
import_client.import_data(data=self.data2)
import_client.import_data(data=self.data3)
def test_importer_rest(self):
""" IMPORTER (REST): test import. """
mw = []
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)}
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data1))
assert_equal(r2.status, 201)
# RSE that not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
protocols = get_rse_protocols(self.old_rse_1)
protocols = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port']} for protocol in protocols['protocols']]
assert_true({'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3'} not in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already existed before
check_rse(self.old_rse_1, self.data1['rses'])
check_protocols(self.old_rse_1, self.data1['rses'])
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
with assert_raises(RSENotFound):
get_rse(self.old_rse_3)
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data2))
assert_equal(r2.status, 201)
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data3))
assert_equal(r2.status, 201)
def import_distances(distances, vo='def', session=None):
for src_rse_name in distances:
src = rse_module.get_rse_id(rse=src_rse_name, vo=vo, session=session)
for dest_rse_name in distances[src_rse_name]:
dest = rse_module.get_rse_id(rse=dest_rse_name,
vo=vo,
session=session)
distance = distances[src_rse_name][dest_rse_name]
if 'src_rse_id' in distance:
del distance['src_rse_id']
if 'dest_rse_id' in distance:
del distance['dest_rse_id']
old_distance = distance_module.get_distances(src_rse_id=src,
dest_rse_id=dest,
session=session)
if old_distance:
distance_module.update_distances(src_rse_id=src,
dest_rse_id=dest,
parameters=distance,
session=session)
else:
distance_module.add_distance(
src_rse_id=src,
dest_rse_id=dest,
ranking=distance.get('ranking'),
agis_distance=distance.get('agis_distance'),
geoip_distance=distance.get('geoip_distance'),
active=distance.get('active'),
submitted=distance.get('submitted'),
transfer_speed=distance.get('transfer_speed'),
finished=distance.get('finished'),
failed=distance.get('failed'),
session=session)
def test_two_sources_one_destination(db_session, vo, file, source_rse,
source2_rse, mock_request):
add_replicas(rse_id=source2_rse['id'],
files=[file],
account=mock_request.account,
session=db_session)
try:
src1_distance, src2_distance = (get_distances(
src_rse_id=src_rse,
dest_rse_id=mock_request.dest_rse_id,
session=db_session) for src_rse in (source_rse['id'],
source2_rse['id']))
assert src1_distance and len(
src1_distance) == 1 and src1_distance[0]['ranking'] == 5
assert src2_distance and len(
src2_distance) == 1 and src2_distance[0]['ranking'] == 2
preparer.run_once(session=db_session)
db_session.commit()
updated_mock_request = db_session.query(models.Request).filter_by(
id=mock_request.id).one() # type: models.Request
assert updated_mock_request.state == RequestState.QUEUED
assert updated_mock_request.source_rse_id == source2_rse[
'id'] # distance 2 < 5
finally:
delete_replicas(rse_id=source2_rse['id'],
files=[file],
session=db_session)
db_session.commit()
def test_two_sources_one_destination(db_session, vo, file, mock_request):
def setup(rse):
add_distance(rse.rse_id, mock_request.dest_rse_id, ranking=2, session=rse.db_session)
add_replicas(rse_id=rse.rse_id, files=[file], account=mock_request.account, session=rse.db_session)
with GeneratedRSE(vo=vo, db_session=db_session, setup_func=setup) as source2_rse:
src1_distance, src2_distance = (
get_distances(
src_rse_id=src_rse,
dest_rse_id=mock_request.dest_rse_id,
session=db_session,
)
for src_rse in (mock_request.source_rse_id, source2_rse.rse_id)
)
assert src1_distance and len(src1_distance) == 1 and src1_distance[0]['ranking'] == 5
assert src2_distance and len(src2_distance) == 1 and src2_distance[0]['ranking'] == 2
preparer.run_once(session=db_session, logger=print)
db_session.commit()
updated_mock_request = (
db_session.query(models.Request).filter_by(id=mock_request.id).one()
) # type: models.Request
assert updated_mock_request.state == RequestState.QUEUED
assert updated_mock_request.source_rse_id == source2_rse.rse_id # distance 2 < 5
delete_replicas(rse_id=source2_rse.rse_id, files=[file], session=db_session)
def setup(self):
self.db_session = session.get_session()
self.db_session.query(models.Distance).delete()
self.db_session.commit()
self.rse_1 = 'MOCK'
self.rse_1_id = get_rse_id(self.rse_1)
self.rse_2 = 'MOCK2'
self.rse_2_id = get_rse_id(self.rse_2)
ranking = 10
add_distance(self.rse_1_id, self.rse_2_id, ranking)
self.distances = {
self.rse_1: {
self.rse_2: get_distances(self.rse_1_id, self.rse_2_id)[0]
}
}
self.distances_core = {
self.rse_1_id: {
self.rse_2_id: get_distances(self.rse_1_id, self.rse_2_id)[0]
}
}
def get_distance(source, destination, issuer):
"""
Get distances between rses.
:param source: The source RSE.
:param destination: The destination RSE.
:param issuer: The issuer account.
:returns distance: List of dictionaries.
"""
return distance_module.get_distances(src_rse_id=rse_module.get_rse_id(source),
dest_rse_id=rse_module.get_rse_id(destination))
def get_distance(source, destination, issuer, vo='def'):
"""
Get distances between rses.
:param source: The source RSE.
:param destination: The destination RSE.
:param issuer: The issuer account.
:param vo: The VO to act on.
:returns distance: List of dictionaries.
"""
distances = distance_module.get_distances(src_rse_id=rse_module.get_rse_id(source, vo=vo),
dest_rse_id=rse_module.get_rse_id(destination, vo=vo))
return [api_update_return_dict(d) for d in distances]
def get_distance(source, destination, issuer):
"""
Get distances between rses.
:param source: The source RSE.
:param destination: The destination RSE.
:param issuer: The issuer account.
:returns distance: List of dictionaries.
"""
distances = distance_module.get_distances(
src_rse_id=rse_module.get_rse_id(source),
dest_rse_id=rse_module.get_rse_id(destination))
for d in distances:
if 'src_rse_id' in d and d['src_rse_id'] is not None:
d['src_rse'] = rse_module.get_rse_name(rse_id=d['src_rse_id'])
if 'dest_rse_id' in d and d['dest_rse_id'] is not None:
d['dest_rse'] = rse_module.get_rse_name(rse_id=d['dest_rse_id'])
return distances
def import_data(data, session=None):
"""
Import data to add and update records in Rucio.
:param data: data to be imported as dictionary.
:param session: database session in use.
"""
# RSEs
rses = data.get('rses')
if rses:
for rse in rses:
protocols = rse.get('protocols')
if protocols:
protocols = protocols.get('protocols')
del rse['protocols']
rse_name = rse['rse']
del rse['rse']
if not rse_module.rse_exists(rse_name, session=session):
rse_module.add_rse(rse_name,
deterministic=rse.get('deterministic'),
volatile=rse.get('volatile'),
city=rse.get('city'),
region_code=rse.get('region_code'),
country_name=rse.get('country_name'),
staging_area=rse.get('staging_area'),
continent=rse.get('continent'),
time_zone=rse.get('time_zone'),
ISP=rse.get('ISP'),
rse_type=rse.get('rse_type'),
latitude=rse.get('latitude'),
longitude=rse.get('longitude'),
ASN=rse.get('ASN'),
availability=rse.get('availability'),
session=session)
else:
rse_module.update_rse(rse_name, rse, session=session)
# Protocols
if protocols:
old_protocols = rse_module.get_rse_protocols(rse=rse_name,
session=session)
for protocol in protocols:
scheme = protocol.get('scheme')
hostname = protocol.get('hostname')
port = protocol.get('port')
intersection = [
old_protocol
for old_protocol in old_protocols['protocols']
if old_protocol['scheme'] == scheme
and old_protocol['hostname'] == hostname
and old_protocol['port'] == port
]
if intersection:
del protocol['scheme']
del protocol['hostname']
del protocol['port']
rse_module.update_protocols(rse=rse_name,
scheme=scheme,
data=protocol,
hostname=hostname,
port=port,
session=session)
else:
rse_module.add_protocol(rse=rse_name,
parameter=protocol,
session=session)
# Limits
limits = rse.get('limits')
if limits:
old_limits = rse_module.get_rse_limits(rse=rse_name,
session=session)
for limit in limits:
if limit in old_limits:
rse_module.delete_rse_limit(rse=rse_name,
name=limit,
session=session)
rse_module.set_rse_limits(rse=rse_name,
name=limit,
value=limits[limit],
session=session)
# Transfer limits
transfer_limits = rse.get('transfer_limits')
if transfer_limits:
for limit in transfer_limits:
old_transfer_limits = rse_module.get_rse_transfer_limits(
rse=rse_name, activity=limit, session=session)
if limit in old_transfer_limits:
rse_module.delete_rse_transfer_limits(rse=rse_name,
activity=limit,
session=session)
max_transfers = transfer_limits[limit].items(
)[0][1]['max_transfers']
rse_module.set_rse_transfer_limits(
rse=rse_name,
activity=limit,
max_transfers=max_transfers,
session=session)
# Attributes
attributes = rse.get('attributes')
if attributes:
old_attributes = rse_module.list_rse_attributes(
rse=rse_name, session=session)
for attr in attributes:
if attr in old_attributes:
rse_module.del_rse_attribute(rse=rse_name,
key=attr,
session=session)
rse_module.add_rse_attribute(rse=rse_name,
key=attr,
value=attributes[attr],
session=session)
# Distances
distances = data.get('distances')
if distances:
for src_rse_name in distances:
src = rse_module.get_rse_id(src_rse_name, session=session)
for dest_rse_name in distances[src_rse_name]:
dest = rse_module.get_rse_id(dest_rse_name, session=session)
distance = distances[src_rse_name][dest_rse_name]
del distance['src_rse_id']
del distance['dest_rse_id']
old_distance = distance_module.get_distances(src_rse_id=src,
dest_rse_id=dest,
session=session)
if old_distance:
distance_module.update_distances(src_rse_id=src,
dest_rse_id=dest,
parameters=distance,
session=session)
else:
distance_module.add_distance(
src_rse_id=src,
dest_rse_id=dest,
ranking=distance.get('ranking'),
agis_distance=distance.get('agis_distance'),
geoip_distance=distance.get('geoip_distance'),
active=distance.get('active'),
submitted=distance.get('submitted'),
transfer_speed=distance.get('transfer_speed'),
finished=distance.get('finished'),
failed=distance.get('failed'),
session=session)
def test_importer_rest(self):
""" IMPORTER (REST): test import. """
mw = []
headers1 = {
'X-Rucio-Account': 'root',
'X-Rucio-Username': '******',
'X-Rucio-Password': '******'
}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass',
headers=headers1,
expect_errors=True)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {
'X-Rucio-Type': 'user',
'X-Rucio-Account': 'root',
'X-Rucio-Auth-Token': str(token)
}
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data1))
assert_equal(r2.status, 201)
# RSE that not existed before
rse = get_rse(self.new_rse)
assert_equal(rse['availability'], 5)
assert_equal(rse['city'], 'NewCity')
assert_equal(rse['rse_type'], RSEType.TAPE)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in get_rse_protocols(self.new_rse)['protocols']]
assert_true({
'scheme': 'scheme',
'hostname': 'hostname',
'port': 1000
} in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['limit1'], 0)
transfer_limits = get_rse_transfer_limits(rse=self.new_rse)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.new_rse)]['max_transfers'], 1)
# RSE 1 that already existed before
rse = get_rse(self.old_rse_1)
assert_equal(rse['rse'], self.old_rse_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol['impl'],
'prefix': protocol['prefix']
} for protocol in get_rse_protocols(self.old_rse_1)['protocols']]
assert_true({
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'prefix': 'prefix',
'impl': 'impl1'
} in protocols)
assert_true({
'scheme': 'scheme2',
'hostname': 'hostname2',
'port': 1001,
'impl': 'impl',
'prefix': ''
} in protocols)
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['limit1'], 0)
assert_equal(limits['limit2'], 2)
transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.old_rse_1)]['max_transfers'], 1)
assert_equal(
transfer_limits['activity2'][get_rse_id(
self.old_rse_1)]['max_transfers'], 2)
# Distances
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data2))
assert_equal(r2.status, 201)
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data3))
assert_equal(r2.status, 201)
def test_importer_client(self):
""" IMPORTER (CLIENT): test import. """
import_client = ImportClient()
import_client.import_data(data=self.data1)
# RSE that had not existed before
rse = get_rse(self.new_rse)
assert_equal(rse['availability'], 5)
assert_equal(rse['city'], 'NewCity')
assert_equal(rse['rse_type'], RSEType.TAPE)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in get_rse_protocols(self.new_rse)['protocols']]
assert_true({
'scheme': 'scheme',
'hostname': 'hostname',
'port': 1000
} in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['limit1'], 0)
transfer_limits = get_rse_transfer_limits(rse=self.new_rse)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.new_rse)]['max_transfers'], 1)
# RSE 1 that already exists
rse = get_rse(self.old_rse_1)
assert_equal(rse['rse'], self.old_rse_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol['impl'],
'prefix': protocol['prefix']
} for protocol in get_rse_protocols(self.old_rse_1)['protocols']]
assert_true({
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'prefix': 'prefix',
'impl': 'impl1'
} in protocols)
assert_true({
'scheme': 'scheme2',
'hostname': 'hostname2',
'port': 1001,
'impl': 'impl',
'prefix': ''
} in protocols)
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['limit1'], 0)
assert_equal(limits['limit2'], 2)
transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.old_rse_1)]['max_transfers'], 1)
assert_equal(
transfer_limits['activity2'][get_rse_id(
self.old_rse_1)]['max_transfers'], 2)
# Distances
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
import_client.import_data(data=self.data2)
import_client.import_data(data=self.data3)
