def test_importer_core(self): """ IMPORTER (CORE): test import. """ import_data(data=deepcopy(self.data1)) # RSE that had not existed before check_rse(self.new_rse, self.data1['rses']) check_protocols(self.new_rse, self.data1['rses']) new_rse_id = get_rse_id(rse=self.new_rse) attributes = list_rse_attributes(rse_id=new_rse_id) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse_id=new_rse_id) assert_equal(limits['MinFreeSpace'], 20000) # RSE 1 that already exists check_rse(self.old_rse_1, self.data1['rses']) # one protocol should be created, one should be updated check_protocols(self.old_rse_1, self.data1['rses']) # one protocol should be removed as it is not specified in the import data protocols = get_rse_protocols(self.old_rse_id_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in protocols['protocols']] assert_true({ 'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3' } not in protocols) attributes = list_rse_attributes(rse_id=self.old_rse_id_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse_id=self.old_rse_id_1) assert_equal(limits['MaxBeingDeletedFiles'], 1000) assert_equal(limits['MinFreeSpace'], 10000) distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) self.check_accounts(self.data1['accounts']) # RSE 4 should be flagged as deleted as it is missing in the import data with assert_raises(RSENotFound): get_rse(rse_id=self.old_rse_id_4) import_data(data=self.data2) import_data(data=self.data3)
def test_importer_client(self): """ IMPORTER (CLIENT): test import. """ import_client = ImportClient() import_client.import_data(data=deepcopy(self.data1)) # RSE that had not existed before check_rse(self.new_rse, self.data1['rses']) check_protocols(self.new_rse, self.data1['rses']) new_rse_id = get_rse_id(rse=self.new_rse) protocols = get_rse_protocols(self.old_rse_id_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in protocols['protocols']] assert_true({ 'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3' } not in protocols) attributes = list_rse_attributes(rse_id=new_rse_id) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse_id=new_rse_id) assert_equal(limits['MinFreeSpace'], 20000) # RSE 1 that already exists check_rse(self.old_rse_1, self.data1['rses']) check_protocols(self.old_rse_1, self.data1['rses']) attributes = list_rse_attributes(rse_id=self.old_rse_id_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse_id=self.old_rse_id_1) assert_equal(limits['MaxBeingDeletedFiles'], 1000) assert_equal(limits['MinFreeSpace'], 10000) distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) self.check_accounts(self.data1['accounts']) with assert_raises(RSENotFound): get_rse(rse_id=self.old_rse_id_4) import_client.import_data(data=self.data2) import_client.import_data(data=self.data3)
def test_importer_rest(self): """ IMPORTER (REST): test import. """ mw = [] headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'} r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) token = str(r1.header('X-Rucio-Auth-Token')) headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)} r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data1)) assert_equal(r2.status, 201) # RSE that not existed before check_rse(self.new_rse, self.data1['rses']) check_protocols(self.new_rse, self.data1['rses']) protocols = get_rse_protocols(self.old_rse_1) protocols = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port']} for protocol in protocols['protocols']] assert_true({'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3'} not in protocols) attributes = list_rse_attributes(rse=self.new_rse) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse=self.new_rse) assert_equal(limits['MinFreeSpace'], 20000) # RSE 1 that already existed before check_rse(self.old_rse_1, self.data1['rses']) check_protocols(self.old_rse_1, self.data1['rses']) attributes = list_rse_attributes(rse=self.old_rse_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse=self.old_rse_1) assert_equal(limits['MaxBeingDeletedFiles'], 1000) assert_equal(limits['MinFreeSpace'], 10000) distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) with assert_raises(RSENotFound): get_rse(self.old_rse_3) r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data2)) assert_equal(r2.status, 201) r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data3)) assert_equal(r2.status, 201)
def import_distances(distances, vo='def', session=None): for src_rse_name in distances: src = rse_module.get_rse_id(rse=src_rse_name, vo=vo, session=session) for dest_rse_name in distances[src_rse_name]: dest = rse_module.get_rse_id(rse=dest_rse_name, vo=vo, session=session) distance = distances[src_rse_name][dest_rse_name] if 'src_rse_id' in distance: del distance['src_rse_id'] if 'dest_rse_id' in distance: del distance['dest_rse_id'] old_distance = distance_module.get_distances(src_rse_id=src, dest_rse_id=dest, session=session) if old_distance: distance_module.update_distances(src_rse_id=src, dest_rse_id=dest, parameters=distance, session=session) else: distance_module.add_distance( src_rse_id=src, dest_rse_id=dest, ranking=distance.get('ranking'), agis_distance=distance.get('agis_distance'), geoip_distance=distance.get('geoip_distance'), active=distance.get('active'), submitted=distance.get('submitted'), transfer_speed=distance.get('transfer_speed'), finished=distance.get('finished'), failed=distance.get('failed'), session=session)
def test_two_sources_one_destination(db_session, vo, file, source_rse, source2_rse, mock_request): add_replicas(rse_id=source2_rse['id'], files=[file], account=mock_request.account, session=db_session) try: src1_distance, src2_distance = (get_distances( src_rse_id=src_rse, dest_rse_id=mock_request.dest_rse_id, session=db_session) for src_rse in (source_rse['id'], source2_rse['id'])) assert src1_distance and len( src1_distance) == 1 and src1_distance[0]['ranking'] == 5 assert src2_distance and len( src2_distance) == 1 and src2_distance[0]['ranking'] == 2 preparer.run_once(session=db_session) db_session.commit() updated_mock_request = db_session.query(models.Request).filter_by( id=mock_request.id).one() # type: models.Request assert updated_mock_request.state == RequestState.QUEUED assert updated_mock_request.source_rse_id == source2_rse[ 'id'] # distance 2 < 5 finally: delete_replicas(rse_id=source2_rse['id'], files=[file], session=db_session) db_session.commit()
def test_two_sources_one_destination(db_session, vo, file, mock_request): def setup(rse): add_distance(rse.rse_id, mock_request.dest_rse_id, ranking=2, session=rse.db_session) add_replicas(rse_id=rse.rse_id, files=[file], account=mock_request.account, session=rse.db_session) with GeneratedRSE(vo=vo, db_session=db_session, setup_func=setup) as source2_rse: src1_distance, src2_distance = ( get_distances( src_rse_id=src_rse, dest_rse_id=mock_request.dest_rse_id, session=db_session, ) for src_rse in (mock_request.source_rse_id, source2_rse.rse_id) ) assert src1_distance and len(src1_distance) == 1 and src1_distance[0]['ranking'] == 5 assert src2_distance and len(src2_distance) == 1 and src2_distance[0]['ranking'] == 2 preparer.run_once(session=db_session, logger=print) db_session.commit() updated_mock_request = ( db_session.query(models.Request).filter_by(id=mock_request.id).one() ) # type: models.Request assert updated_mock_request.state == RequestState.QUEUED assert updated_mock_request.source_rse_id == source2_rse.rse_id # distance 2 < 5 delete_replicas(rse_id=source2_rse.rse_id, files=[file], session=db_session)
def setup(self): self.db_session = session.get_session() self.db_session.query(models.Distance).delete() self.db_session.commit() self.rse_1 = 'MOCK' self.rse_1_id = get_rse_id(self.rse_1) self.rse_2 = 'MOCK2' self.rse_2_id = get_rse_id(self.rse_2) ranking = 10 add_distance(self.rse_1_id, self.rse_2_id, ranking) self.distances = { self.rse_1: { self.rse_2: get_distances(self.rse_1_id, self.rse_2_id)[0] } } self.distances_core = { self.rse_1_id: { self.rse_2_id: get_distances(self.rse_1_id, self.rse_2_id)[0] } }
def get_distance(source, destination, issuer): """ Get distances between rses. :param source: The source RSE. :param destination: The destination RSE. :param issuer: The issuer account. :returns distance: List of dictionaries. """ return distance_module.get_distances(src_rse_id=rse_module.get_rse_id(source), dest_rse_id=rse_module.get_rse_id(destination))
def get_distance(source, destination, issuer, vo='def'): """ Get distances between rses. :param source: The source RSE. :param destination: The destination RSE. :param issuer: The issuer account. :param vo: The VO to act on. :returns distance: List of dictionaries. """ distances = distance_module.get_distances(src_rse_id=rse_module.get_rse_id(source, vo=vo), dest_rse_id=rse_module.get_rse_id(destination, vo=vo)) return [api_update_return_dict(d) for d in distances]
def get_distance(source, destination, issuer): """ Get distances between rses. :param source: The source RSE. :param destination: The destination RSE. :param issuer: The issuer account. :returns distance: List of dictionaries. """ distances = distance_module.get_distances( src_rse_id=rse_module.get_rse_id(source), dest_rse_id=rse_module.get_rse_id(destination)) for d in distances: if 'src_rse_id' in d and d['src_rse_id'] is not None: d['src_rse'] = rse_module.get_rse_name(rse_id=d['src_rse_id']) if 'dest_rse_id' in d and d['dest_rse_id'] is not None: d['dest_rse'] = rse_module.get_rse_name(rse_id=d['dest_rse_id']) return distances
def import_data(data, session=None): """ Import data to add and update records in Rucio. :param data: data to be imported as dictionary. :param session: database session in use. """ # RSEs rses = data.get('rses') if rses: for rse in rses: protocols = rse.get('protocols') if protocols: protocols = protocols.get('protocols') del rse['protocols'] rse_name = rse['rse'] del rse['rse'] if not rse_module.rse_exists(rse_name, session=session): rse_module.add_rse(rse_name, deterministic=rse.get('deterministic'), volatile=rse.get('volatile'), city=rse.get('city'), region_code=rse.get('region_code'), country_name=rse.get('country_name'), staging_area=rse.get('staging_area'), continent=rse.get('continent'), time_zone=rse.get('time_zone'), ISP=rse.get('ISP'), rse_type=rse.get('rse_type'), latitude=rse.get('latitude'), longitude=rse.get('longitude'), ASN=rse.get('ASN'), availability=rse.get('availability'), session=session) else: rse_module.update_rse(rse_name, rse, session=session) # Protocols if protocols: old_protocols = rse_module.get_rse_protocols(rse=rse_name, session=session) for protocol in protocols: scheme = protocol.get('scheme') hostname = protocol.get('hostname') port = protocol.get('port') intersection = [ old_protocol for old_protocol in old_protocols['protocols'] if old_protocol['scheme'] == scheme and old_protocol['hostname'] == hostname and old_protocol['port'] == port ] if intersection: del protocol['scheme'] del protocol['hostname'] del protocol['port'] rse_module.update_protocols(rse=rse_name, scheme=scheme, data=protocol, hostname=hostname, port=port, session=session) else: rse_module.add_protocol(rse=rse_name, parameter=protocol, session=session) # Limits limits = rse.get('limits') if limits: old_limits = rse_module.get_rse_limits(rse=rse_name, session=session) for limit in limits: if limit in old_limits: rse_module.delete_rse_limit(rse=rse_name, name=limit, session=session) rse_module.set_rse_limits(rse=rse_name, name=limit, value=limits[limit], session=session) # Transfer limits transfer_limits = rse.get('transfer_limits') if transfer_limits: for limit in transfer_limits: old_transfer_limits = rse_module.get_rse_transfer_limits( rse=rse_name, activity=limit, session=session) if limit in old_transfer_limits: rse_module.delete_rse_transfer_limits(rse=rse_name, activity=limit, session=session) max_transfers = transfer_limits[limit].items( )[0][1]['max_transfers'] rse_module.set_rse_transfer_limits( rse=rse_name, activity=limit, max_transfers=max_transfers, session=session) # Attributes attributes = rse.get('attributes') if attributes: old_attributes = rse_module.list_rse_attributes( rse=rse_name, session=session) for attr in attributes: if attr in old_attributes: rse_module.del_rse_attribute(rse=rse_name, key=attr, session=session) rse_module.add_rse_attribute(rse=rse_name, key=attr, value=attributes[attr], session=session) # Distances distances = data.get('distances') if distances: for src_rse_name in distances: src = rse_module.get_rse_id(src_rse_name, session=session) for dest_rse_name in distances[src_rse_name]: dest = rse_module.get_rse_id(dest_rse_name, session=session) distance = distances[src_rse_name][dest_rse_name] del distance['src_rse_id'] del distance['dest_rse_id'] old_distance = distance_module.get_distances(src_rse_id=src, dest_rse_id=dest, session=session) if old_distance: distance_module.update_distances(src_rse_id=src, dest_rse_id=dest, parameters=distance, session=session) else: distance_module.add_distance( src_rse_id=src, dest_rse_id=dest, ranking=distance.get('ranking'), agis_distance=distance.get('agis_distance'), geoip_distance=distance.get('geoip_distance'), active=distance.get('active'), submitted=distance.get('submitted'), transfer_speed=distance.get('transfer_speed'), finished=distance.get('finished'), failed=distance.get('failed'), session=session)
def test_importer_rest(self): """ IMPORTER (REST): test import. """ mw = [] headers1 = { 'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******' } r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) token = str(r1.header('X-Rucio-Auth-Token')) headers2 = { 'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token) } r2 = TestApp(import_app.wsgifunc(*mw)).post( '/', headers=headers2, expect_errors=True, params=render_json(**self.data1)) assert_equal(r2.status, 201) # RSE that not existed before rse = get_rse(self.new_rse) assert_equal(rse['availability'], 5) assert_equal(rse['city'], 'NewCity') assert_equal(rse['rse_type'], RSEType.TAPE) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in get_rse_protocols(self.new_rse)['protocols']] assert_true({ 'scheme': 'scheme', 'hostname': 'hostname', 'port': 1000 } in protocols) attributes = list_rse_attributes(rse=self.new_rse) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse=self.new_rse) assert_equal(limits['limit1'], 0) transfer_limits = get_rse_transfer_limits(rse=self.new_rse) assert_equal( transfer_limits['activity1'][get_rse_id( self.new_rse)]['max_transfers'], 1) # RSE 1 that already existed before rse = get_rse(self.old_rse_1) assert_equal(rse['rse'], self.old_rse_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl'], 'prefix': protocol['prefix'] } for protocol in get_rse_protocols(self.old_rse_1)['protocols']] assert_true({ 'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'prefix': 'prefix', 'impl': 'impl1' } in protocols) assert_true({ 'scheme': 'scheme2', 'hostname': 'hostname2', 'port': 1001, 'impl': 'impl', 'prefix': '' } in protocols) attributes = list_rse_attributes(rse=self.old_rse_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse=self.old_rse_1) assert_equal(limits['limit1'], 0) assert_equal(limits['limit2'], 2) transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1) assert_equal( transfer_limits['activity1'][get_rse_id( self.old_rse_1)]['max_transfers'], 1) assert_equal( transfer_limits['activity2'][get_rse_id( self.old_rse_1)]['max_transfers'], 2) # Distances distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) r2 = TestApp(import_app.wsgifunc(*mw)).post( '/', headers=headers2, expect_errors=True, params=render_json(**self.data2)) assert_equal(r2.status, 201) r2 = TestApp(import_app.wsgifunc(*mw)).post( '/', headers=headers2, expect_errors=True, params=render_json(**self.data3)) assert_equal(r2.status, 201)
def test_importer_client(self): """ IMPORTER (CLIENT): test import. """ import_client = ImportClient() import_client.import_data(data=self.data1) # RSE that had not existed before rse = get_rse(self.new_rse) assert_equal(rse['availability'], 5) assert_equal(rse['city'], 'NewCity') assert_equal(rse['rse_type'], RSEType.TAPE) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in get_rse_protocols(self.new_rse)['protocols']] assert_true({ 'scheme': 'scheme', 'hostname': 'hostname', 'port': 1000 } in protocols) attributes = list_rse_attributes(rse=self.new_rse) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse=self.new_rse) assert_equal(limits['limit1'], 0) transfer_limits = get_rse_transfer_limits(rse=self.new_rse) assert_equal( transfer_limits['activity1'][get_rse_id( self.new_rse)]['max_transfers'], 1) # RSE 1 that already exists rse = get_rse(self.old_rse_1) assert_equal(rse['rse'], self.old_rse_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl'], 'prefix': protocol['prefix'] } for protocol in get_rse_protocols(self.old_rse_1)['protocols']] assert_true({ 'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'prefix': 'prefix', 'impl': 'impl1' } in protocols) assert_true({ 'scheme': 'scheme2', 'hostname': 'hostname2', 'port': 1001, 'impl': 'impl', 'prefix': '' } in protocols) attributes = list_rse_attributes(rse=self.old_rse_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse=self.old_rse_1) assert_equal(limits['limit1'], 0) assert_equal(limits['limit2'], 2) transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1) assert_equal( transfer_limits['activity1'][get_rse_id( self.old_rse_1)]['max_transfers'], 1) assert_equal( transfer_limits['activity2'][get_rse_id( self.old_rse_1)]['max_transfers'], 2) # Distances distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) import_client.import_data(data=self.data2) import_client.import_data(data=self.data3)