class TestRSEExpressionParserClient(object): def __init__(self): self.rse1 = rse_name_generator() self.rse2 = rse_name_generator() self.rse3 = rse_name_generator() self.rse4 = rse_name_generator() self.rse5 = rse_name_generator() self.rse1_id = rse.add_rse(self.rse1) self.rse2_id = rse.add_rse(self.rse2) self.rse3_id = rse.add_rse(self.rse3) self.rse4_id = rse.add_rse(self.rse4) self.rse5_id = rse.add_rse(self.rse5) # Add Attributes self.attribute = attribute_name_generator() rse.add_rse_attribute(self.rse1_id, self.attribute, "at") rse.add_rse_attribute(self.rse2_id, self.attribute, "de") rse.add_rse_attribute(self.rse3_id, self.attribute, "fr") rse.add_rse_attribute(self.rse4_id, self.attribute, "uk") rse.add_rse_attribute(self.rse5_id, self.attribute, "us") # Add Tags self.tag1 = tag_generator() self.tag2 = tag_generator() rse.add_rse_attribute(self.rse1_id, self.tag1, True) rse.add_rse_attribute(self.rse2_id, self.tag1, True) rse.add_rse_attribute(self.rse3_id, self.tag1, True) rse.add_rse_attribute(self.rse4_id, self.tag2, True) rse.add_rse_attribute(self.rse5_id, self.tag2, True) self.rse_client = RSEClient() def test_complicated_expression(self): """ RSE_EXPRESSION_PARSER (CLIENT) Test some complicated expression""" rses = [ item['rse'] for item in self.rse_client.list_rses("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" % (self.tag1, self.attribute, self.tag2, self.attribute, self.attribute)) ] assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse5])) def test_complicated_expression_1(self): """ RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 1""" rses = [ item['rse'] for item in self.rse_client.list_rses( "(%s|%s)\\%s|%s&%s" % (self.tag1, self.tag2, self.tag2, self.tag2, self.tag1)) ] assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse3]))
class TestRSEExpressionParserClient(unittest.TestCase): def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.rse1 = rse_name_generator() self.rse2 = rse_name_generator() self.rse3 = rse_name_generator() self.rse4 = rse_name_generator() self.rse5 = rse_name_generator() self.rse1_id = rse.add_rse(self.rse1, **self.vo) self.rse2_id = rse.add_rse(self.rse2, **self.vo) self.rse3_id = rse.add_rse(self.rse3, **self.vo) self.rse4_id = rse.add_rse(self.rse4, **self.vo) self.rse5_id = rse.add_rse(self.rse5, **self.vo) # Add Attributes self.attribute = attribute_name_generator() rse.add_rse_attribute(self.rse1_id, self.attribute, "at") rse.add_rse_attribute(self.rse2_id, self.attribute, "de") rse.add_rse_attribute(self.rse3_id, self.attribute, "fr") rse.add_rse_attribute(self.rse4_id, self.attribute, "uk") rse.add_rse_attribute(self.rse5_id, self.attribute, "us") # Add Tags self.tag1 = tag_generator() self.tag2 = tag_generator() rse.add_rse_attribute(self.rse1_id, self.tag1, True) rse.add_rse_attribute(self.rse2_id, self.tag1, True) rse.add_rse_attribute(self.rse3_id, self.tag1, True) rse.add_rse_attribute(self.rse4_id, self.tag2, True) rse.add_rse_attribute(self.rse5_id, self.tag2, True) self.rse_client = RSEClient() def test_complicated_expression(self): """ RSE_EXPRESSION_PARSER (CLIENT) Test some complicated expression""" rses = sorted([item['rse'] for item in self.rse_client.list_rses("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" % (self.tag1, self.attribute, self.tag2, self.attribute, self.attribute))]) expected = sorted([self.rse1, self.rse2, self.rse5]) assert rses == expected def test_complicated_expression_1(self): """ RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 1""" rses = sorted([item['rse'] for item in self.rse_client.list_rses("(%s|%s)\\%s|%s&%s" % (self.tag1, self.tag2, self.tag2, self.tag2, self.tag1))]) expected = sorted([self.rse1, self.rse2, self.rse3]) assert rses == expected
class TestRSEExpressionParserClient(): def setup(self): self.rse1 = rse_name_generator() self.rse2 = rse_name_generator() self.rse3 = rse_name_generator() self.rse4 = rse_name_generator() self.rse5 = rse_name_generator() self.rse1_id = rse.add_rse(self.rse1) self.rse2_id = rse.add_rse(self.rse2) self.rse3_id = rse.add_rse(self.rse3) self.rse4_id = rse.add_rse(self.rse4) self.rse5_id = rse.add_rse(self.rse5) # Add Attributes self.attribute = attribute_name_generator() rse.add_rse_attribute(self.rse1, self.attribute, "at") rse.add_rse_attribute(self.rse2, self.attribute, "de") rse.add_rse_attribute(self.rse3, self.attribute, "fr") rse.add_rse_attribute(self.rse4, self.attribute, "uk") rse.add_rse_attribute(self.rse5, self.attribute, "us") # Add Tags self.tag1 = tag_generator() self.tag2 = tag_generator() rse.add_rse_attribute(self.rse1, self.tag1, True) rse.add_rse_attribute(self.rse2, self.tag1, True) rse.add_rse_attribute(self.rse3, self.tag1, True) rse.add_rse_attribute(self.rse4, self.tag2, True) rse.add_rse_attribute(self.rse5, self.tag2, True) self.rse_client = RSEClient() def test_complicated_expression(self): """ RSE_EXPRESSION_PARSER (CLIENT) Test some complicated expression""" rses = [item['rse'] for item in self.rse_client.list_rses("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" % (self.tag1, self.attribute, self.tag2, self.attribute, self.attribute))] assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse5])) def test_complicated_expression_1(self): """ RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 1""" rses = [item['rse'] for item in self.rse_client.list_rses("(%s|%s)\\%s|%s&%s" % (self.tag1, self.tag2, self.tag2, self.tag2, self.tag1))] assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse3]))
def test_rses_at_different_vos(self): """ MULTI VO (CLIENT): Test that RSEs from 2nd vo don't interfere """ # Set up RSEs at two VOs rse_client = RSEClient() rse_str = ''.join(choice(ascii_uppercase) for x in range(10)) tst = 'TST_%s' % rse_str new = 'NEW_%s' % rse_str shr = 'SHR_%s' % rse_str rse_client.add_rse(tst) rse_client.add_rse(shr) add_rse(new, 'root', **self.new_vo) shr_id_new_original = add_rse(shr, 'root', **self.new_vo) # Accurate rse_id for shared RSE at 'new' # Check the cached rse-id from each VO does not interfere shr_id_tst = get_rse_id(shr, **self.vo) shr_id_new = get_rse_id(shr, **self.new_vo) assert_equal(shr_id_new, shr_id_new_original) assert_not_equal(shr_id_new, shr_id_tst) # Check that when listing RSEs we only get RSEs for our VO rse_list_tst = [r['rse'] for r in rse_client.list_rses()] rse_list_new = [r['rse'] for r in list_rses(filters={}, **self.new_vo)] assert_true(tst in rse_list_tst) assert_false(new in rse_list_tst) assert_true(shr in rse_list_tst) assert_false(tst in rse_list_new) assert_true(new in rse_list_new) assert_true(shr in rse_list_new) # Check the cached attribute-value results do not interfere and only give results from the appropriate VO attribute_value = generate_uuid() add_rse_attribute(new, 'test', attribute_value, 'root', **self.new_vo) rses_tst_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo)) rses_new_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo)) rses_tst_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo)) rses_new_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo)) assert_equal(len(rses_tst_1), 0) assert_not_equal(len(rses_new_1), 0) assert_equal(len(rses_tst_2), 0) assert_not_equal(len(rses_new_2), 0) # check parse_expression rses_tst_3 = parse_expression(shr, filter={'vo': self.vo['vo']}) rses_tst_4 = parse_expression(tst, filter={'vo': self.vo['vo']}) rses_new_3 = parse_expression(shr, filter={'vo': self.new_vo['vo']}) with assert_raises(InvalidRSEExpression): parse_expression(tst, filter={'vo': self.new_vo['vo']}) assert_equal(len(rses_tst_3), 1) assert_equal(shr_id_tst, rses_tst_3[0]['id']) assert_equal(len(rses_tst_4), 1) assert_equal(tst, rses_tst_4[0]['rse']) assert_equal(len(rses_new_3), 1) assert_equal(shr_id_new, rses_new_3[0]['id'])
from rucio.client.rseclient import RSEClient import uuid c = RSEClient(account="ivm") for rse in c.list_rses(): u = uuid.UUID(rse["id"]) print rse["rse"], u.hex
from rucio.client.rseclient import RSEClient from rucio.client.didclient import DIDClient from rucio.client.replicaclient import ReplicaClient from kafka import KafkaProducer from datetime import datetime rucio_account = 'navila' # DID Client didclient = DIDClient(account=rucio_account) # Replica Client replicaclient = ReplicaClient(account=rucio_account) # RSE Client rseclient = RSEClient(account=rucio_account) rselist = rseclient.list_rses() # kafka scripts logger = logging.getLogger(__name__) logging.basicConfig( level=logging.DEBUG, format='%(asctime)s [%(levelname)s] %(name)s - %(message)s') def vs(doc): return json.dumps(doc).encode(encoding='utf-8', errors='replace') kafka_bootstrap_servers = [ 'lssrv03.fnal.gov:9092', 'lssrv04.fnal.gov:9092', 'lssrv05.fnal.gov:9092' ]