class TestRSEExpressionParserClient(object):
def __init__(self):
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1)
self.rse2_id = rse.add_rse(self.rse2)
self.rse3_id = rse.add_rse(self.rse3)
self.rse4_id = rse.add_rse(self.rse4)
self.rse5_id = rse.add_rse(self.rse5)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1_id, self.attribute, "at")
rse.add_rse_attribute(self.rse2_id, self.attribute, "de")
rse.add_rse_attribute(self.rse3_id, self.attribute, "fr")
rse.add_rse_attribute(self.rse4_id, self.attribute, "uk")
rse.add_rse_attribute(self.rse5_id, self.attribute, "us")
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1_id, self.tag1, True)
rse.add_rse_attribute(self.rse2_id, self.tag1, True)
rse.add_rse_attribute(self.rse3_id, self.tag1, True)
rse.add_rse_attribute(self.rse4_id, self.tag2, True)
rse.add_rse_attribute(self.rse5_id, self.tag2, True)
self.rse_client = RSEClient()
def test_complicated_expression(self):
""" RSE_EXPRESSION_PARSER (CLIENT) Test some complicated expression"""
rses = [
item['rse'] for item in
self.rse_client.list_rses("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" %
(self.tag1, self.attribute, self.tag2,
self.attribute, self.attribute))
]
assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse5]))
def test_complicated_expression_1(self):
""" RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 1"""
rses = [
item['rse'] for item in self.rse_client.list_rses(
"(%s|%s)\\%s|%s&%s" %
(self.tag1, self.tag2, self.tag2, self.tag2, self.tag1))
]
assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse3]))
class TestRSEExpressionParserClient(unittest.TestCase):
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1, **self.vo)
self.rse2_id = rse.add_rse(self.rse2, **self.vo)
self.rse3_id = rse.add_rse(self.rse3, **self.vo)
self.rse4_id = rse.add_rse(self.rse4, **self.vo)
self.rse5_id = rse.add_rse(self.rse5, **self.vo)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1_id, self.attribute, "at")
rse.add_rse_attribute(self.rse2_id, self.attribute, "de")
rse.add_rse_attribute(self.rse3_id, self.attribute, "fr")
rse.add_rse_attribute(self.rse4_id, self.attribute, "uk")
rse.add_rse_attribute(self.rse5_id, self.attribute, "us")
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1_id, self.tag1, True)
rse.add_rse_attribute(self.rse2_id, self.tag1, True)
rse.add_rse_attribute(self.rse3_id, self.tag1, True)
rse.add_rse_attribute(self.rse4_id, self.tag2, True)
rse.add_rse_attribute(self.rse5_id, self.tag2, True)
self.rse_client = RSEClient()
def test_complicated_expression(self):
""" RSE_EXPRESSION_PARSER (CLIENT) Test some complicated expression"""
rses = sorted([item['rse'] for item in self.rse_client.list_rses("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" % (self.tag1, self.attribute, self.tag2, self.attribute, self.attribute))])
expected = sorted([self.rse1, self.rse2, self.rse5])
assert rses == expected
def test_complicated_expression_1(self):
""" RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 1"""
rses = sorted([item['rse'] for item in self.rse_client.list_rses("(%s|%s)\\%s|%s&%s" % (self.tag1, self.tag2, self.tag2, self.tag2, self.tag1))])
expected = sorted([self.rse1, self.rse2, self.rse3])
assert rses == expected
class TestRSEExpressionParserClient():
def setup(self):
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1)
self.rse2_id = rse.add_rse(self.rse2)
self.rse3_id = rse.add_rse(self.rse3)
self.rse4_id = rse.add_rse(self.rse4)
self.rse5_id = rse.add_rse(self.rse5)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1, self.attribute, "at")
rse.add_rse_attribute(self.rse2, self.attribute, "de")
rse.add_rse_attribute(self.rse3, self.attribute, "fr")
rse.add_rse_attribute(self.rse4, self.attribute, "uk")
rse.add_rse_attribute(self.rse5, self.attribute, "us")
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1, self.tag1, True)
rse.add_rse_attribute(self.rse2, self.tag1, True)
rse.add_rse_attribute(self.rse3, self.tag1, True)
rse.add_rse_attribute(self.rse4, self.tag2, True)
rse.add_rse_attribute(self.rse5, self.tag2, True)
self.rse_client = RSEClient()
def test_complicated_expression(self):
""" RSE_EXPRESSION_PARSER (CLIENT) Test some complicated expression"""
rses = [item['rse'] for item in self.rse_client.list_rses("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" % (self.tag1, self.attribute, self.tag2, self.attribute, self.attribute))]
assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse5]))
def test_complicated_expression_1(self):
""" RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 1"""
rses = [item['rse'] for item in self.rse_client.list_rses("(%s|%s)\\%s|%s&%s" % (self.tag1, self.tag2, self.tag2, self.tag2, self.tag1))]
assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse3]))
def test_rses_at_different_vos(self):
""" MULTI VO (CLIENT): Test that RSEs from 2nd vo don't interfere """
# Set up RSEs at two VOs
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst = 'TST_%s' % rse_str
new = 'NEW_%s' % rse_str
shr = 'SHR_%s' % rse_str
rse_client.add_rse(tst)
rse_client.add_rse(shr)
add_rse(new, 'root', **self.new_vo)
shr_id_new_original = add_rse(shr, 'root', **self.new_vo) # Accurate rse_id for shared RSE at 'new'
# Check the cached rse-id from each VO does not interfere
shr_id_tst = get_rse_id(shr, **self.vo)
shr_id_new = get_rse_id(shr, **self.new_vo)
assert_equal(shr_id_new, shr_id_new_original)
assert_not_equal(shr_id_new, shr_id_tst)
# Check that when listing RSEs we only get RSEs for our VO
rse_list_tst = [r['rse'] for r in rse_client.list_rses()]
rse_list_new = [r['rse'] for r in list_rses(filters={}, **self.new_vo)]
assert_true(tst in rse_list_tst)
assert_false(new in rse_list_tst)
assert_true(shr in rse_list_tst)
assert_false(tst in rse_list_new)
assert_true(new in rse_list_new)
assert_true(shr in rse_list_new)
# Check the cached attribute-value results do not interfere and only give results from the appropriate VO
attribute_value = generate_uuid()
add_rse_attribute(new, 'test', attribute_value, 'root', **self.new_vo)
rses_tst_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo))
rses_new_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo))
rses_tst_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo))
rses_new_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo))
assert_equal(len(rses_tst_1), 0)
assert_not_equal(len(rses_new_1), 0)
assert_equal(len(rses_tst_2), 0)
assert_not_equal(len(rses_new_2), 0)
# check parse_expression
rses_tst_3 = parse_expression(shr, filter={'vo': self.vo['vo']})
rses_tst_4 = parse_expression(tst, filter={'vo': self.vo['vo']})
rses_new_3 = parse_expression(shr, filter={'vo': self.new_vo['vo']})
with assert_raises(InvalidRSEExpression):
parse_expression(tst, filter={'vo': self.new_vo['vo']})
assert_equal(len(rses_tst_3), 1)
assert_equal(shr_id_tst, rses_tst_3[0]['id'])
assert_equal(len(rses_tst_4), 1)
assert_equal(tst, rses_tst_4[0]['rse'])
assert_equal(len(rses_new_3), 1)
assert_equal(shr_id_new, rses_new_3[0]['id'])
from rucio.client.rseclient import RSEClient
import uuid
c = RSEClient(account="ivm")
for rse in c.list_rses():
u = uuid.UUID(rse["id"])
print rse["rse"], u.hex
from rucio.client.rseclient import RSEClient
from rucio.client.didclient import DIDClient
from rucio.client.replicaclient import ReplicaClient
from kafka import KafkaProducer
from datetime import datetime
rucio_account = 'navila'
# DID Client
didclient = DIDClient(account=rucio_account)
# Replica Client
replicaclient = ReplicaClient(account=rucio_account)
# RSE Client
rseclient = RSEClient(account=rucio_account)
rselist = rseclient.list_rses()
# kafka scripts
logger = logging.getLogger(__name__)
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s [%(levelname)s] %(name)s - %(message)s')
def vs(doc):
return json.dumps(doc).encode(encoding='utf-8', errors='replace')
kafka_bootstrap_servers = [
'lssrv03.fnal.gov:9092', 'lssrv04.fnal.gov:9092', 'lssrv05.fnal.gov:9092'
]
