def __init__(self):
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1)
self.rse2_id = rse.add_rse(self.rse2)
self.rse3_id = rse.add_rse(self.rse3)
self.rse4_id = rse.add_rse(self.rse4)
self.rse5_id = rse.add_rse(self.rse5)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1_id, self.attribute, "at")
rse.add_rse_attribute(self.rse2_id, self.attribute, "de")
rse.add_rse_attribute(self.rse3_id, self.attribute, "fr")
rse.add_rse_attribute(self.rse4_id, self.attribute, "uk")
rse.add_rse_attribute(self.rse5_id, self.attribute, "us")
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1_id, self.tag1, True)
rse.add_rse_attribute(self.rse2_id, self.tag1, True)
rse.add_rse_attribute(self.rse3_id, self.tag1, True)
rse.add_rse_attribute(self.rse4_id, self.tag2, True)
rse.add_rse_attribute(self.rse5_id, self.tag2, True)
self.rse_client = RSEClient()
def get_rse_client(rse, vo='def', **kwarg):
'''
get_rse_client
'''
from rucio.client.rseclient import RSEClient
client = RSEClient(vo=vo)
return client.get_rse(rse)
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def setUp(self):
self.account_client = AccountClient()
self.rse_client = RSEClient()
self.account = generate_uuid()[:10]
self.rse = rse_name_generator()
self.account_client.add_account(self.account, 'SERVICE',
'*****@*****.**')
self.rse_client.add_rse(self.rse)
def __lfns2pfns_client(self, lfns):
""" Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if neccessary.
:param scope: list of DIDs
:returns: dict with scope:name as keys and PFN as value (in case of errors the Rucio exception si assigned to the key)
"""
client = RSEClient()
lfns = [lfns] if isinstance(lfns, dict) else lfns
lfn_query = ["%s:%s" % (lfn['scope'], lfn['name']) for lfn in lfns]
return client.lfns2pfns(self.rse['rse'], lfn_query, scheme=self.attributes['scheme'])
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def test_rses_at_different_vos(self):
""" MULTI VO (CLIENT): Test that RSEs from 2nd vo don't interfere """
# Set up RSEs at two VOs
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst = 'TST_%s' % rse_str
new = 'NEW_%s' % rse_str
shr = 'SHR_%s' % rse_str
rse_client.add_rse(tst)
rse_client.add_rse(shr)
add_rse(new, 'root', **self.new_vo)
shr_id_new_original = add_rse(shr, 'root', **self.new_vo) # Accurate rse_id for shared RSE at 'new'
# Check the cached rse-id from each VO does not interfere
shr_id_tst = get_rse_id(shr, **self.vo)
shr_id_new = get_rse_id(shr, **self.new_vo)
assert_equal(shr_id_new, shr_id_new_original)
assert_not_equal(shr_id_new, shr_id_tst)
# Check that when listing RSEs we only get RSEs for our VO
rse_list_tst = [r['rse'] for r in rse_client.list_rses()]
rse_list_new = [r['rse'] for r in list_rses(filters={}, **self.new_vo)]
assert_true(tst in rse_list_tst)
assert_false(new in rse_list_tst)
assert_true(shr in rse_list_tst)
assert_false(tst in rse_list_new)
assert_true(new in rse_list_new)
assert_true(shr in rse_list_new)
# Check the cached attribute-value results do not interfere and only give results from the appropriate VO
attribute_value = generate_uuid()
add_rse_attribute(new, 'test', attribute_value, 'root', **self.new_vo)
rses_tst_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo))
rses_new_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo))
rses_tst_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo))
rses_new_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo))
assert_equal(len(rses_tst_1), 0)
assert_not_equal(len(rses_new_1), 0)
assert_equal(len(rses_tst_2), 0)
assert_not_equal(len(rses_new_2), 0)
# check parse_expression
rses_tst_3 = parse_expression(shr, filter={'vo': self.vo['vo']})
rses_tst_4 = parse_expression(tst, filter={'vo': self.vo['vo']})
rses_new_3 = parse_expression(shr, filter={'vo': self.new_vo['vo']})
with assert_raises(InvalidRSEExpression):
parse_expression(tst, filter={'vo': self.new_vo['vo']})
assert_equal(len(rses_tst_3), 1)
assert_equal(shr_id_tst, rses_tst_3[0]['id'])
assert_equal(len(rses_tst_4), 1)
assert_equal(tst, rses_tst_4[0]['rse'])
assert_equal(len(rses_new_3), 1)
assert_equal(shr_id_new, rses_new_3[0]['id'])
class TestRSEExpressionParserClient(object):
def __init__(self):
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1)
self.rse2_id = rse.add_rse(self.rse2)
self.rse3_id = rse.add_rse(self.rse3)
self.rse4_id = rse.add_rse(self.rse4)
self.rse5_id = rse.add_rse(self.rse5)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1_id, self.attribute, "at")
rse.add_rse_attribute(self.rse2_id, self.attribute, "de")
rse.add_rse_attribute(self.rse3_id, self.attribute, "fr")
rse.add_rse_attribute(self.rse4_id, self.attribute, "uk")
rse.add_rse_attribute(self.rse5_id, self.attribute, "us")
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1_id, self.tag1, True)
rse.add_rse_attribute(self.rse2_id, self.tag1, True)
rse.add_rse_attribute(self.rse3_id, self.tag1, True)
rse.add_rse_attribute(self.rse4_id, self.tag2, True)
rse.add_rse_attribute(self.rse5_id, self.tag2, True)
self.rse_client = RSEClient()
def test_complicated_expression(self):
""" RSE_EXPRESSION_PARSER (CLIENT) Test some complicated expression"""
rses = [
item['rse'] for item in
self.rse_client.list_rses("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" %
(self.tag1, self.attribute, self.tag2,
self.attribute, self.attribute))
]
assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse5]))
def test_complicated_expression_1(self):
""" RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 1"""
rses = [
item['rse'] for item in self.rse_client.list_rses(
"(%s|%s)\\%s|%s&%s" %
(self.tag1, self.tag2, self.tag2, self.tag2, self.tag1))
]
assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse3]))
class TestRSEExpressionParserClient(unittest.TestCase):
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1, **self.vo)
self.rse2_id = rse.add_rse(self.rse2, **self.vo)
self.rse3_id = rse.add_rse(self.rse3, **self.vo)
self.rse4_id = rse.add_rse(self.rse4, **self.vo)
self.rse5_id = rse.add_rse(self.rse5, **self.vo)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1_id, self.attribute, "at")
rse.add_rse_attribute(self.rse2_id, self.attribute, "de")
rse.add_rse_attribute(self.rse3_id, self.attribute, "fr")
rse.add_rse_attribute(self.rse4_id, self.attribute, "uk")
rse.add_rse_attribute(self.rse5_id, self.attribute, "us")
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1_id, self.tag1, True)
rse.add_rse_attribute(self.rse2_id, self.tag1, True)
rse.add_rse_attribute(self.rse3_id, self.tag1, True)
rse.add_rse_attribute(self.rse4_id, self.tag2, True)
rse.add_rse_attribute(self.rse5_id, self.tag2, True)
self.rse_client = RSEClient()
def test_complicated_expression(self):
""" RSE_EXPRESSION_PARSER (CLIENT) Test some complicated expression"""
rses = sorted([item['rse'] for item in self.rse_client.list_rses("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" % (self.tag1, self.attribute, self.tag2, self.attribute, self.attribute))])
expected = sorted([self.rse1, self.rse2, self.rse5])
assert rses == expected
def test_complicated_expression_1(self):
""" RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 1"""
rses = sorted([item['rse'] for item in self.rse_client.list_rses("(%s|%s)\\%s|%s&%s" % (self.tag1, self.tag2, self.tag2, self.tag2, self.tag1))])
expected = sorted([self.rse1, self.rse2, self.rse3])
assert rses == expected
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def setup(self):
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1)
self.rse2_id = rse.add_rse(self.rse2)
self.rse3_id = rse.add_rse(self.rse3)
self.rse4_id = rse.add_rse(self.rse4)
self.rse5_id = rse.add_rse(self.rse5)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1, self.attribute, "at")
rse.add_rse_attribute(self.rse2, self.attribute, "de")
rse.add_rse_attribute(self.rse3, self.attribute, "fr")
rse.add_rse_attribute(self.rse4, self.attribute, "uk")
rse.add_rse_attribute(self.rse5, self.attribute, "us")
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1, self.tag1, True)
rse.add_rse_attribute(self.rse2, self.tag1, True)
rse.add_rse_attribute(self.rse3, self.tag1, True)
rse.add_rse_attribute(self.rse4, self.tag2, True)
rse.add_rse_attribute(self.rse5, self.tag2, True)
self.rse_client = RSEClient()
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
self.vo = {}
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1, **self.vo)
self.rse2_id = rse.add_rse(self.rse2, **self.vo)
self.rse3_id = rse.add_rse(self.rse3, **self.vo)
self.rse4_id = rse.add_rse(self.rse4, **self.vo)
self.rse5_id = rse.add_rse(self.rse5, **self.vo)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1_id, self.attribute, "at")
rse.add_rse_attribute(self.rse2_id, self.attribute, "de")
rse.add_rse_attribute(self.rse3_id, self.attribute, "fr")
rse.add_rse_attribute(self.rse4_id, self.attribute, "uk")
rse.add_rse_attribute(self.rse5_id, self.attribute, "us")
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1_id, self.tag1, True)
rse.add_rse_attribute(self.rse2_id, self.tag1, True)
rse.add_rse_attribute(self.rse3_id, self.tag1, True)
rse.add_rse_attribute(self.rse4_id, self.tag2, True)
rse.add_rse_attribute(self.rse5_id, self.tag2, True)
self.rse_client = RSEClient()
def test_account_counters_at_different_vos(self):
""" MULTI VO (CLIENT): Test that account counters from 2nd vo don't interfere """
session = db_session.get_session()
# add some RSEs to test create_counters_for_new_account
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst_rse1 = 'TST1_%s' % rse_str
new_rse1 = 'NEW1_%s' % rse_str
rse_client.add_rse(tst_rse1)
add_rse(new_rse1, 'root', **self.new_vo)
# add an account - should have counters created for RSEs on the same VO
usr_uuid = str(generate_uuid()).lower()[:16]
new_acc_str = 'shr-%s' % usr_uuid
new_acc = InternalAccount(new_acc_str, **self.new_vo)
add_account(new_acc_str, 'USER', '*****@*****.**', 'root', **self.new_vo)
query = session.query(models.AccountUsage.account, models.AccountUsage.rse_id).\
distinct(models.AccountUsage.account, models.AccountUsage.rse_id).\
filter_by(account=new_acc)
acc_counters = list(query.all())
assert_not_equal(0, len(acc_counters))
for counter in acc_counters:
rse_id = counter[1]
vo = get_rse_vo(rse_id)
assert_equal(vo, self.new_vo['vo'])
# add an RSE - should have counters created for accounts on the same VO
new_rse2 = 'NEW2_' + rse_str
new_rse2_id = add_rse(new_rse2, 'root', **self.new_vo)
query = session.query(models.AccountUsage.account, models.AccountUsage.rse_id).\
distinct(models.AccountUsage.account, models.AccountUsage.rse_id).\
filter_by(rse_id=new_rse2_id)
rse_counters = list(query.all())
assert_not_equal(0, len(rse_counters))
for counter in rse_counters:
account = counter[0]
assert_equal(account.vo, self.new_vo['vo'])
session.commit()
class TestRSEExpressionParserClient():
def setup(self):
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1)
self.rse2_id = rse.add_rse(self.rse2)
self.rse3_id = rse.add_rse(self.rse3)
self.rse4_id = rse.add_rse(self.rse4)
self.rse5_id = rse.add_rse(self.rse5)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1, self.attribute, "at")
rse.add_rse_attribute(self.rse2, self.attribute, "de")
rse.add_rse_attribute(self.rse3, self.attribute, "fr")
rse.add_rse_attribute(self.rse4, self.attribute, "uk")
rse.add_rse_attribute(self.rse5, self.attribute, "us")
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1, self.tag1, True)
rse.add_rse_attribute(self.rse2, self.tag1, True)
rse.add_rse_attribute(self.rse3, self.tag1, True)
rse.add_rse_attribute(self.rse4, self.tag2, True)
rse.add_rse_attribute(self.rse5, self.tag2, True)
self.rse_client = RSEClient()
def test_complicated_expression(self):
""" RSE_EXPRESSION_PARSER (CLIENT) Test some complicated expression"""
rses = [item['rse'] for item in self.rse_client.list_rses("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" % (self.tag1, self.attribute, self.tag2, self.attribute, self.attribute))]
assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse5]))
def test_complicated_expression_1(self):
""" RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 1"""
rses = [item['rse'] for item in self.rse_client.list_rses("(%s|%s)\\%s|%s&%s" % (self.tag1, self.tag2, self.tag2, self.tag2, self.tag1))]
assert_equal(sorted(rses), sorted([self.rse1, self.rse2, self.rse3]))
def setupClass(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
self.vo = {}
self.rse_client = RSEClient()
self.tmp_rse_name = rse_name_generator()
self.rse_client.add_rse(self.tmp_rse_name, vo=self.vo)
self.tmp_rse = self.rse_client.get_rse(self.tmp_rse_name)['id']
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
cls.rse_client = RSEClient()
cls.tmp_rse_name = rse_name_generator()
cls.rse_client.add_rse(cls.tmp_rse_name, vo=cls.vo)
cls.tmp_rse = cls.rse_client.get_rse(cls.tmp_rse_name)['id']
def test_automatix(self):
""" MULTI VO (DAEMON): Test that automatix runs on a single VO """
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
shr_scope = 'shr_%s' % scope_uuid
scope_client.add_scope('root', shr_scope)
add_scope(shr_scope, 'root', 'root', **self.new_vo)
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
shr_rse = 'SHR_%s' % rse_str
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/automatix',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
rse_client.add_rse(shr_rse)
rse_client.add_rse_attribute(rse=shr_rse, key='verify_checksum', value=False)
rse_client.add_protocol(shr_rse, mock_protocol)
add_rse(shr_rse, 'root', **self.new_vo)
add_rse_attribute(rse=shr_rse, key='verify_checksum', value=False, issuer='root', **self.new_vo)
add_protocol(rse=shr_rse, data=mock_protocol, issuer='root', **self.new_vo)
automatix(sites=[shr_rse], inputfile='/opt/rucio/etc/automatix.json', sleep_time=30, account='root', once=True, scope=shr_scope)
did_list_tst = list(DIDClient().list_dids(shr_scope, {}))
did_list_new = list(list_dids(shr_scope, {}, **self.new_vo))
assert_not_equal(len(did_list_tst), 0)
assert_equal(len(did_list_new), 0)
did_dicts = [{'scope': shr_scope, 'name': n} for n in did_list_tst]
replicas_tst = list(ReplicaClient().list_replicas(did_dicts, rse_expression=shr_rse))
replicas_new = list(list_replicas(did_dicts, rse_expression=shr_rse, **self.new_vo))
assert_not_equal(len(replicas_tst), 0)
assert_equal(len(replicas_new), 0)
from rucio.client.rseclient import RSEClient
import uuid, sys, pprint
rse_name = sys.argv[1]
scheme = sys.argv[2]
c = RSEClient(account="ivm")
protocols = c.get_protocols(rse_name)
for proto in protocols:
if (scheme == "-" or proto.get("scheme")
== scheme) and "extended_attributes" in proto:
print "Scheme:", proto["scheme"]
print "TFC:"
pprint.pprint(proto["extended_attributes"]["tfc"])
class TestDIDClients:
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def test_list_dids(self):
""" DATA IDENTIFIERS (CLIENT): List dids by pattern."""
tmp_scope = scope_name_generator()
tmp_files = []
tmp_files.append('file_a_1%s' % generate_uuid())
tmp_files.append('file_a_2%s' % generate_uuid())
tmp_files.append('file_b_1%s' % generate_uuid())
tmp_rse = 'MOCK'
self.scope_client.add_scope('jdoe', tmp_scope)
for tmp_file in tmp_files:
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1, '0cc737eb')
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file_a_*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file_a_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 1)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file_*_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='file'):
results.append(result)
assert_equal(len(results), 3)
results = []
filters = {'name': 'file*', 'created_after': datetime.utcnow() - timedelta(hours=1)}
for result in self.did_client.list_dids(tmp_scope, filters):
results.append(result)
assert_equal(len(results), 0)
with assert_raises(UnsupportedOperation):
self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='whateverytype')
def test_list_recursive(self):
""" DATA IDENTIFIERS (CLIENT): List did recursive """
# Create nested containers and datast
tmp_scope_1 = 'list-did-recursive'
tmp_scope_2 = 'list-did-recursive-2'
self.scope_client.add_scope('root', tmp_scope_1)
self.scope_client.add_scope('root', tmp_scope_2)
tmp_container_1 = 'container_%s' % generate_uuid()
self.did_client.add_container(scope=tmp_scope_1, name=tmp_container_1)
tmp_container_2 = 'container_%s' % generate_uuid()
self.did_client.add_container(scope=tmp_scope_1, name=tmp_container_2)
tmp_dataset_1 = 'dataset_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope_2, name=tmp_dataset_1)
tmp_dataset_2 = 'dataset_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope_1, name=tmp_dataset_2)
self.did_client.attach_dids(scope=tmp_scope_1, name=tmp_container_1, dids=[{'scope': tmp_scope_2, 'name': tmp_dataset_1}])
self.did_client.attach_dids(scope=tmp_scope_1, name=tmp_container_2, dids=[{'scope': tmp_scope_1, 'name': tmp_dataset_2}])
self.did_client.attach_dids(scope=tmp_scope_1, name=tmp_container_1, dids=[{'scope': tmp_scope_1, 'name': tmp_container_2}])
# List DIDs not recursive - only the first container is expected
dids = [str(did) for did in self.did_client.list_dids(scope=tmp_scope_1, recursive=False, type='all', filters={'name': tmp_container_1})]
assert_equal(dids, [tmp_container_1])
# List DIDs recursive - first container and all attached collections are expected
dids = [str(did) for did in self.did_client.list_dids(scope=tmp_scope_1, recursive=True, type='all', filters={'name': tmp_container_1})]
assert_true(tmp_container_1 in dids)
assert_true(tmp_container_2 in dids)
assert_true(tmp_dataset_1 in dids)
assert_true(tmp_dataset_2 in dids)
assert_equal(len(dids), 4)
# List DIDs recursive - only containers are expected
dids = [str(did) for did in self.did_client.list_dids(scope=tmp_scope_1, recursive=True, type='container', filters={'name': tmp_container_1})]
assert_true(tmp_container_1 in dids)
assert_true(tmp_container_2 in dids)
assert_true(tmp_dataset_1 not in dids)
assert_true(tmp_dataset_2 not in dids)
assert_equal(len(dids), 2)
def test_list_by_length(self):
""" DATA IDENTIFIERS (CLIENT): List did with length """
tmp_scope = 'mock'
tmp_dsn = 'dsn_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn)
dids = self.did_client.list_dids(tmp_scope, {'length.gt': 0})
results = []
for d in dids:
results.append(d)
assert_not_equal(len(results), 0)
dids = self.did_client.list_dids(tmp_scope, {'length.gt': -1, 'length.lt': 1})
results = []
for d in dids:
results.append(d)
assert_equal(len(results), 0)
dids = self.did_client.list_dids(tmp_scope, {'length': 0})
results = []
for d in dids:
results.append(d)
assert_equal(len(results), 0)
def test_list_by_metadata(self):
""" DATA IDENTIFIERS (CLIENT): List did with metadata"""
dsns = []
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn1)
dataset_meta = {'project': 'data12_8TeV',
'run_number': 400000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m920',
}
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn1, meta=dataset_meta)
tmp_dsn2 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn2)
dataset_meta['run_number'] = 400001
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn2, meta=dataset_meta)
tmp_dsn3 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn3)
dataset_meta['stream_name'] = 'physics_Egamma'
dataset_meta['datatype'] = 'NTUP_SMWZ'
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn3, meta=dataset_meta)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'version': 'f392_m920'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn1)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'run_number': 400001})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn2)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'stream_name': 'physics_Egamma', 'datatype': 'NTUP_SMWZ'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
with assert_raises(KeyNotFound):
self.did_client.list_dids(tmp_scope, {'NotReallyAKey': 'NotReallyAValue'})
def test_add_did(self):
""" DATA IDENTIFIERS (CLIENT): Add, populate, list did content and create a sample"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
tmp_dsn = 'dsn_%s' % generate_uuid()
root = InternalAccount('root')
set_local_account_limit(root, get_rse_id('MOCK'), -1)
set_local_account_limit(root, get_rse_id('CERN-PROD_TZERO'), -1)
# PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename
dataset_meta = {'project': 'data13_hip',
'run_number': 300000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m927',
}
rules = [{'copies': 1, 'rse_expression': 'MOCK', 'account': 'root'}]
with assert_raises(ScopeNotFound):
self.did_client.add_dataset(scope='Nimportnawak', name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
files = [{'scope': tmp_scope, 'name': 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()), 'bytes': 724963570, 'adler32': '0cc737eb'}, ]
with assert_raises(DataIdentifierNotFound):
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files)
with assert_raises(DataIdentifierNotFound):
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files)
files = []
for i in range(5):
lfn = 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 10}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2), 'account': 'root'}]
with assert_raises(InvalidPath):
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files, rse=tmp_rse)
files_without_pfn = [{'scope': i['scope'], 'name': i['name'], 'bytes': i['bytes'], 'adler32': i['adler32'], 'meta': i['meta']} for i in files]
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files_without_pfn, rse=tmp_rse)
with assert_raises(DataIdentifierAlreadyExists):
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
files = []
for i in range(5):
lfn = '%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 100}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}]
with assert_raises(InvalidPath):
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
files_without_pfn = [{'scope': i['scope'], 'name': i['name'], 'bytes': i['bytes'], 'adler32': i['adler32'], 'meta': i['meta']} for i in files]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files_without_pfn, rse=tmp_rse)
self.did_client.close(scope=tmp_scope, name=tmp_dsn)
tmp_dsn_output = 'dsn_%s' % generate_uuid()
self.did_client.create_did_sample(input_scope=tmp_scope, input_name=tmp_dsn, output_scope=tmp_scope, output_name=tmp_dsn_output, nbfiles=2)
files = [f for f in self.did_client.list_files(scope=tmp_scope, name=tmp_dsn_output)]
assert_equal(len(files), 2)
def test_attach_dids_to_dids(self):
""" DATA IDENTIFIERS (CLIENT): Attach dids to dids"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
nb_datasets = 5
nb_files = 5
attachments, dsns = list(), list()
guid_to_query = None
dsn = {}
for i in range(nb_datasets):
attachment = {}
attachment['scope'] = tmp_scope
attachment['name'] = 'dsn.%s' % str(generate_uuid())
attachment['rse'] = tmp_rse
files = []
for i in range(nb_files):
files.append({'scope': tmp_scope, 'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}})
attachment['dids'] = files
guid_to_query = files[0]['meta']['guid']
dsn = {'scope': tmp_scope, 'name': attachment['name']}
dsns.append(dsn)
attachments.append(attachment)
self.did_client.add_datasets(dsns=dsns)
self.did_client.attach_dids_to_dids(attachments=attachments)
dsns_l = [i for i in self.did_client.get_dataset_by_guid(guid_to_query)]
assert_equal([dsn], dsns_l)
cnt_name = 'cnt_%s' % generate_uuid()
self.did_client.add_container(scope='mock', name=cnt_name)
with assert_raises(UnsupportedOperation):
self.did_client.attach_dids_to_dids([{'scope': 'mock', 'name': cnt_name, 'rse': tmp_rse, 'dids': attachment['dids']}])
def test_add_files_to_datasets(self):
""" DATA IDENTIFIERS (CLIENT): Add files to Datasets"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
dsn1 = 'dsn.%s' % str(generate_uuid())
dsn2 = 'dsn.%s' % str(generate_uuid())
meta = {'transient': True}
files1, files2, nb_files = [], [], 5
for i in range(nb_files):
files1.append({'scope': tmp_scope, 'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}})
files2.append({'scope': tmp_scope, 'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}})
self.did_client.add_dataset(scope=tmp_scope, name=dsn1, files=files1,
rse=tmp_rse, meta=meta)
self.did_client.add_dataset(scope=tmp_scope, name=dsn2, files=files2,
rse=tmp_rse, meta=meta)
attachments = [{'scope': tmp_scope, 'name': dsn1, 'dids': files2, 'rse': tmp_rse},
{'scope': tmp_scope, 'name': dsn2, 'dids': files1, 'rse': tmp_rse}]
self.did_client.add_files_to_datasets(attachments)
files = [f for f in self.did_client.list_files(scope=tmp_scope, name=dsn1)]
assert_equal(len(files), 10)
with assert_raises(FileAlreadyExists):
self.did_client.add_files_to_datasets(attachments)
for attachment in attachments:
for i in range(nb_files):
attachment['dids'].append({'scope': tmp_scope,
'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570,
'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()),
'events': 100}})
self.did_client.add_files_to_datasets(attachments, ignore_duplicate=True)
files = [f for f in self.did_client.list_files(scope=tmp_scope, name=dsn1)]
assert_equal(len(files), 15)
# Corrupt meta-data
files = []
for attachment in attachments:
for file in attachment['dids']:
file['bytes'] = 1000
break
with assert_raises(FileConsistencyMismatch):
self.did_client.add_files_to_datasets(attachments, ignore_duplicate=True)
def test_add_dataset(self):
""" DATA IDENTIFIERS (CLIENT): Add dataset """
tmp_scope = 'mock'
tmp_dsn = 'dsn_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, meta={'project': 'data13_hip'})
did = self.did_client.get_did(tmp_scope, tmp_dsn)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_dsn)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_add_datasets(self):
""" DATA IDENTIFIERS (CLIENT): Bulk add datasets """
tmp_scope = 'mock'
dsns = list()
for i in range(500):
tmp_dsn = {'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'meta': {'project': 'data13_hip'}}
dsns.append(tmp_dsn)
self.did_client.add_datasets(dsns)
def test_exists(self):
""" DATA IDENTIFIERS (CLIENT): Check if data identifier exists """
tmp_scope = 'mock'
tmp_file = 'file_%s' % generate_uuid()
tmp_rse = 'MOCK'
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1, '0cc737eb')
did = self.did_client.get_did(tmp_scope, tmp_file)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_file)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_did_hierarchy(self):
""" DATA IDENTIFIERS (CLIENT): Check did hierarchy rule """
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(4)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1, '0cc737eb')
for i in range(4):
self.did_client.add_did(scope, dst[i], 'DATASET', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_did(scope, cnt[i], 'CONTAINER', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': cnt[2]}, {'scope': scope, 'name': cnt[3]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
pass
# TODO: fix, fix, fix
# if r['name'] == cnt[1]:
# assert_equal(r['type'], 'container')
# assert_equal(r['level'], 0)
# if (r['name'] == cnt[0]) or (r['name'] == dst[0]) or (r['name'] == file[8]) or (r['name'] == file[9]):
# assert_equal(r['level'], 0)
# else:
# assert_equal(r['level'], 1)
def test_detach_did(self):
""" DATA IDENTIFIERS (CLIENT): Detach dids from a did"""
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(5)]
cnt = ['cnt_%s' % generate_uuid() for i in range(2)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1, '0cc737eb')
for i in range(5):
self.did_client.add_dataset(scope, dst[i], statuses=None, meta=None, rules=None)
for i in range(2):
self.did_client.add_container(scope, cnt[i], statuses=None, meta=None, rules=None)
for i in range(5):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': dst[2]}, {'scope': scope, 'name': dst[3]}])
with assert_raises(UnsupportedOperation):
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': cnt[1]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
self.did_client.detach_dids(scope, cnt[0], [{'scope': scope, 'name': dst[1]}])
self.did_client.detach_dids(scope, dst[3], [{'scope': scope, 'name': file[6]}, {'scope': scope, 'name': file[7]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
if r['name'] == dst[1]:
assert_equal(r['level'], 0)
if r['type'] == 'file':
if (r['name'] in file[6:9]):
assert_equal(r['level'], 0)
else:
assert_not_equal(r['level'], 0)
with assert_raises(UnsupportedOperation):
self.did_client.detach_dids(scope=scope, name=cnt[0], dids=[{'scope': scope, 'name': cnt[0]}])
self.did_client.close(scope, dst[4])
metadata = self.did_client.get_metadata(scope, dst[4])
i_bytes, i_length = metadata['bytes'], metadata['length']
metadata = self.did_client.get_metadata(scope, file[8])
file1_bytes = metadata['bytes']
metadata = self.did_client.get_metadata(scope, file[9])
file2_bytes = metadata['bytes']
self.did_client.detach_dids(scope, dst[4], [{'scope': scope, 'name': file[8]}, {'scope': scope, 'name': file[9]}])
metadata = self.did_client.get_metadata(scope, dst[4])
f_bytes, f_length = metadata['bytes'], metadata['length']
assert_equal(i_bytes, f_bytes + file1_bytes + file2_bytes)
assert_equal(i_length, f_length + 1 + 1)
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in range(3)]
self.tmp_scopes = [scope_name_generator() for i in range(3)]
self.tmp_rses = [rse_name_generator() for i in range(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in range(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in range(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in range(3)]
# add dummy data to the catalogue
for i in range(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1, '0cc737eb')
# put files in datasets
for i in range(3):
for j in range(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
# put datasets in containers
for i in range(3):
for j in range(3):
datasets = [{'scope': self.tmp_scopes[j], 'name': self.tmp_datasets[j]}]
self.did_client.add_container(self.tmp_scopes[i], self.tmp_containers[j])
self.did_client.add_datasets_to_container(self.tmp_scopes[i], self.tmp_containers[j], datasets)
# reverse check if everything is in order
for i in range(3):
result = self.did_client.scope_list(self.tmp_scopes[i], recursive=True)
r_topdids = []
r_otherscopedids = []
r_scope = []
for r in result:
if r['level'] == 0:
r_topdids.append(r['scope'] + ':' + r['name'])
r_scope.append(r['scope'])
if r['scope'] != self.tmp_scopes[i]:
r_otherscopedids.append(r['scope'] + ':' + r['name'])
assert_in(r['level'], [1, 2])
for j in range(3):
assert_equal(self.tmp_scopes[i], r_scope[j])
if j != i:
assert_in(self.tmp_scopes[j] + ':' + self.tmp_files[j], r_otherscopedids)
assert_not_in(self.tmp_scopes[i] + ':' + self.tmp_files[i], r_topdids)
def test_get_did(self):
""" DATA IDENTIFIERS (CLIENT): add a new data identifier and try to retrieve it back"""
rse = 'MOCK'
scope = 'mock'
file = generate_uuid()
dsn = generate_uuid()
self.replica_client.add_replica(rse, scope, file, 1, '0cc737eb')
did = self.did_client.get_did(scope, file)
assert_equal(did['scope'], scope)
assert_equal(did['name'], file)
self.did_client.add_dataset(scope=scope, name=dsn, lifetime=10000000)
did2 = self.did_client.get_did(scope, dsn)
assert_equal(type(did2['expired_at']), datetime)
def test_get_meta(self):
""" DATA IDENTIFIERS (CLIENT): add a new meta data for an identifier and try to retrieve it back"""
rse = 'MOCK'
scope = 'mock'
file = generate_uuid()
keys = ['project', 'run_number']
values = ['data13_hip', 12345678]
self.replica_client.add_replica(rse, scope, file, 1, '0cc737eb')
for i in range(2):
self.did_client.set_metadata(scope, file, keys[i], values[i])
meta = self.did_client.get_metadata(scope, file)
for i in range(2):
assert_equal(meta[keys[i]], values[i])
def test_list_content(self):
""" DATA IDENTIFIERS (CLIENT): test to list contents for an identifier"""
rse = 'MOCK'
scope = 'mock'
nbfiles = 5
dataset1 = generate_uuid()
dataset2 = generate_uuid()
container = generate_uuid()
files1 = [{'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'} for i in range(nbfiles)]
files2 = [{'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'} for i in range(nbfiles)]
self.did_client.add_dataset(scope, dataset1)
with assert_raises(DataIdentifierAlreadyExists):
self.did_client.add_dataset(scope, dataset1)
self.did_client.add_files_to_dataset(scope, dataset1, files1, rse=rse)
self.did_client.add_dataset(scope, dataset2)
self.did_client.add_files_to_dataset(scope, dataset2, files2, rse=rse)
self.did_client.add_container(scope, container)
datasets = [{'scope': scope, 'name': dataset1}, {'scope': scope, 'name': dataset2}]
self.did_client.add_datasets_to_container(scope, container, datasets)
contents = self.did_client.list_content(scope, container)
datasets_s = [d['name'] for d in contents]
assert_in(dataset1, datasets_s)
assert_in(dataset2, datasets_s)
def test_list_files(self):
""" DATA IDENTIFIERS (CLIENT): List files for a container"""
rse = 'MOCK'
scope = 'mock'
dataset1 = generate_uuid()
dataset2 = generate_uuid()
container = generate_uuid()
files1 = []
files2 = []
for i in range(10):
files1.append({'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'})
files2.append({'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'})
for i in range(10):
self.replica_client.add_replica(rse, scope, files1[i]['name'], 1, '0cc737eb')
self.replica_client.add_replica(rse, scope, files2[i]['name'], 1, '0cc737eb')
self.did_client.add_dataset(scope, dataset1)
self.did_client.add_files_to_dataset(scope, dataset1, files1)
self.did_client.add_dataset(scope, dataset2)
self.did_client.add_files_to_dataset(scope, dataset2, files2)
datasets = [{'scope': scope, 'name': dataset1}, {'scope': scope, 'name': dataset2}]
self.did_client.add_container(scope, container)
self.did_client.add_datasets_to_container(scope, container, datasets)
# List file content
content = self.did_client.list_files(scope, files1[i]['name'])
assert_true(content is not None)
for d in content:
assert_true(d['name'] == files1[i]['name'])
# List container content
for d in [{'name': x['name'], 'scope': x['scope'], 'bytes': x['bytes'], 'adler32': x['adler32']} for x in self.did_client.list_files(scope, container)]:
assert_in(d, files1 + files2)
# List non-existing data identifier content
with assert_raises(DataIdentifierNotFound):
self.did_client.list_files(scope, 'Nimportnawak')
def test_list_replicas(self):
""" DATA IDENTIFIERS (CLIENT): List replicas for a container"""
rse = 'MOCK'
scope = 'mock'
dsn1 = generate_uuid()
dsn2 = generate_uuid()
cnt = generate_uuid()
files1 = []
files2 = []
for i in range(10):
files1.append({'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'})
files2.append({'scope': scope, 'name': generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'})
self.did_client.add_dataset(scope, dsn1)
self.did_client.add_files_to_dataset(scope, dsn1, files1, rse=rse)
self.did_client.add_dataset(scope, dsn2)
self.did_client.add_files_to_dataset(scope, dsn2, files2, rse=rse)
self.did_client.add_container(scope, cnt)
self.did_client.add_datasets_to_container(scope, cnt, [{'scope': scope, 'name': dsn1}, {'scope': scope, 'name': dsn2}])
replicas = self.replica_client.list_replicas(dids=[{'scope': scope, 'name': dsn1}])
assert_true(replicas is not None)
replicas = self.replica_client.list_replicas(dids=[{'scope': scope, 'name': cnt}])
assert_true(replicas is not None)
@raises(UnsupportedOperation)
def test_close(self):
""" DATA IDENTIFIERS (CLIENT): test to close data identifiers"""
tmp_rse = 'MOCK'
tmp_scope = 'mock'
# Add dataset
tmp_dataset = 'dsn_%s' % generate_uuid()
# Add file replica
tmp_file = 'file_%s' % generate_uuid()
self.replica_client.add_replica(rse=tmp_rse, scope=tmp_scope, name=tmp_file, bytes=1, adler32='0cc737eb')
# Add dataset
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dataset)
# Add files to dataset
files = [{'scope': tmp_scope, 'name': tmp_file, 'bytes': 1, 'adler32': '0cc737eb'}, ]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dataset, files=files)
# Add a second file replica
tmp_file = 'file_%s' % generate_uuid()
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1, '0cc737eb')
# Add files to dataset
files = [{'scope': tmp_scope, 'name': tmp_file, 'bytes': 1, 'adler32': '0cc737eb'}, ]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dataset, files=files)
# Close dataset
with assert_raises(UnsupportedStatus):
self.did_client.set_status(scope=tmp_scope, name=tmp_dataset, close=False)
self.did_client.set_status(scope=tmp_scope, name=tmp_dataset, open=False)
# Add a third file replica
tmp_file = 'file_%s' % generate_uuid()
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1, '0cc737eb')
# Add files to dataset
files = [{'scope': tmp_scope, 'name': tmp_file, 'bytes': 1, 'adler32': '0cc737eb'}, ]
self.did_client.attach_dids(scope=tmp_scope, name=tmp_dataset, dids=files)
@raises
def test_open(self):
""" DATA IDENTIFIERS (CLIENT): test to re-open data identifiers for priv account"""
tmp_rse = 'MOCK'
tmp_scope = 'mock'
# Add dataset
tmp_dataset = 'dsn_%s' % generate_uuid()
# Add file replica
tmp_file = 'file_%s' % generate_uuid()
self.replica_client.add_replica(rse=tmp_rse, scope=tmp_scope, name=tmp_file, bytes=1, adler32='0cc737eb')
# Add dataset
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dataset)
# Add files to dataset
files = [{'scope': tmp_scope, 'name': tmp_file, 'bytes': 1, 'adler32': '0cc737eb'}, ]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dataset, files=files)
# Add a second file replica
tmp_file = 'file_%s' % generate_uuid()
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1, '0cc737eb')
# Add files to dataset
files = [{'scope': tmp_scope, 'name': tmp_file, 'bytes': 1, 'adler32': '0cc737eb'}, ]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dataset, files=files)
# Close dataset
with assert_raises(UnsupportedStatus):
self.did_client.set_status(scope=tmp_scope, name=tmp_dataset, close=False)
self.did_client.set_status(scope=tmp_scope, name=tmp_dataset, open=False)
# Add a third file replica
self.did_client.set_status(scope=tmp_scope, name=tmp_dataset, open=True)
def test_bulk_get_meta(self):
""" DATA IDENTIFIERS (CLIENT): Add a new meta data for a list of DIDs and try to retrieve them back"""
key = 'project'
rse = 'MOCK'
scope = 'mock'
files = ['file_%s' % generate_uuid() for _ in range(4)]
dst = ['dst_%s' % generate_uuid() for _ in range(4)]
cnt = ['cnt_%s' % generate_uuid() for _ in range(4)]
meta_mapping = {}
list_dids = []
for idx in range(4):
self.replica_client.add_replica(rse, scope, files[idx], 1, '0cc737eb')
self.did_client.set_metadata(scope, files[idx], key, 'file_%s' % idx)
list_dids.append({'scope': scope, 'name': files[idx]})
meta_mapping['%s:%s' % (scope, files[idx])] = (key, 'file_%s' % idx)
for idx in range(4):
self.did_client.add_did(scope, dst[idx], 'DATASET', statuses=None, meta={key: 'dsn_%s' % idx}, rules=None)
list_dids.append({'scope': scope, 'name': dst[idx]})
meta_mapping['%s:%s' % (scope, dst[idx])] = (key, 'dsn_%s' % idx)
for idx in range(4):
self.did_client.add_did(scope, cnt[idx], 'CONTAINER', statuses=None, meta={key: 'cnt_%s' % idx}, rules=None)
list_dids.append({'scope': scope, 'name': cnt[idx]})
meta_mapping['%s:%s' % (scope, cnt[idx])] = (key, 'cnt_%s' % idx)
list_meta = [_ for _ in self.did_client.get_metadata_bulk(list_dids)]
res_list_dids = [{'scope': entry['scope'], 'name': entry['name']} for entry in list_meta]
res_list_dids.sort()
list_dids.sort()
assert_equal(list_dids, res_list_dids)
for meta in list_meta:
did = '%s:%s' % (meta['scope'], meta['name'])
met = meta_mapping[did]
assert_equal((key, meta[key]), met)
cnt = ['cnt_%s' % generate_uuid() for _ in range(4)]
for idx in range(4):
list_dids.append({'scope': scope, 'name': cnt[idx]})
list_meta = [_ for _ in self.did_client.get_metadata_bulk(list_dids)]
assert_equal(len(list_meta), 12)
list_dids = []
for idx in range(4):
list_dids.append({'scope': scope, 'name': cnt[idx]})
list_meta = [_ for _ in self.did_client.get_metadata_bulk(list_dids)]
assert_equal(len(list_meta), 0)
import logging
from rucio.client.rseclient import RSEClient
from rucio.client.didclient import DIDClient
from rucio.client.replicaclient import ReplicaClient
from kafka import KafkaProducer
from datetime import datetime
rucio_account = 'navila'
# DID Client
didclient = DIDClient(account=rucio_account)
# Replica Client
replicaclient = ReplicaClient(account=rucio_account)
# RSE Client
rseclient = RSEClient(account=rucio_account)
rselist = rseclient.list_rses()
# kafka scripts
logger = logging.getLogger(__name__)
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s [%(levelname)s] %(name)s - %(message)s')
def vs(doc):
return json.dumps(doc).encode(encoding='utf-8', errors='replace')
kafka_bootstrap_servers = [
'lssrv03.fnal.gov:9092', 'lssrv04.fnal.gov:9092', 'lssrv05.fnal.gov:9092'
def setUp(self):
""" Setup the Test Case """
self.did_client = DIDClient()
self.meta_client = MetaClient()
self.rse_client = RSEClient()
self.scope_client = ScopeClient()
import rucio
from rucio.client import Client
from rucio.client.didclient import DIDClient
from rucio.client.replicaclient import ReplicaClient
from rucio.client.rseclient import RSEClient
from rucio.client.ruleclient import RuleClient
from rucio.client.accountclient import AccountClient
from rucio.common import exception
DID = namedtuple('DID', ['scope', 'name', 'ftype', 'owner', 'path', 'dataset'])
did_client = DIDClient()
rep_client = ReplicaClient()
rse_client = RSEClient()
rule_client = RuleClient()
admin_client = AccountClient()
def list_uk_localgroupdisks():
"""Return a list of known uk localgroup disks
"""
global rse_client
r = rse_client.list_rses(
rse_expression='spacetoken=ATLASLOCALGROUPDISK&cloud=UK')
return sorted(x['rse'] for x in r)
def get_rse_usage(rse, filters=None):
return rse_client.get_rse_usage(rse, filters=filters)
class TestDIDClients:
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def test_list_dids(self):
""" DATA IDENTIFIERS (CLIENT): List dids by pattern."""
tmp_scope = scope_name_generator()
tmp_files = []
tmp_files.append('file_a_1%s' % generate_uuid())
tmp_files.append('file_a_2%s' % generate_uuid())
tmp_files.append('file_b_1%s' % generate_uuid())
tmp_rse = 'MOCK'
self.scope_client.add_scope('jdoe', tmp_scope)
for tmp_file in tmp_files:
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L, '0cc737eb')
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\_a\_*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\_a\_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 1)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\__\_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='file'):
results.append(result)
assert_equal(len(results), 3)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}):
results.append(result)
assert_equal(len(results), 0)
with assert_raises(UnsupportedOperation):
self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='whateverytype')
def test_list_by_metadata(self):
""" DATA IDENTIFIERS (CLIENT): List did with metadata"""
dsns = []
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn1)
dataset_meta = {'project': 'data12_8TeV',
'run_number': 400000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m920',
}
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn1, meta=dataset_meta)
tmp_dsn2 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn2)
dataset_meta['run_number'] = 400001
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn2, meta=dataset_meta)
tmp_dsn3 = 'dsn_%s' % generate_uuid()
dsns.append(tmp_dsn3)
dataset_meta['stream_name'] = 'physics_Egamma'
dataset_meta['datatype'] = 'NTUP_SMWZ'
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn3, meta=dataset_meta)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'version': 'f392_m920'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn1)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'run_number': 400001})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
dsns.remove(tmp_dsn2)
dids = self.did_client.list_dids(tmp_scope, {'project': 'data12_8TeV', 'stream_name': 'physics_Egamma', 'datatype': 'NTUP_SMWZ'})
results = []
for d in dids:
results.append(d)
for dsn in dsns:
assert_in(dsn, results)
with assert_raises(KeyNotFound):
self.did_client.list_dids(tmp_scope, {'NotReallyAKey': 'NotReallyAValue'})
def test_add_did(self):
""" DATA IDENTIFIERS (CLIENT): Add, populate and list did content"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
tmp_dsn = 'dsn_%s' % generate_uuid()
# PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename
dataset_meta = {'project': 'data13_hip',
'run_number': 300000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m927',
}
rules = [{'copies': 1, 'rse_expression': 'MOCK', 'account': 'root'}]
with assert_raises(ScopeNotFound):
self.did_client.add_dataset(scope='Nimportnawak', name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
with assert_raises(DataIdentifierNotFound):
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=[{'scope': tmp_scope, 'name': 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()),
'bytes': 724963570L, 'adler32': '0cc737eb'}, ])
files = []
for i in xrange(5):
lfn = 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 10}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570L, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
files = []
for i in xrange(5):
lfn = '%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 100}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570L, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
self.did_client.close(scope=tmp_scope, name=tmp_dsn)
def test_attach_dids_to_dids(self):
""" DATA IDENTIFIERS (CLIENT): Attach dids to dids"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
nb_datasets = 5
nb_files = 5
attachments, dsns = list(), list()
guid_to_query = None
dsn = {}
for i in xrange(nb_datasets):
attachment = {}
attachment['scope'] = tmp_scope
attachment['name'] = 'dsn.%s' % str(generate_uuid())
attachment['rse'] = tmp_rse
files = []
for i in xrange(nb_files):
files.append({'scope': tmp_scope, 'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570L, 'adler32': '0cc737eb',
'meta': {'guid': str(generate_uuid()), 'events': 100}})
attachment['dids'] = files
guid_to_query = files[0]['meta']['guid']
dsn = {'scope': tmp_scope, 'name': attachment['name']}
dsns.append(dsn)
attachments.append(attachment)
self.did_client.add_datasets(dsns=dsns)
self.did_client.attach_dids_to_dids(attachments=attachments)
l = [i for i in self.did_client.get_dataset_by_guid(guid_to_query)]
assert_equal([dsn], l)
cnt_name = 'cnt_%s' % generate_uuid()
self.did_client.add_container(scope='mock', name=cnt_name)
with assert_raises(UnsupportedOperation):
self.did_client.attach_dids_to_dids([{'scope': 'mock', 'name': cnt_name, 'rse': tmp_rse, 'dids': attachment['dids']}])
def test_add_dataset(self):
""" DATA IDENTIFIERS (CLIENT): Add dataset """
tmp_scope = 'mock'
tmp_dsn = 'dsn_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, meta={'project': 'data13_hip'})
did = self.did_client.get_did(tmp_scope, tmp_dsn)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_dsn)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_add_datasets(self):
""" DATA IDENTIFIERS (CLIENT): Bulk add datasets """
tmp_scope = 'mock'
dsns = list()
for i in xrange(500):
tmp_dsn = {'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'meta': {'project': 'data13_hip'}}
dsns.append(tmp_dsn)
self.did_client.add_datasets(dsns)
def test_exists(self):
""" DATA IDENTIFIERS (CLIENT): Check if data identifier exists """
tmp_scope = 'mock'
tmp_file = 'file_%s' % generate_uuid()
tmp_rse = 'MOCK'
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L, '0cc737eb')
did = self.did_client.get_did(tmp_scope, tmp_file)
assert_equal(did['scope'], tmp_scope)
assert_equal(did['name'], tmp_file)
with assert_raises(DataIdentifierNotFound):
self.did_client.get_did('i_dont_exist', 'neither_do_i')
def test_did_hierarchy(self):
""" DATA IDENTIFIERS (CLIENT): Check did hierarchy rule """
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(4)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1, '0cc737eb')
for i in range(4):
self.did_client.add_did(scope, dst[i], 'DATASET', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_did(scope, cnt[i], 'CONTAINER', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1L, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1L, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': cnt[2]}, {'scope': scope, 'name': cnt[3]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
pass
# TODO: fix, fix, fix
# if r['name'] == cnt[1]:
# assert_equal(r['type'], 'container')
# assert_equal(r['level'], 0)
# if (r['name'] == cnt[0]) or (r['name'] == dst[0]) or (r['name'] == file[8]) or (r['name'] == file[9]):
# assert_equal(r['level'], 0)
# else:
# assert_equal(r['level'], 1)
def test_detach_did(self):
""" DATA IDENTIFIERS (CLIENT): Detach dids from a did"""
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(2)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1L, '0cc737eb')
for i in range(4):
self.did_client.add_dataset(scope, dst[i], statuses=None, meta=None, rules=None)
for i in range(2):
self.did_client.add_container(scope, cnt[i], statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1L, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1L, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': dst[2]}, {'scope': scope, 'name': dst[3]}])
with assert_raises(UnsupportedOperation):
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': cnt[1]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
self.did_client.detach_dids(scope, cnt[0], [{'scope': scope, 'name': dst[1]}])
self.did_client.detach_dids(scope, dst[3], [{'scope': scope, 'name': file[6]}, {'scope': scope, 'name': file[7]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
if r['name'] == dst[1]:
assert_equal(r['level'], 0)
if r['type'] is 'file':
if (r['name'] in file[6:9]):
assert_equal(r['level'], 0)
else:
assert_not_equal(r['level'], 0)
with assert_raises(UnsupportedOperation):
self.did_client.detach_dids(scope=scope, name=cnt[0], dids=[{'scope': scope, 'name': cnt[0]}])
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in xrange(3)]
self.tmp_scopes = [scope_name_generator() for i in xrange(3)]
self.tmp_rses = [rse_name_generator() for i in xrange(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in xrange(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in xrange(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in xrange(3)]
# add dummy data to the catalogue
for i in xrange(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1L, '0cc737eb')
# put files in datasets
for i in xrange(3):
for j in xrange(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1L, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
if args.node_fts_servers:
servers = PhEDEx_node_FTS_servers(args.node_fts_servers)
print "FTS servers used by " + args.node_fts_servers + ' PhEDEx node:'
for s in servers:
print s
sys.exit()
if args.node_protocols:
PhEDEx_node_protocols(args.node_protocols)
sys.exit()
if args.test_auth:
whoami(account=args.account)
# create re-usable RSE client connection:
rse_client = RSEClient(account=args.account, auth_type='x509_proxy')
if args.list_rses:
list_rses()
if args.get_rse_distance:
(s, d) = args.get_rse_distance
pprint.pprint(get_rse_distance(s, d))
sys.exit()
if args.link_attributes:
(s, d) = args.link_attributes
pprint.pprint(PhEDEx_link_attributes(s, d))
sys.exit()
# Handle RSE additions and configuration update
from rucio.client.rseclient import RSEClient
import uuid
c = RSEClient(account="ivm")
for rse in c.list_rses():
u = uuid.UUID(rse["id"])
print rse["rse"], u.hex
def test_subscriptions_at_different_vos(self):
""" MULTI VO (CLIENT): Test that subscriptions from 2nd vo don't interfere """
account_client = AccountClient()
usr_uuid = str(generate_uuid()).lower()[:16]
shr_acc = 'shr-%s' % usr_uuid
account_client.add_account(shr_acc, 'USER', '*****@*****.**')
add_account(shr_acc, 'USER', '*****@*****.**', 'root', **self.new_vo)
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
tst_scope = 'tst_%s' % scope_uuid
new_scope = 'new_%s' % scope_uuid
scope_client.add_scope('root', tst_scope)
add_scope(new_scope, 'root', 'root', **self.new_vo)
did_client = DIDClient()
did_uuid = str(generate_uuid()).lower()[:16]
tst_did = 'tstset_%s' % did_uuid
new_did = 'newset_%s' % did_uuid
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst_rse1 = 'TST1_%s' % rse_str
tst_rse2 = 'TST2_%s' % rse_str
new_rse1 = 'NEW1_%s' % rse_str
new_rse2 = 'NEW2_%s' % rse_str
rse_client.add_rse(tst_rse1)
rse_client.add_rse(tst_rse2)
add_rse(new_rse1, 'root', **self.new_vo)
add_rse(new_rse2, 'root', **self.new_vo)
acc_lim_client = AccountLimitClient()
acc_lim_client.set_local_account_limit(shr_acc, tst_rse1, 10)
acc_lim_client.set_local_account_limit(shr_acc, tst_rse2, 10)
set_local_account_limit(shr_acc, new_rse1, 10, 'root', **self.new_vo)
set_local_account_limit(shr_acc, new_rse2, 10, 'root', **self.new_vo)
did_client.add_did(tst_scope, tst_did, 'DATASET', rse=tst_rse1)
add_did(new_scope, new_did, 'DATASET', 'root', rse=new_rse1, **self.new_vo)
sub_client = SubscriptionClient()
sub_str = generate_uuid()
tst_sub = 'tstsub_' + sub_str
new_sub = 'newsub_' + sub_str
shr_sub = 'shrsub_' + sub_str
tst_sub_id = sub_client.add_subscription(tst_sub, shr_acc, {'scope': [tst_scope]},
[{'copies': 1, 'rse_expression': tst_rse2, 'weight': 0,
'activity': 'User Subscriptions'}],
'', None, 0, 0)
shr_tst_sub_id = sub_client.add_subscription(shr_sub, shr_acc, {'scope': [tst_scope]},
[{'copies': 1, 'rse_expression': tst_rse2, 'weight': 0,
'activity': 'User Subscriptions'}],
'', None, 0, 0)
new_sub_id = add_subscription(new_sub, shr_acc, {'scope': [new_scope]},
[{'copies': 1, 'rse_expression': new_rse2, 'weight': 0, 'activity': 'User Subscriptions'}],
'', False, 0, 0, 3, 'root', **self.new_vo)
shr_new_sub_id = add_subscription(shr_sub, shr_acc, {'scope': [new_scope]},
[{'copies': 1, 'rse_expression': new_rse2, 'weight': 0, 'activity': 'User Subscriptions'}],
'', False, 0, 0, 3, 'root', **self.new_vo)
tst_subs = [s['id'] for s in sub_client.list_subscriptions()]
assert_in(tst_sub_id, tst_subs)
assert_in(shr_tst_sub_id, tst_subs)
assert_not_in(new_sub_id, tst_subs)
assert_not_in(shr_new_sub_id, tst_subs)
new_subs = [s['id'] for s in list_subscriptions(**self.new_vo)]
assert_in(new_sub_id, new_subs)
assert_in(shr_new_sub_id, new_subs)
assert_not_in(tst_sub_id, new_subs)
assert_not_in(shr_tst_sub_id, new_subs)
shr_tst_subs = [s['id'] for s in sub_client.list_subscriptions(name=shr_sub)]
assert_in(shr_tst_sub_id, shr_tst_subs)
assert_not_in(shr_new_sub_id, shr_tst_subs)
shr_new_subs = [s['id'] for s in list_subscriptions(name=shr_sub, **self.new_vo)]
assert_in(shr_new_sub_id, shr_new_subs)
assert_not_in(shr_tst_sub_id, shr_new_subs)
acc_tst_subs = [s['id'] for s in sub_client.list_subscriptions(account=shr_acc)]
assert_in(tst_sub_id, acc_tst_subs)
assert_in(shr_tst_sub_id, acc_tst_subs)
assert_not_in(new_sub_id, acc_tst_subs)
assert_not_in(shr_new_sub_id, acc_tst_subs)
acc_new_subs = [s['id'] for s in list_subscriptions(account=shr_acc, **self.new_vo)]
assert_in(new_sub_id, acc_new_subs)
assert_in(shr_new_sub_id, acc_new_subs)
assert_not_in(tst_sub_id, acc_new_subs)
assert_not_in(shr_tst_sub_id, acc_new_subs)
#! /usr/bin/env python3
from rucio.client.rseclient import RSEClient
from rucio.common.exception import Duplicate, RSEProtocolNotSupported
rseclient = RSEClient()
rse_name = 'T3_US_Theta'
rse_properties = {
'ASN': 'ASN',
'availability': 7,
'deterministic': True,
'volatile': False,
'city': 'Chicago',
'region_code': 'IL',
'country_name': 'US',
'continent': 'NA',
'time_zone': 'America/Chicago',
'ISP': None,
'staging_area': False,
'rse_type': 'DISK',
'longitude': 41.88,
'latitude': -87.63
}
try:
r = rseclient.add_rse(rse_name, **rse_properties) # r is true on success
print('Added Theta %s' % r)
except Duplicate:
print('Theta existed')
prefix = '/lus/theta-fs0/projects/HighLumin/uscms/' # Be sure to use a relative path for your endpoint
def rse_client():
return RSEClient()
def get_rse_client(rse, **kwarg):
'''
get_rse_client
'''
from rucio.client.rseclient import RSEClient
return RSEClient().get_rse(rse)
def rse_client():
from rucio.client.rseclient import RSEClient
return RSEClient()
class TestBoolean(unittest.TestCase):
def setUp(self):
self.account_client = AccountClient()
self.rse_client = RSEClient()
self.account = generate_uuid()[:10]
self.rse = rse_name_generator()
self.account_client.add_account(self.account, 'SERVICE',
'*****@*****.**')
self.rse_client.add_rse(self.rse)
def tearDown(self):
self.account_client.delete_account(self.account)
self.rse_client.delete_rse(self.rse)
def test_booleanstring_account_attribute(self):
self.account_client.add_account_attribute(self.account,
'teststringtrue', 'true')
self.account_client.add_account_attribute(self.account, 'testinttrue',
'1')
self.account_client.add_account_attribute(self.account,
'teststringfalse', 'false')
self.account_client.add_account_attribute(self.account, 'testintfalse',
'0')
result = {}
for account in self.account_client.list_account_attributes(
self.account):
for res in account:
result[res['key']] = res['value']
assert result['teststringtrue'] is True
assert result['testinttrue'] == '1'
assert result['teststringfalse'] is False
assert result['testintfalse'] == '0'
def test_booleanstring_rse_attribute(self):
self.rse_client.add_rse_attribute(self.rse, 'teststringtrue', 'true')
self.rse_client.add_rse_attribute(self.rse, 'testinttrue', '1')
self.rse_client.add_rse_attribute(self.rse, 'teststringfalse', 'false')
self.rse_client.add_rse_attribute(self.rse, 'testintfalse', '0')
result = self.rse_client.list_rse_attributes(self.rse)
assert result['teststringtrue'] is True
assert result['testinttrue'] == '1'
assert result['teststringfalse'] is False
assert result['testintfalse'] == '0'
print s
sys.exit()
if args.node_protocols:
PhEDEx_node_protocols(args.node_protocols)
sys.exit()
if args.test_auth:
whoami(account=args.account)
# create re-usable RSE client connection:
if args.password and args.username:
creds = {'username': args.username, 'password': args.password}
rse_client = RSEClient(account=args.account,
auth_type='userpass',
creds=creds,
rucio_host=args.host,
auth_host=args.auth_host)
else:
rse_client = RSEClient(account=args.account, auth_type='x509_proxy')
if args.list_rses:
list_rses()
if args.get_rse_distance:
(s, d) = args.get_rse_distance
pprint.pprint(get_rse_distance(s, d))
sys.exit()
if args.link_attributes:
(s, d) = args.link_attributes
class TestQoS(object):
@classmethod
def setupClass(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
self.vo = {}
self.rse_client = RSEClient()
self.tmp_rse_name = rse_name_generator()
self.rse_client.add_rse(self.tmp_rse_name, vo=self.vo)
self.tmp_rse = self.rse_client.get_rse(self.tmp_rse_name)['id']
@classmethod
def teardownClass(self):
self.rse_client.delete_rse(self.tmp_rse_name)
def test_update_and_remove_rse_qos_class(self):
""" QoS (CORE): Update and remove QoS class for RSE """
update_rse(self.tmp_rse, {'qos_class': 'fast_and_expensive'})
rse = get_rse(self.tmp_rse)
assert_equal(rse['qos_class'], 'fast_and_expensive')
update_rse(self.tmp_rse, {'qos_class': 'slow_but_cheap'})
rse = get_rse(self.tmp_rse)
assert_equal(rse['qos_class'], 'slow_but_cheap')
update_rse(self.tmp_rse, {'qos_class': None})
rse = get_rse(self.tmp_rse)
assert_equal(rse['qos_class'], None)
def test_update_and_remove_rse_qos_class_client(self):
""" QoS (CLIENT): Update and remove QoS class for RSE """
self.rse_client.update_rse(self.tmp_rse_name,
{'qos_class': 'fast_and_expensive'})
rse = self.rse_client.get_rse(self.tmp_rse_name)
assert_equal(rse['qos_class'], 'fast_and_expensive')
self.rse_client.update_rse(self.tmp_rse_name,
{'qos_class': 'slow_but_cheap'})
rse = self.rse_client.get_rse(self.tmp_rse_name)
assert_equal(rse['qos_class'], 'slow_but_cheap')
self.rse_client.update_rse(self.tmp_rse_name, {'qos_class': None})
rse = self.rse_client.get_rse(self.tmp_rse_name)
assert_equal(rse['qos_class'], None)
def test_qos_policies(self):
""" QoS (CLIENT): Add QoS policy for RSE """
self.rse_client.add_qos_policy(self.tmp_rse_name, 'FOO')
policies = self.rse_client.list_qos_policies(self.tmp_rse_name)
assert_equal(policies, ['FOO'])
self.rse_client.add_qos_policy(self.tmp_rse_name, 'BAR')
policies = sorted(self.rse_client.list_qos_policies(self.tmp_rse_name))
assert_equal(policies, ['BAR', 'FOO'])
self.rse_client.delete_qos_policy(self.tmp_rse_name, 'BAR')
policies = self.rse_client.list_qos_policies(self.tmp_rse_name)
assert_equal(policies, ['FOO'])
self.rse_client.delete_qos_policy(self.tmp_rse_name, 'FOO')
policies = self.rse_client.list_qos_policies(self.tmp_rse_name)
assert_equal(policies, [])
