def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def test_scopes_at_different_vos(self):
""" MULTI VO (CLIENT): Test that scopes from 2nd vo don't interfere """
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
tst = 'tst_%s' % scope_uuid
new = 'new_%s' % scope_uuid
shr = 'shr_%s' % scope_uuid
scope_client.add_scope('root', tst)
scope_client.add_scope('root', shr)
add_scope(new, 'root', 'root', **self.new_vo)
add_scope(shr, 'root', 'root', **self.new_vo)
scope_list_tst = list(scope_client.list_scopes())
scope_list_new = list(list_scopes(filter={}, **self.new_vo))
assert_true(tst in scope_list_tst)
assert_false(new in scope_list_tst)
assert_true(shr in scope_list_tst)
assert_false(tst in scope_list_new)
assert_true(new in scope_list_new)
assert_true(shr in scope_list_new)
def test_automatix(self):
""" MULTI VO (DAEMON): Test that automatix runs on a single VO """
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
shr_scope = 'shr_%s' % scope_uuid
scope_client.add_scope('root', shr_scope)
add_scope(shr_scope, 'root', 'root', **self.new_vo)
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
shr_rse = 'SHR_%s' % rse_str
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/automatix',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
rse_client.add_rse(shr_rse)
rse_client.add_rse_attribute(rse=shr_rse, key='verify_checksum', value=False)
rse_client.add_protocol(shr_rse, mock_protocol)
add_rse(shr_rse, 'root', **self.new_vo)
add_rse_attribute(rse=shr_rse, key='verify_checksum', value=False, issuer='root', **self.new_vo)
add_protocol(rse=shr_rse, data=mock_protocol, issuer='root', **self.new_vo)
automatix(sites=[shr_rse], inputfile='/opt/rucio/etc/automatix.json', sleep_time=30, account='root', once=True, scope=shr_scope)
did_list_tst = list(DIDClient().list_dids(shr_scope, {}))
did_list_new = list(list_dids(shr_scope, {}, **self.new_vo))
assert_not_equal(len(did_list_tst), 0)
assert_equal(len(did_list_new), 0)
did_dicts = [{'scope': shr_scope, 'name': n} for n in did_list_tst]
replicas_tst = list(ReplicaClient().list_replicas(did_dicts, rse_expression=shr_rse))
replicas_new = list(list_replicas(did_dicts, rse_expression=shr_rse, **self.new_vo))
assert_not_equal(len(replicas_tst), 0)
assert_equal(len(replicas_new), 0)
def setUp(self):
""" Setup the Test Case """
self.did_client = DIDClient()
self.meta_client = MetaClient()
self.rse_client = RSEClient()
self.scope_client = ScopeClient()
def test_subscriptions_at_different_vos(self):
""" MULTI VO (CLIENT): Test that subscriptions from 2nd vo don't interfere """
account_client = AccountClient()
usr_uuid = str(generate_uuid()).lower()[:16]
shr_acc = 'shr-%s' % usr_uuid
account_client.add_account(shr_acc, 'USER', '*****@*****.**')
add_account(shr_acc, 'USER', '*****@*****.**', 'root', **self.new_vo)
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
tst_scope = 'tst_%s' % scope_uuid
new_scope = 'new_%s' % scope_uuid
scope_client.add_scope('root', tst_scope)
add_scope(new_scope, 'root', 'root', **self.new_vo)
did_client = DIDClient()
did_uuid = str(generate_uuid()).lower()[:16]
tst_did = 'tstset_%s' % did_uuid
new_did = 'newset_%s' % did_uuid
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst_rse1 = 'TST1_%s' % rse_str
tst_rse2 = 'TST2_%s' % rse_str
new_rse1 = 'NEW1_%s' % rse_str
new_rse2 = 'NEW2_%s' % rse_str
rse_client.add_rse(tst_rse1)
rse_client.add_rse(tst_rse2)
add_rse(new_rse1, 'root', **self.new_vo)
add_rse(new_rse2, 'root', **self.new_vo)
acc_lim_client = AccountLimitClient()
acc_lim_client.set_local_account_limit(shr_acc, tst_rse1, 10)
acc_lim_client.set_local_account_limit(shr_acc, tst_rse2, 10)
set_local_account_limit(shr_acc, new_rse1, 10, 'root', **self.new_vo)
set_local_account_limit(shr_acc, new_rse2, 10, 'root', **self.new_vo)
did_client.add_did(tst_scope, tst_did, 'DATASET', rse=tst_rse1)
add_did(new_scope, new_did, 'DATASET', 'root', rse=new_rse1, **self.new_vo)
sub_client = SubscriptionClient()
sub_str = generate_uuid()
tst_sub = 'tstsub_' + sub_str
new_sub = 'newsub_' + sub_str
shr_sub = 'shrsub_' + sub_str
tst_sub_id = sub_client.add_subscription(tst_sub, shr_acc, {'scope': [tst_scope]},
[{'copies': 1, 'rse_expression': tst_rse2, 'weight': 0,
'activity': 'User Subscriptions'}],
'', None, 0, 0)
shr_tst_sub_id = sub_client.add_subscription(shr_sub, shr_acc, {'scope': [tst_scope]},
[{'copies': 1, 'rse_expression': tst_rse2, 'weight': 0,
'activity': 'User Subscriptions'}],
'', None, 0, 0)
new_sub_id = add_subscription(new_sub, shr_acc, {'scope': [new_scope]},
[{'copies': 1, 'rse_expression': new_rse2, 'weight': 0, 'activity': 'User Subscriptions'}],
'', False, 0, 0, 3, 'root', **self.new_vo)
shr_new_sub_id = add_subscription(shr_sub, shr_acc, {'scope': [new_scope]},
[{'copies': 1, 'rse_expression': new_rse2, 'weight': 0, 'activity': 'User Subscriptions'}],
'', False, 0, 0, 3, 'root', **self.new_vo)
tst_subs = [s['id'] for s in sub_client.list_subscriptions()]
assert_in(tst_sub_id, tst_subs)
assert_in(shr_tst_sub_id, tst_subs)
assert_not_in(new_sub_id, tst_subs)
assert_not_in(shr_new_sub_id, tst_subs)
new_subs = [s['id'] for s in list_subscriptions(**self.new_vo)]
assert_in(new_sub_id, new_subs)
assert_in(shr_new_sub_id, new_subs)
assert_not_in(tst_sub_id, new_subs)
assert_not_in(shr_tst_sub_id, new_subs)
shr_tst_subs = [s['id'] for s in sub_client.list_subscriptions(name=shr_sub)]
assert_in(shr_tst_sub_id, shr_tst_subs)
assert_not_in(shr_new_sub_id, shr_tst_subs)
shr_new_subs = [s['id'] for s in list_subscriptions(name=shr_sub, **self.new_vo)]
assert_in(shr_new_sub_id, shr_new_subs)
assert_not_in(shr_tst_sub_id, shr_new_subs)
acc_tst_subs = [s['id'] for s in sub_client.list_subscriptions(account=shr_acc)]
assert_in(tst_sub_id, acc_tst_subs)
assert_in(shr_tst_sub_id, acc_tst_subs)
assert_not_in(new_sub_id, acc_tst_subs)
assert_not_in(shr_new_sub_id, acc_tst_subs)
acc_new_subs = [s['id'] for s in list_subscriptions(account=shr_acc, **self.new_vo)]
assert_in(new_sub_id, acc_new_subs)
assert_in(shr_new_sub_id, acc_new_subs)
assert_not_in(tst_sub_id, acc_new_subs)
assert_not_in(shr_tst_sub_id, acc_new_subs)
def setUp(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
def scope_client():
from rucio.client.scopeclient import ScopeClient
return ScopeClient()
def __init__(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
def _rucio_register(beamline, uid, filenames):
"""
Register the file in rucio for replication to SDCC.
"""
scope = beamline
container = uid
replica_client = ReplicaClient()
didclient = DIDClient()
scopeclient = ScopeClient()
ruleclient = RuleClient()
for root, ending, filename in filenames:
#size = os.stat(str(filename)).st_size
#adler = adler32(str(filename))
files = [{
'scope': scope,
'name': filename.split('/')[-1],
'bytes': 1000,
#'adler32': "unknown",
'pfn': pfn + filename
}]
dataset = os.path.join(root, ending)
dataset = '.'.join(dataset.split('/')[1:-1])
print("DATASET", dataset)
breakpoint()
try:
scopeclient.add_scope(account='nsls2data', scope=scope)
except rucio.common.exception.Duplicate:
pass
replica_client.add_replicas(rse=rse, files=files)
# Create a new container if it doesn't exist.
try:
didclient.add_did(scope=scope, name=uid, type='container')
except rucio.common.exception.DataIdentifierAlreadyExists:
pass
# Create a replication rule.
try:
dids = [{'scope': scope, 'name': container}]
ruleclient.add_replication_rule(
dids=dids,
copies=1,
rse_expression='SDCC',
lifetime=86400, # Seconds
account='nsls2data',
source_replica_expression='NSLS2',
purge_replicas=True,
comment='purge_replicas in 24 hours')
except rucio.common.exception.DuplicateRule:
pass
# Create a new dataset if it doesn't exist.
try:
didclient.add_did(scope=scope, name=dataset, type='dataset')
except rucio.common.exception.DataIdentifierAlreadyExists:
pass
attachment = {
'scope': scope,
'name': uid,
'dids': [{
'scope': scope,
'name': dataset
}]
}
try:
didclient.add_files_to_dataset(scope, dataset, files)
except rucio.common.exception.FileAlreadyExists:
pass
try:
didclient.add_datasets_to_containers([attachment])
except rucio.common.exception.DuplicateContent:
pass
