class TestScopeClient():
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
def test_create_scope(self):
""" SCOPE (CLIENTS): create a new scope."""
account = 'jdoe'
scope = scope_name_generator()
ret = self.scope_client.add_scope(account, scope)
assert_true(ret)
with assert_raises(InvalidObject):
self.scope_client.add_scope(account, 'tooooolooooongscooooooooooooope')
with assert_raises(InvalidObject):
self.scope_client.add_scope(account, '$?!')
@raises(AccountNotFound)
def test_create_scope_no_account(self):
""" SCOPE (CLIENTS): try to create scope for not existing account."""
account = str(uuid()).lower()[:30]
scope = scope_name_generator()
self.scope_client.add_scope(account, scope)
@raises(Duplicate)
def test_create_scope_duplicate(self):
""" SCOPE (CLIENTS): try to create a duplicate scope."""
account = 'jdoe'
scope = scope_name_generator()
self.scope_client.add_scope(account, scope)
self.scope_client.add_scope(account, scope)
def test_list_scopes(self):
""" SCOPE (CLIENTS): try to list scopes for an account."""
account = 'jdoe'
scope_list = [scope_name_generator() for i in xrange(5)]
for scope in scope_list:
self.scope_client.add_scope(account, scope)
svr_list = self.scope_client.list_scopes_for_account(account)
for scope in scope_list:
if scope not in svr_list:
assert_true(False)
@raises(AccountNotFound)
def test_list_scopes_account_not_found(self):
""" SCOPE (CLIENTS): try to list scopes for a non existing account."""
account = account_name_generator()
self.scope_client.list_scopes_for_account(account)
@raises(ScopeNotFound)
def test_list_scopes_no_scopes(self):
""" SCOPE (CLIENTS): try to list scopes for an account without scopes."""
account = account_name_generator()
self.account_client.add_account(account, 'USER')
self.scope_client.list_scopes_for_account(account)
def setUp(self):
self.account = InternalAccount('root')
self.scope = InternalScope('mock')
self.upload_client = UploadClient()
self.account_client = AccountClient()
self.file_sizes = 2
self.rse = 'MOCK4'
self.rse_id = get_rse_id(self.rse)
self.session = get_session()
class TestScopeClient(unittest.TestCase):
def setUp(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
def test_create_scope(self):
""" SCOPE (CLIENTS): create a new scope."""
account = 'jdoe'
scope = scope_name_generator()
ret = self.scope_client.add_scope(account, scope)
assert ret
with pytest.raises(InvalidObject):
self.scope_client.add_scope(account,
'tooooolooooongscooooooooooooope')
with pytest.raises(InvalidObject):
self.scope_client.add_scope(account, '$?!')
def test_create_scope_no_account(self):
""" SCOPE (CLIENTS): try to create scope for not existing account."""
account = str(uuid()).lower()[:30]
scope = scope_name_generator()
with pytest.raises(AccountNotFound):
self.scope_client.add_scope(account, scope)
def test_create_scope_duplicate(self):
""" SCOPE (CLIENTS): try to create a duplicate scope."""
account = 'jdoe'
scope = scope_name_generator()
self.scope_client.add_scope(account, scope)
with pytest.raises(Duplicate):
self.scope_client.add_scope(account, scope)
def test_list_scopes(self):
""" SCOPE (CLIENTS): try to list scopes for an account."""
account = 'jdoe'
scope_list = [scope_name_generator() for _ in range(5)]
for scope in scope_list:
self.scope_client.add_scope(account, scope)
svr_list = self.scope_client.list_scopes_for_account(account)
for scope in scope_list:
if scope not in svr_list:
assert False
def test_list_scopes_account_not_found(self):
""" SCOPE (CLIENTS): try to list scopes for a non existing account."""
account = account_name_generator()
with pytest.raises(AccountNotFound):
self.scope_client.list_scopes_for_account(account)
def test_list_scopes_no_scopes(self):
""" SCOPE (CLIENTS): try to list scopes for an account without scopes."""
account = account_name_generator()
self.account_client.add_account(account, 'USER', '*****@*****.**')
with pytest.raises(ScopeNotFound):
self.scope_client.list_scopes_for_account(account)
def setUp(self):
self.account = 'root'
self.scope = 'mock'
self.upload_client = UploadClient()
self.account_client = AccountClient()
self.file_sizes = 2
self.rse = 'MOCK4'
self.rse_id = get_rse(self.rse).id
self.session = get_session()
def setUp(self):
self.account_client = AccountClient()
self.rse_client = RSEClient()
self.account = generate_uuid()[:10]
self.rse = rse_name_generator()
self.account_client.add_account(self.account, 'SERVICE',
'*****@*****.**')
self.rse_client.add_rse(self.rse)
def create_accounts(account_list, user_type):
""" Registers a set of accounts
:param account_list: the list of accounts to be added
:param user_type: the type of accounts
"""
account_client = AccountClient()
for account in account_list:
try:
account_client.add_account(account, user_type, email=None)
except exception.Duplicate:
pass # Account already exists, no need to create it
def whoami(account='natasha', auth_type='x509_proxy'):
"""Runs whoami command for a given account via client tool,
requires a valid proxy
"""
if args.password and args.username:
creds = {'username': args.username, 'password': args.password}
account_client = AccountClient(account=account,
auth_type='userpass',
creds=creds,
rucio_host=args.host,
auth_host=args.auth_host)
else:
account_client = AccountClient(account=account, auth_type='x509_proxy')
print("Connected to rucio as %s" % account_client.whoami()['account'])
def setUp(self):
self.rse = 'MOCK4'
self.file_sizes = 2
self.upload_client = UploadClient()
self.account_client = AccountClient()
self.session = get_session()
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
self.account = InternalAccount('root', **self.vo)
self.scope = InternalScope('mock', **self.vo)
self.rse_id = get_rse_id(self.rse, session=self.session, **self.vo)
class TestAbacusAccount(unittest.TestCase):
def setUp(self):
self.rse = 'MOCK4'
self.file_sizes = 2
self.upload_client = UploadClient()
self.account_client = AccountClient()
self.session = get_session()
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
self.account = InternalAccount('root', **self.vo)
self.scope = InternalScope('mock', **self.vo)
self.rse_id = get_rse_id(self.rse, session=self.session, **self.vo)
def tearDown(self):
undertaker.run(once=True)
cleaner.run(once=True)
if self.vo:
reaper.run(once=True, include_rses='vo=%s&(%s)' % (self.vo['vo'], self.rse), greedy=True)
else:
reaper.run(once=True, include_rses=self.rse, greedy=True)
def test_abacus_account(self):
""" ABACUS (ACCOUNT): Test update of account usage """
self.session.query(models.UpdatedAccountCounter).delete() # pylint: disable=no-member
self.session.query(models.AccountUsage).delete() # pylint: disable=no-member
self.session.commit() # pylint: disable=no-member
# Upload files -> account usage should increase
self.files = [{'did_scope': self.scope.external, 'did_name': 'file_' + generate_uuid(), 'path': file_generator(size=self.file_sizes), 'rse': self.rse, 'lifetime': -1} for i in range(0, 2)]
self.upload_client.upload(self.files)
[os.remove(file['path']) for file in self.files]
account.run(once=True)
account_usage = get_local_account_usage(account=self.account, rse_id=self.rse_id)[0]
assert account_usage['bytes'] == len(self.files) * self.file_sizes
assert account_usage['files'] == len(self.files)
# Update and check the account history with the core method
update_account_counter_history(account=self.account, rse_id=self.rse_id)
usage_history = get_usage_history(rse_id=self.rse_id, account=self.account)
assert usage_history[-1]['bytes'] == len(self.files) * self.file_sizes
assert usage_history[-1]['files'] == len(self.files)
# Check the account history with the client
usage_history = self.account_client.get_account_usage_history(rse=self.rse, account=self.account.external)
assert usage_history[-1]['bytes'] == len(self.files) * self.file_sizes
assert usage_history[-1]['files'] == len(self.files)
# Delete rules -> account usage should decrease
cleaner.run(once=True)
account.run(once=True)
# set account limit because return value of get_local_account_usage differs if a limit is set or not
set_local_account_limit(account=self.account, rse_id=self.rse_id, bytes=10)
account_usages = get_local_account_usage(account=self.account, rse_id=self.rse_id)[0]
assert account_usages['bytes'] == 0
assert account_usages['files'] == 0
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def test_accounts_at_different_vos(self):
""" MULTI VO (CLIENT): Test that accounts from 2nd vo don't interfere """
account_client = AccountClient()
usr_uuid = str(generate_uuid()).lower()[:16]
tst = 'tst-%s' % usr_uuid
new = 'new-%s' % usr_uuid
shr = 'shr-%s' % usr_uuid
account_client.add_account(tst, 'USER', '*****@*****.**')
account_client.add_account(shr, 'USER', '*****@*****.**')
add_account(new, 'USER', '*****@*****.**', 'root', **self.new_vo)
add_account(shr, 'USER', '*****@*****.**', 'root', **self.new_vo)
account_list_tst = [a['account'] for a in account_client.list_accounts()]
account_list_new = [a['account'] for a in list_accounts(filter={}, **self.new_vo)]
assert_true(tst in account_list_tst)
assert_false(new in account_list_tst)
assert_true(shr in account_list_tst)
assert_false(tst in account_list_new)
assert_true(new in account_list_new)
assert_true(shr in account_list_new)
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
self.account_client = AccountClient()
self.scope_client = ScopeClient()
self.meta_client = MetaClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.rse_client = RSEClient()
def setUpClass(cls):
cls.upload_client = UploadClient()
cls.account_client = AccountClient()
cls.session = get_session()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
cls.account = InternalAccount('root', **cls.vo)
cls.scope = InternalScope('mock', **cls.vo)
cls.rse_id = get_rse_id(cls.rse, session=cls.session, **cls.vo)
class TestAccountClient():
@classmethod
def setUpClass(cls):
# Add test account
cls.account = ''.join(random.choice(string.ascii_uppercase) for x in range(10))
add_account(account=cls.account, type=AccountType.USER)
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse2 = 'MOCK2'
cls.rse1_id = get_rse(cls.rse1).id
cls.rse2_id = get_rse(cls.rse2).id
def setup(self):
self.client = AccountClient()
self.alclient = AccountLimitClient()
def test_listing_account_limits(self):
""" ACCOUNT (CLIENTS): Test listing account limits """
account_limit.set_account_limit(account=self.account, rse_id=self.rse1_id, bytes=12345)
account_limit.set_account_limit(account=self.account, rse_id=self.rse2_id, bytes=12345)
limits = self.client.get_account_limits(account=self.account)
assert_in((self.rse1, 12345), limits.items())
assert_in((self.rse2, 12345), limits.items())
account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
account_limit.delete_account_limit(account=self.account, rse_id=self.rse2_id)
def test_listing_account_limit(self):
""" ACCOUNT (CLIENTS): Test listing account limit """
account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
account_limit.set_account_limit(account=self.account, rse_id=self.rse1_id, bytes=333)
limit = self.client.get_account_limit(account=self.account, rse=self.rse1)
assert_equal(limit, {self.rse1: 333})
account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
def test_setting_account_limit(self):
""" ACCOUNTLIMIT (CLIENTS): Test setting account limit """
self.alclient.set_account_limit(account=self.account, rse=self.rse1, bytes=987)
limit = self.client.get_account_limit(account=self.account, rse=self.rse1)
assert_equal(limit[self.rse1], 987)
account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
def test_deleting_account_limit(self):
""" ACCOUNTLIMIT (CLIENTS): Test deleting account limit """
self.alclient.set_account_limit(account=self.account, rse=self.rse1, bytes=786)
limit = self.client.get_account_limit(account=self.account, rse=self.rse1)
assert_equal(limit, {self.rse1: 786})
self.alclient.delete_account_limit(account=self.account, rse=self.rse1)
limit = self.client.get_account_limit(account=self.account, rse=self.rse1)
assert_equal(limit[self.rse1], float('Inf'))
account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
def setup(self):
self.client = AccountClient()
self.alclient = AccountLimitClient()
def test_subscriptions_at_different_vos(self):
""" MULTI VO (CLIENT): Test that subscriptions from 2nd vo don't interfere """
account_client = AccountClient()
usr_uuid = str(generate_uuid()).lower()[:16]
shr_acc = 'shr-%s' % usr_uuid
account_client.add_account(shr_acc, 'USER', '*****@*****.**')
add_account(shr_acc, 'USER', '*****@*****.**', 'root', **self.new_vo)
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
tst_scope = 'tst_%s' % scope_uuid
new_scope = 'new_%s' % scope_uuid
scope_client.add_scope('root', tst_scope)
add_scope(new_scope, 'root', 'root', **self.new_vo)
did_client = DIDClient()
did_uuid = str(generate_uuid()).lower()[:16]
tst_did = 'tstset_%s' % did_uuid
new_did = 'newset_%s' % did_uuid
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst_rse1 = 'TST1_%s' % rse_str
tst_rse2 = 'TST2_%s' % rse_str
new_rse1 = 'NEW1_%s' % rse_str
new_rse2 = 'NEW2_%s' % rse_str
rse_client.add_rse(tst_rse1)
rse_client.add_rse(tst_rse2)
add_rse(new_rse1, 'root', **self.new_vo)
add_rse(new_rse2, 'root', **self.new_vo)
acc_lim_client = AccountLimitClient()
acc_lim_client.set_local_account_limit(shr_acc, tst_rse1, 10)
acc_lim_client.set_local_account_limit(shr_acc, tst_rse2, 10)
set_local_account_limit(shr_acc, new_rse1, 10, 'root', **self.new_vo)
set_local_account_limit(shr_acc, new_rse2, 10, 'root', **self.new_vo)
did_client.add_did(tst_scope, tst_did, 'DATASET', rse=tst_rse1)
add_did(new_scope, new_did, 'DATASET', 'root', rse=new_rse1, **self.new_vo)
sub_client = SubscriptionClient()
sub_str = generate_uuid()
tst_sub = 'tstsub_' + sub_str
new_sub = 'newsub_' + sub_str
shr_sub = 'shrsub_' + sub_str
tst_sub_id = sub_client.add_subscription(tst_sub, shr_acc, {'scope': [tst_scope]},
[{'copies': 1, 'rse_expression': tst_rse2, 'weight': 0,
'activity': 'User Subscriptions'}],
'', None, 0, 0)
shr_tst_sub_id = sub_client.add_subscription(shr_sub, shr_acc, {'scope': [tst_scope]},
[{'copies': 1, 'rse_expression': tst_rse2, 'weight': 0,
'activity': 'User Subscriptions'}],
'', None, 0, 0)
new_sub_id = add_subscription(new_sub, shr_acc, {'scope': [new_scope]},
[{'copies': 1, 'rse_expression': new_rse2, 'weight': 0, 'activity': 'User Subscriptions'}],
'', False, 0, 0, 3, 'root', **self.new_vo)
shr_new_sub_id = add_subscription(shr_sub, shr_acc, {'scope': [new_scope]},
[{'copies': 1, 'rse_expression': new_rse2, 'weight': 0, 'activity': 'User Subscriptions'}],
'', False, 0, 0, 3, 'root', **self.new_vo)
tst_subs = [s['id'] for s in sub_client.list_subscriptions()]
assert_in(tst_sub_id, tst_subs)
assert_in(shr_tst_sub_id, tst_subs)
assert_not_in(new_sub_id, tst_subs)
assert_not_in(shr_new_sub_id, tst_subs)
new_subs = [s['id'] for s in list_subscriptions(**self.new_vo)]
assert_in(new_sub_id, new_subs)
assert_in(shr_new_sub_id, new_subs)
assert_not_in(tst_sub_id, new_subs)
assert_not_in(shr_tst_sub_id, new_subs)
shr_tst_subs = [s['id'] for s in sub_client.list_subscriptions(name=shr_sub)]
assert_in(shr_tst_sub_id, shr_tst_subs)
assert_not_in(shr_new_sub_id, shr_tst_subs)
shr_new_subs = [s['id'] for s in list_subscriptions(name=shr_sub, **self.new_vo)]
assert_in(shr_new_sub_id, shr_new_subs)
assert_not_in(shr_tst_sub_id, shr_new_subs)
acc_tst_subs = [s['id'] for s in sub_client.list_subscriptions(account=shr_acc)]
assert_in(tst_sub_id, acc_tst_subs)
assert_in(shr_tst_sub_id, acc_tst_subs)
assert_not_in(new_sub_id, acc_tst_subs)
assert_not_in(shr_new_sub_id, acc_tst_subs)
acc_new_subs = [s['id'] for s in list_subscriptions(account=shr_acc, **self.new_vo)]
assert_in(new_sub_id, acc_new_subs)
assert_in(shr_new_sub_id, acc_new_subs)
assert_not_in(tst_sub_id, acc_new_subs)
assert_not_in(shr_tst_sub_id, acc_new_subs)
from rucio.client import Client
from rucio.client.didclient import DIDClient
from rucio.client.replicaclient import ReplicaClient
from rucio.client.rseclient import RSEClient
from rucio.client.ruleclient import RuleClient
from rucio.client.accountclient import AccountClient
from rucio.common import exception
DID = namedtuple('DID', ['scope', 'name', 'ftype', 'owner', 'path', 'dataset'])
did_client = DIDClient()
rep_client = ReplicaClient()
rse_client = RSEClient()
rule_client = RuleClient()
admin_client = AccountClient()
def list_uk_localgroupdisks():
"""Return a list of known uk localgroup disks
"""
global rse_client
r = rse_client.list_rses(
rse_expression='spacetoken=ATLASLOCALGROUPDISK&cloud=UK')
return sorted(x['rse'] for x in r)
def get_rse_usage(rse, filters=None):
return rse_client.get_rse_usage(rse, filters=filters)
#!/usr/bin/env python
from __future__ import print_function
from multiprocessing import Pool
from rucio.client import Client
from rucio.client.accountclient import AccountClient
rclient = Client()
accclient = AccountClient()
MY_RSE='UNI-BONN_LOCALGROUPDISK'
def get_usage(acc):
usage = list(accclient.get_local_account_usage(acc['account'], MY_RSE))
if usage:
return usage[0]
else:
return {}
pool = Pool(24)
accdict = list(accclient.list_accounts(filters={'country-de' : 'user'})) + list(accclient.list_accounts(filters={'country-de' : 'admin'}))
res = pool.map(get_usage, accdict)
print("name","mail","identity","files","bytes","limit",sep="\t")
for A in range(0,len(accdict)):
if res[A] and res[A]['bytes']!=0:
limit = accclient.get_local_account_limit(accdict[A]['account'], MY_RSE)
print(accdict[A]['account'],accdict[A]['email'],res[A]['files'],res[A]['bytes'],limit[MY_RSE],sep="\t")
#identities = list(accclient.list_identities(accdict[A]['account']))
#for ident in range(0,len(identities)):
# if ident == 0:
class TestAccountClient():
def setup(self):
self.client = AccountClient()
def test_add_account_success(self):
""" ACCOUNT (CLIENTS): create a new account and get information about account."""
account = account_name_generator()
type = 'USER'
ret = self.client.add_account(account, type)
assert_true(ret)
with assert_raises(Duplicate):
self.client.add_account(account, type)
with assert_raises(InvalidObject):
self.client.add_account('BAD_ACCOUNT_NAME', type)
with assert_raises(InvalidObject):
self.client.add_account('toooooooloooooonaccounnnnnnnntnammmmme', type)
acc_info = self.client.get_account(account)
assert_equal(acc_info['account'], account)
@raises(AccountNotFound)
def test_get_account_notfound(self):
""" ACCOUNT (CLIENTS): try to get information about not existing account."""
account = str(uuid())
self.client.get_account(account)
def test_list_accounts(self):
""" ACCOUNT (CLIENTS): get list of all accounts."""
dn = '/C=CH/ST=Geneva/O=CERN/OU=PH-ADP-CO/CN=DDMLAB Client Certificate/[email protected]'
acc_list = [account_name_generator() for i in xrange(5)]
for account in acc_list:
self.client.add_account(account, 'USER')
svr_list = [a['account'] for a in self.client.list_accounts(account_type='SERVICE', identity=dn)]
assert_true('root' in svr_list)
svr_list = [a['account'] for a in self.client.list_accounts(account_type='USER')]
for account in acc_list:
assert_true(account in svr_list)
def setup(self):
self.rule_client = RuleClient()
self.did_client = DIDClient()
self.subscription_client = SubscriptionClient()
self.account_client = AccountClient()
self.lock_client = LockClient()
class TestAccountClient():
@classmethod
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': 'tst'}
else:
cls.vo = {}
# Add test account
cls.account = InternalAccount(
''.join(random.choice(string.ascii_uppercase) for x in range(10)),
**cls.vo)
add_account(account=cls.account,
type=AccountType.USER,
email='*****@*****.**')
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse2 = 'MOCK2'
cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo)
cls.rse2_id = get_rse_id(rse=cls.rse2, **cls.vo)
def setup(self):
self.client = AccountClient()
self.alclient = AccountLimitClient()
def test_listing_account_limits(self):
""" ACCOUNT (CLIENTS): Test listing account limits """
account_limit.set_account_limit(account=self.account,
rse_id=self.rse1_id,
bytes=12345)
account_limit.set_account_limit(account=self.account,
rse_id=self.rse2_id,
bytes=12345)
limits = self.client.get_account_limits(account=self.account.external)
assert_in((self.rse1, 12345), limits.items())
assert_in((self.rse2, 12345), limits.items())
account_limit.delete_account_limit(account=self.account,
rse_id=self.rse1_id)
account_limit.delete_account_limit(account=self.account,
rse_id=self.rse2_id)
def test_listing_account_limit(self):
""" ACCOUNT (CLIENTS): Test listing account limit """
account_limit.delete_account_limit(account=self.account,
rse_id=self.rse1_id)
account_limit.set_account_limit(account=self.account,
rse_id=self.rse1_id,
bytes=333)
limit = self.client.get_account_limit(account=self.account.external,
rse=self.rse1)
assert_equal(limit, {self.rse1: 333})
account_limit.delete_account_limit(account=self.account,
rse_id=self.rse1_id)
def test_setting_account_limit(self):
""" ACCOUNTLIMIT (CLIENTS): Test setting account limit """
self.alclient.set_account_limit(account=self.account.external,
rse=self.rse1,
bytes=987)
limit = self.client.get_account_limit(account=self.account.external,
rse=self.rse1)
assert_equal(limit[self.rse1], 987)
account_limit.delete_account_limit(account=self.account,
rse_id=self.rse1_id)
def test_deleting_account_limit(self):
""" ACCOUNTLIMIT (CLIENTS): Test deleting account limit """
self.alclient.set_account_limit(account=self.account.external,
rse=self.rse1,
bytes=786)
limit = self.client.get_account_limit(account=self.account.external,
rse=self.rse1)
assert_equal(limit, {self.rse1: 786})
self.alclient.delete_account_limit(account=self.account.external,
rse=self.rse1)
limit = self.client.get_account_limit(account=self.account.external,
rse=self.rse1)
assert_equal(limit[self.rse1], None)
account_limit.delete_account_limit(account=self.account,
rse_id=self.rse1_id)
class TestBoolean(unittest.TestCase):
def setUp(self):
self.account_client = AccountClient()
self.rse_client = RSEClient()
self.account = generate_uuid()[:10]
self.rse = rse_name_generator()
self.account_client.add_account(self.account, 'SERVICE',
'*****@*****.**')
self.rse_client.add_rse(self.rse)
def tearDown(self):
self.account_client.delete_account(self.account)
self.rse_client.delete_rse(self.rse)
def test_booleanstring_account_attribute(self):
self.account_client.add_account_attribute(self.account,
'teststringtrue', 'true')
self.account_client.add_account_attribute(self.account, 'testinttrue',
'1')
self.account_client.add_account_attribute(self.account,
'teststringfalse', 'false')
self.account_client.add_account_attribute(self.account, 'testintfalse',
'0')
result = {}
for account in self.account_client.list_account_attributes(
self.account):
for res in account:
result[res['key']] = res['value']
assert result['teststringtrue'] is True
assert result['testinttrue'] == '1'
assert result['teststringfalse'] is False
assert result['testintfalse'] == '0'
def test_booleanstring_rse_attribute(self):
self.rse_client.add_rse_attribute(self.rse, 'teststringtrue', 'true')
self.rse_client.add_rse_attribute(self.rse, 'testinttrue', '1')
self.rse_client.add_rse_attribute(self.rse, 'teststringfalse', 'false')
self.rse_client.add_rse_attribute(self.rse, 'testintfalse', '0')
result = self.rse_client.list_rse_attributes(self.rse)
assert result['teststringtrue'] is True
assert result['testinttrue'] == '1'
assert result['teststringfalse'] is False
assert result['testintfalse'] == '0'
#!/usr/bin/env python
from __future__ import print_function
from multiprocessing import Pool
from rucio.client import Client
from rucio.client.accountclient import AccountClient
from rucio.client.accountlimitclient import AccountLimitClient
import getopt
import sys
rclient = Client()
accclient = AccountClient()
acclimitclient = AccountLimitClient()
def bytes_to_floored_tb(space):
return space // (1000 * 1000 * 1000 * 1000)
def tb_to_bytes(space):
return space * 1000 * 1000 * 1000 * 1000
MY_RSE = 'UNI-BONN_LOCALGROUPDISK'
MY_DEF_QUOTA = 10 * 1000 * 1000 * 1000 * 1000
MY_ADM_DEF_QUOTA = 50 * 1000 * 1000 * 1000 * 1000
DEF_IN_TB = bytes_to_floored_tb(MY_DEF_QUOTA)
BREATHE_SPACE_TB = 5
STEP_SPACE_TB = 10
def setUp(self):
self.client = AccountClient()
self.alclient = AccountLimitClient()
self.db_session.query(models.AccountLimit).delete()
self.db_session.query(models.AccountGlobalLimit).delete()
self.db_session.commit()
class TestAccountClient(unittest.TestCase):
@classmethod
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': get_vo()}
cls.multi_vo = True
else:
cls.vo = {}
cls.multi_vo = False
# Add test account
cls.account = InternalAccount(''.join(random.choice(string.ascii_uppercase) for x in range(10)), **cls.vo)
add_account(account=cls.account, type_=AccountType.USER, email='*****@*****.**')
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse2 = 'MOCK2'
cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo)
cls.rse2_id = get_rse_id(rse=cls.rse2, **cls.vo)
cls.db_session = session.get_session()
@classmethod
def tearDownClass(cls):
cls.db_session.query(models.AccountLimit).delete()
cls.db_session.query(models.AccountGlobalLimit).delete()
cls.db_session.commit()
cls.db_session.close()
def setUp(self):
self.client = AccountClient()
self.alclient = AccountLimitClient()
self.db_session.query(models.AccountLimit).delete()
self.db_session.query(models.AccountGlobalLimit).delete()
self.db_session.commit()
def test_set_global_account_limit(self):
""" ACCOUNT_LIMIT (CLIENTS): Set global account limit """
self.alclient.set_global_account_limit(self.account.external, 'MOCK', 200000)
assert account_limit.get_global_account_limit(account=self.account, rse_expression='MOCK') == 200000
assert account_limit.get_global_account_limit(account=self.account, rse_expression='MOCK2') is None
def test_get_global_account_limits(self):
""" ACCOUNT_LIMIT (CLIENTS): Get global account limits """
expression = 'MOCK'
resolved_rses = ['MOCK']
resolved_rse_ids = [get_rse_id('MOCK', **self.vo)]
limit = 10
account_limit.set_global_account_limit(self.account, expression, limit)
results = self.client.get_global_account_limits(account=self.account.external)
assert len(results) == 1
assert results[expression]['resolved_rses'] == resolved_rses
assert results[expression]['resolved_rse_ids'] == resolved_rse_ids
assert results[expression]['limit'] == limit
def test_get_global_account_limit(self):
""" ACCOUNT_LIMIT (CLIENTS): Get global account limit. """
expression = 'MOCK'
limit = 10
account_limit.set_global_account_limit(self.account, expression, limit)
result = self.client.get_global_account_limit(account=self.account.external, rse_expression=expression)
assert result[expression] == limit
def test_get_local_account_limits(self):
""" ACCOUNT_LIMIT (CLIENTS): Get local account limits """
account_limit.set_local_account_limit(account=self.account, rse_id=self.rse1_id, bytes_=12345)
account_limit.set_local_account_limit(account=self.account, rse_id=self.rse2_id, bytes_=12345)
limits = self.client.get_local_account_limits(account=self.account.external)
assert (self.rse1, 12345) in limits.items()
assert (self.rse2, 12345) in limits.items()
account_limit.delete_local_account_limit(account=self.account, rse_id=self.rse1_id)
account_limit.delete_local_account_limit(account=self.account, rse_id=self.rse2_id)
def test_get_local_account_limit(self):
""" ACCOUNT_LIMIT (CLIENTS): Get local account limit """
account_limit.delete_local_account_limit(account=self.account, rse_id=self.rse1_id)
account_limit.set_local_account_limit(account=self.account, rse_id=self.rse1_id, bytes_=333)
limit = self.client.get_local_account_limit(account=self.account.external, rse=self.rse1)
assert limit == {self.rse1: 333}
account_limit.delete_local_account_limit(account=self.account, rse_id=self.rse1_id)
def test_set_local_account_limit(self):
""" ACCOUNTLIMIT (CLIENTS): Set local account limit """
self.alclient.set_local_account_limit(account=self.account.external, rse=self.rse1, bytes_=987)
limit = self.client.get_local_account_limit(account=self.account.external, rse=self.rse1)
assert limit[self.rse1] == 987
account_limit.delete_local_account_limit(account=self.account, rse_id=self.rse1_id)
def test_delete_local_account_limit(self):
""" ACCOUNTLIMIT (CLIENTS): Delete local account limit """
self.alclient.set_local_account_limit(account=self.account.external, rse=self.rse1, bytes_=786)
limit = self.client.get_local_account_limit(account=self.account.external, rse=self.rse1)
assert limit == {self.rse1: 786}
self.alclient.delete_local_account_limit(account=self.account.external, rse=self.rse1)
limit = self.client.get_local_account_limit(account=self.account.external, rse=self.rse1)
assert limit[self.rse1] is None
account_limit.delete_local_account_limit(account=self.account, rse_id=self.rse1_id)
def test_delete_global_account_limit(self):
""" ACCOUNTLIMIT (CLIENTS): Delete global account limit """
rse_exp = 'MOCK'
account_limit.set_global_account_limit(account=self.account, rse_expression=rse_exp, bytes_=10, session=self.db_session)
self.alclient.delete_global_account_limit(account=self.account.external, rse_expression=rse_exp)
result = account_limit.get_global_account_limit(account=self.account, rse_expression=rse_exp)
assert result is None
def __init__(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
for user_record in cric_data:
dn = user_record['DN']
username = user_record['LOGIN']
if dn in current_dns:
new_accounts.add(username)
identity_map[dn].add(username)
# And the email for each DN
for email, record in cric_users.items():
if record['dn'] in current_dns:
email_map[record['dn']] = email
# Now we have new_accounts which tells us which accounts to create
# and identity_map which tells us how to map the identites to those accounts
ac = AccountClient()
# Make all the accounts from this list
for account in new_accounts:
try:
print('Add account %s' % account)
ac.add_account(account=account, type='USER', email=None)
except (Duplicate, InvalidObject):
print(' Account %s already exists or invalid' % account)
# Add all the identities to the list
for dn, accounts in identity_map.items():
email = email_map.get(dn, '*****@*****.**')
for account in accounts:
try:
print('Add identity %s for %s' % (dn, account))
def setup(self):
self.client = AccountClient()
def setup(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
def setUp(self):
self.account_client = AccountClient()
self.scope_client = ScopeClient()
def setup(self):
self.client = AccountClient()
self.alclient = AccountLimitClient()
def whoami(account='natasha', auth_type='x509_proxy'):
"""Runs whoami command for a given account via client tool,
requires a valid proxy
"""
account_client = AccountClient(account=account, auth_type='x509_proxy')
print("Connected to rucio as %s" % account_client.whoami()['account'])
def __init__(self):
self.client = AccountClient()
import json
from rucio.client.accountclient import AccountClient
try:
with open('accounts.json', 'r') as json_file:
all_accounts = json.load(json_file)
except IOError:
all_accounts = {}
try:
with open('dn_emails.json', 'r') as json_file:
dn_emails = json.load(json_file)
except IOError:
dn_emails = {}
ac = AccountClient()
for account in ac.list_accounts():
account_name = account['account']
account_email = account['email']
print(account)
if account_name not in all_accounts:
all_accounts.update({account_name: {'dns': set()}})
else:
all_accounts[account_name]['email'] = account_email
all_accounts[account_name]['dns'] = set(
all_accounts[account_name]['dns'])
for identity in ac.list_identities(account_name):
class TestAccountClient():
def __init__(self):
self.client = AccountClient()
def test_add_account_success(self):
""" ACCOUNT (CLIENTS): create a new account and get information about account."""
account = account_name_generator()
type, email = 'USER', '*****@*****.**'
ret = self.client.add_account(account, type, email)
assert_true(ret)
with assert_raises(Duplicate):
self.client.add_account(account, type, email)
with assert_raises(InvalidObject):
self.client.add_account('BAD_ACCOUNT_NAME', type, email)
with assert_raises(InvalidObject):
self.client.add_account('toooooooloooooonaccounnnnnnnntnammmmme',
type, email)
acc_info = self.client.get_account(account)
assert_equal(acc_info['account'], account)
@raises(AccountNotFound)
def test_get_account_notfound(self):
""" ACCOUNT (CLIENTS): try to get information about not existing account."""
account = str(uuid())
self.client.get_account(account)
def test_list_accounts(self):
""" ACCOUNT (CLIENTS): get list of all accounts."""
dn = config_get('bootstrap', 'x509_identity')
acc_list = [account_name_generator() for _ in range(5)]
for account in acc_list:
self.client.add_account(account, 'USER', '*****@*****.**')
svr_list = [
a['account']
for a in self.client.list_accounts(account_type='SERVICE',
identity=dn)
]
assert_true('root' in svr_list)
svr_list = [
a['account']
for a in self.client.list_accounts(account_type='USER')
]
for account in acc_list:
assert_true(account in svr_list)
def test_update_account(self):
""" ACCOUNT (CLIENTS): create a new account and update it."""
account = account_name_generator()
type, email = 'USER', '*****@*****.**'
ret = self.client.add_account(account, type, email)
assert_true(ret)
self.client.update_account(account=account,
key='status',
value='SUSPENDED')
status = self.client.get_account(account=account)['status']
assert_equal(status, 'SUSPENDED')
self.client.update_account(account=account,
key='status',
value='ACTIVE')
status = self.client.get_account(account=account)['status']
assert_equal(status, 'ACTIVE')
self.client.update_account(account=account, key='email', value='test')
email = self.client.get_account(account=account)['email']
assert_equal(email, 'test')
def setUp(self):
self.client = AccountClient()
class TestReplicationRuleClient():
@classmethod
def setUpClass(cls):
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse3 = 'MOCK3'
cls.rse4 = 'MOCK4'
cls.rse5 = 'MOCK5'
cls.rse1_id = get_rse(cls.rse1).id
cls.rse3_id = get_rse(cls.rse3).id
cls.rse4_id = get_rse(cls.rse4).id
cls.rse5_id = get_rse(cls.rse5).id
# Add Tags
cls.T1 = tag_generator()
cls.T2 = tag_generator()
add_rse_attribute(cls.rse1, cls.T1, True)
add_rse_attribute(cls.rse3, cls.T1, True)
add_rse_attribute(cls.rse4, cls.T2, True)
add_rse_attribute(cls.rse5, cls.T1, True)
# Add fake weights
add_rse_attribute(cls.rse1, "fakeweight", 10)
add_rse_attribute(cls.rse3, "fakeweight", 0)
add_rse_attribute(cls.rse4, "fakeweight", 0)
add_rse_attribute(cls.rse5, "fakeweight", 0)
def setup(self):
self.rule_client = RuleClient()
self.did_client = DIDClient()
self.subscription_client = SubscriptionClient()
self.account_client = AccountClient()
self.lock_client = LockClient()
def test_add_rule(self):
""" REPLICATION RULE (CLIENT): Add a replication rule """
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
ret = self.rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE')
assert_is_instance(ret, list)
def test_delete_rule(self):
""" REPLICATION RULE (CLIENT): Delete a replication rule """
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
ret = self.rule_client.delete_replication_rule(rule_id=rule_id)
assert(ret is True)
assert_raises(RuleNotFound, self.rule_client.delete_replication_rule, rule_id)
def test_list_rules_by_did(self):
""" DID (CLIENT): List Replication Rules per DID """
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
rule_id_2 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
ret = self.did_client.list_did_rules(scope=scope, name=dataset)
ids = [rule['id'] for rule in ret]
assert_in(rule_id_1, ids)
assert_in(rule_id_2, ids)
def test_get_rule(self):
""" REPLICATION RULE (CLIENT): Get Replication Rule by id """
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
ret = self.rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE')
get = self.rule_client.get_replication_rule(ret[0])
assert(ret[0] == get['id'])
def test_get_rule_by_account(self):
""" ACCOUNT (CLIENT): Get Replication Rule by account """
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
ret = self.rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE')
get = self.account_client.list_account_rules('jdoe')
rules = [rule['id'] for rule in get]
assert_in(ret[0], rules)
def test_locked_rule(self):
""" REPLICATION RULE (CLIENT): Delete a locked replication rule"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=True, subscription_id=None)[0]
assert_raises(AccessDenied, delete_rule, rule_id_1)
self.rule_client.update_replication_rule(rule_id=rule_id_1, options={'locked': False})
delete_rule(rule_id=rule_id_1)
def test_dataset_lock(self):
""" DATASETLOCK (CLIENT): Get a datasetlock for a specific dataset"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight='fakeweight', lifetime=None, locked=True, subscription_id=None)[0]
rule_ids = [lock['rule_id'] for lock in self.lock_client.get_dataset_locks(scope=scope, name=dataset)]
assert_in(rule_id_1, rule_ids)
def test_change_rule_lifetime(self):
""" REPLICATION RULE (CLIENT): Change rule lifetime"""
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight='fakeweight', lifetime=150, locked=True, subscription_id=None)[0]
get = self.rule_client.get_replication_rule(rule_id_1)
self.rule_client.update_replication_rule(rule_id_1, options={'lifetime': 10000})
get2 = self.rule_client.get_replication_rule(rule_id_1)
assert(get['expires_at'] != get2['expires_at'])
class TestAbacusAccount():
def setUp(self):
self.account = InternalAccount('root')
self.scope = InternalScope('mock')
self.upload_client = UploadClient()
self.account_client = AccountClient()
self.file_sizes = 2
self.rse = 'MOCK4'
self.rse_id = get_rse_id(self.rse)
self.session = get_session()
def tearDown(self):
undertaker.run(once=True)
cleaner.run(once=True)
reaper.run(once=True, rses=[self.rse], greedy=True)
def test_abacus_account(self):
""" ABACUS (ACCOUNT): Test update of account usage """
self.session.query(models.UpdatedAccountCounter).delete() # pylint: disable=no-member
self.session.query(models.AccountUsage).delete() # pylint: disable=no-member
self.session.commit() # pylint: disable=no-member
# Upload files -> account usage should increase
self.files = [{
'did_scope': self.scope.external,
'did_name': 'file_' + generate_uuid(),
'path': file_generator(size=self.file_sizes),
'rse': self.rse,
'lifetime': -1
} for i in range(0, 2)]
self.upload_client.upload(self.files)
[os.remove(file['path']) for file in self.files]
account.run(once=True)
account_usage = get_local_account_usage(account=self.account,
rse_id=self.rse_id)[0]
assert_equal(account_usage['bytes'], len(self.files) * self.file_sizes)
assert_equal(account_usage['files'], len(self.files))
# Update and check the account history with the core method
update_account_counter_history(account=self.account,
rse_id=self.rse_id)
usage_history = get_usage_history(rse_id=self.rse_id,
account=self.account)
assert_equal(usage_history[-1]['bytes'],
len(self.files) * self.file_sizes)
assert_equal(usage_history[-1]['files'], len(self.files))
# Check the account history with the client
usage_history = self.account_client.get_account_usage_history(
rse=self.rse, account=self.account.external)
assert_equal(usage_history[-1]['bytes'],
len(self.files) * self.file_sizes)
assert_equal(usage_history[-1]['files'], len(self.files))
# Delete rules -> account usage should decrease
cleaner.run(once=True)
account.run(once=True)
# set account limit because return value of get_local_account_usage differs if a limit is set or not
set_local_account_limit(account=self.account,
rse_id=self.rse_id,
bytes=10)
account_usages = get_local_account_usage(account=self.account,
rse_id=self.rse_id)[0]
assert_equal(account_usages['bytes'], 0)
assert_equal(account_usages['files'], 0)
