def test_to_repair_a_rule_with_only_1_rse_whose_site_is_blacklisted(self):
""" JUDGE REPAIRER: Test to repair a rule with only 1 rse whose site is blacklisted"""
rse = rse_name_generator()
rse_id = add_rse(rse)
update_rse(rse_id, {'availability_write': False})
set_account_limit('jdoe', rse_id, -1)
rule_repairer(once=True) # Clean out the repairer
scope = 'mock'
files = create_files(4, scope, self.rse4_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=rse, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=True, activity='DebugJudge')[0]
assert(RuleState.STUCK == get_rule(rule_id)['state'])
rule_repairer(once=True)
# Stil assert STUCK because of ignore_availability:
assert(RuleState.STUCK == get_rule(rule_id)['state'])
region = make_region().configure('dogpile.cache.memcached',
expiration_time=3600,
arguments={'url': "127.0.0.1:11211", 'distributed_lock': True})
region.delete(sha256(rse).hexdigest())
update_rse(rse_id, {'availability_write': True})
rule_repairer(once=True)
assert(RuleState.REPLICATING == get_rule(rule_id)['state'])
def test_list_expired_dids_with_locked_rules(self):
""" UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules"""
tmp_scope = 'mock'
# Add quota
set_account_limit('jdoe', get_rse_id('MOCK'), -1)
dsn = {
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': 'jdoe',
'copies': 1,
'rse_expression': 'MOCK',
'locked': True,
'grouping': 'DATASET'
}]
}
add_dids(dids=[dsn], account='root')
for did in list_expired_dids(limit=1000):
assert (did['scope'] != dsn['scope']
and did['name'] != dsn['name'])
def generate_rse(endpoint, token):
rse_name = 'RSE%s' % generate_uuid().upper()
scheme = 'https'
impl = 'rucio.rse.protocols.webdav.Default'
if not endpoint.startswith('https://'):
scheme = 'srm'
impl = 'rucio.rse.protocols.srm.Default'
tmp_proto = {
'impl': impl,
'scheme': scheme,
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 1, 'write': 1, 'delete': 1}}}
rse_id = rse.add_rse(rse_name)
tmp_proto['hostname'] = endpoint.split(':')[1][2:]
tmp_proto['port'] = endpoint.split(':')[2].split('/')[0]
tmp_proto['prefix'] = '/'.join([''] + endpoint.split(':')[2].split('/')[1:])
if scheme == 'srm':
tmp_proto['extended_attributes'] = {'space_token': token,
'web_service_path': '/srm/managerv2?SFN='}
rse.add_protocol(rse_id=rse_id, parameter=tmp_proto)
rse.add_rse_attribute(rse_id=rse_id, key='fts', value='https://fts3-pilot.cern.ch:8446')
account_limit.set_account_limit(account='root', rse_id=rsemanager.get_rse_info(rse_name)['id'], bytes=-1)
return rsemanager.get_rse_info(rse_name)
def set_account_limit(account, rse, bytes, issuer):
"""
Set an account limit..
:param account: The account name.
:param rse: The rse name.
:param bytes: The limit in bytes.
:param issuer: The issuer account_core.
"""
rse_id = get_rse_id(rse=rse)
kwargs = {'account': account, 'rse': rse, 'rse_id': rse_id, 'bytes': bytes}
if not rucio.api.permission.has_permission(
issuer=issuer, action='set_account_limit', kwargs=kwargs):
raise rucio.common.exception.AccessDenied(
'Account %s can not set account limits.' % (issuer))
account = InternalAccount(account)
if not account_exists(account=account):
raise rucio.common.exception.AccountNotFound(
'Account %s does not exist' % (account))
account_limit_core.set_account_limit(account=account,
rse_id=rse_id,
bytes=bytes)
def test_add_did(self):
""" DATA IDENTIFIERS (CLIENT): Add, populate, list did content and create a sample"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
tmp_dsn = 'dsn_%s' % generate_uuid()
set_account_limit('root', get_rse_id('MOCK'), -1)
set_account_limit('root', get_rse_id('CERN-PROD_TZERO'), -1)
# PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename
dataset_meta = {
'project': 'data13_hip',
'run_number': 300000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m927',
}
rules = [{'copies': 1, 'rse_expression': 'MOCK', 'account': 'root'}]
with assert_raises(ScopeNotFound):
self.did_client.add_dataset(scope='Nimportnawak',
name=tmp_dsn,
statuses={'monotonic': True},
meta=dataset_meta,
rules=rules)
files = [
{
'scope': tmp_scope,
'name': 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()),
'bytes': 724963570L,
'adler32': '0cc737eb'
},
]
def test_list_rules_states(self):
""" SUBSCRIPTION (API): Test listing of rule states for subscription """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
add_rse(site_a)
add_rse(site_b)
# Add quota
set_account_limit('root', get_rse_id(site_a), -1)
set_account_limit('root', get_rse_id(site_b), -1)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account='root')
subscription_name = uuid()
subid = add_subscription(name=subscription_name, account='root', filter={'account': ['root', ], 'scope': [tmp_scope, ]},
replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root')
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
for rule in list_subscription_rule_states(account='root', name=subscription_name):
assert_equal(rule[3], 2)
def test_listing_account_limit(self):
""" ACCOUNT (CLIENTS): Test listing account limit """
account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
account_limit.set_account_limit(account=self.account, rse_id=self.rse1_id, bytes=333)
limit = self.client.get_account_limit(account=self.account, rse=self.rse1)
assert_equal(limit, {self.rse1: 333})
account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
def test_listing_account_limit(self):
""" ACCOUNT (CLIENTS): Test listing account limit """
account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
account_limit.set_account_limit(account=self.account, rse_id=self.rse1_id, bytes=333)
limit = self.client.get_account_limit(account=self.account, rse=self.rse1)
assert_equal(limit, {self.rse1: 333})
account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
def test_delete_rule(self):
"""CLIENT(USER): rule deletion"""
set_account_limit('root', get_rse_id(self.def_rse), -1)
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
set_account_limit('root', get_rse_id(tmp_rse), -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rules
cmd = "rucio add-rule {0}:{1} 1 'spacetoken=ATLASSCRATCHDISK'".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(err)
print(out)
# get the rules for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
(rule1, rule2) = out.split()
# delete the rules for the file
cmd = "rucio delete-rule {0}".format(rule1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = "rucio delete-rule {0}".format(rule2)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the file
cmd = "rucio list-dids {0}:{1}".format(self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(5, len(out.splitlines()))
def test_set_account_limit(self):
""" ACCOUNT_LIMIT (CORE): Setting account limit """
account_limit.set_account_limit(account=self.account,
rse_id=self.rse1_id,
bytes=100000)
assert_equal(
account_limit.get_account_limit(account=self.account,
rse_id=self.rse1_id), 100000)
assert_equal(
account_limit.get_account_limit(account=self.account,
rse_id=self.rse2_id), None)
def test_listing_account_limits(self):
""" ACCOUNT (CLIENTS): Test listing account limits """
account_limit.set_account_limit(account=self.account, rse_id=self.rse1_id, bytes=12345)
account_limit.set_account_limit(account=self.account, rse_id=self.rse2_id, bytes=12345)
limits = self.client.get_account_limits(account=self.account)
assert_in((self.rse1, 12345), limits.items())
assert_in((self.rse2, 12345), limits.items())
account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
account_limit.delete_account_limit(account=self.account, rse_id=self.rse2_id)
def test_listing_account_limits(self):
""" ACCOUNT (CLIENTS): Test listing account limits """
account_limit.set_account_limit(account=self.account, rse_id=self.rse1_id, bytes=12345)
account_limit.set_account_limit(account=self.account, rse_id=self.rse2_id, bytes=12345)
limits = self.client.get_account_limits(account=self.account)
assert_in((self.rse1, 12345), limits.items())
assert_in((self.rse2, 12345), limits.items())
account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
account_limit.delete_account_limit(account=self.account, rse_id=self.rse2_id)
def test_rule_add_fails_account_limit(self):
""" REPLICATION RULE (CORE): Test if a rule fails correctly when account limit conflict"""
scope = 'mock'
files = create_files(3, scope, self.rse1, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
set_account_limit(account='jdoe', rse_id=self.rse1_id, bytes=5)
assert_raises(InsufficientAccountLimit, add_rule, dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
delete_account_limit(account='jdoe', rse_id=self.rse1_id)
def setup(self):
try:
remove('/tmp/.rucio_root/auth_token_root')
except OSError as e:
if e.args[0] != 2:
raise e
self.marker = '$> '
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.user = '******'
self.def_rse = 'MOCK4'
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
set_account_limit('root', get_rse_id(self.def_rse), -1)
def test_abacus_account(self):
""" ABACUS (ACCOUNT): Test update of account usage """
self.session.query(models.UpdatedAccountCounter).delete() # pylint: disable=no-member
self.session.query(models.AccountUsage).delete() # pylint: disable=no-member
self.session.commit() # pylint: disable=no-member
# Upload files -> account usage should increase
self.files = [{
'did_scope': self.scope.external,
'did_name': 'file_' + generate_uuid(),
'path': file_generator(size=self.file_sizes),
'rse': self.rse,
'lifetime': -1
} for i in range(0, 2)]
self.upload_client.upload(self.files)
[os.remove(file['path']) for file in self.files]
account.run(once=True)
account_usage = get_account_usage(account=self.account,
rse_id=self.rse_id)[0]
assert_equal(account_usage['bytes'], len(self.files) * self.file_sizes)
assert_equal(account_usage['files'], len(self.files))
# Update and check the account history with the core method
update_account_counter_history(account=self.account,
rse_id=self.rse_id)
usage_history = get_usage_history(rse_id=self.rse_id,
account=self.account)
assert_equal(usage_history[-1]['bytes'],
len(self.files) * self.file_sizes)
assert_equal(usage_history[-1]['files'], len(self.files))
# Check the account history with the client
usage_history = self.account_client.get_account_usage_history(
rse=self.rse, account=self.account.external)
assert_equal(usage_history[-1]['bytes'],
len(self.files) * self.file_sizes)
assert_equal(usage_history[-1]['files'], len(self.files))
# Delete rules -> account usage should decrease
cleaner.run(once=True)
account.run(once=True)
# set account limit because return value of get_account_usage differs if a limit is set or not
set_account_limit(account=self.account, rse_id=self.rse_id, bytes=10)
account_usages = get_account_usage(account=self.account,
rse_id=self.rse_id)[0]
assert_equal(account_usages['bytes'], 0)
assert_equal(account_usages['files'], 0)
def test_undertaker(self):
""" UNDERTAKER (CORE): Test the undertaker. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
set_account_limit('jdoe', get_rse_id('MOCK'), -1)
dsns1 = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1
} for i in xrange(nbdatasets)]
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': 'jdoe',
'copies': 1,
'rse_expression': 'MOCK',
'grouping': 'DATASET'
}]
} for i in xrange(nbdatasets)]
add_dids(dids=dsns1 + dsns2, account='root')
replicas = list()
for dsn in dsns1 + dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1L,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for i in xrange(nbfiles)]
attach_dids(scope=tmp_scope,
def test_list_rules_states(self):
""" SUBSCRIPTION (REST): Test listing of rule states for subscription """
tmp_scope = InternalScope('mock_' + uuid()[:8])
root = InternalAccount('root')
add_scope(tmp_scope, root)
mw = []
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
site_a_id = add_rse(site_a)
site_b_id = add_rse(site_b)
# Add quota
set_account_limit(root, site_a_id, -1)
set_account_limit(root, site_b_id, -1)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account=root)
subscription_name = uuid()
subid = add_subscription(name=subscription_name, account='root', filter={'account': ['root', ], 'scope': [tmp_scope.external, ]},
replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery', issuer='root')
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
res1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(res1.status, 200)
token = str(res1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
res2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True)
for line in res2.body.split('\n'):
print(line)
rs = loads(line)
if rs[1] == subscription_name:
break
assert_equal(rs[3], 2)
def set_account_limit(account, rse, bytes, issuer):
"""
Set an account limit..
:param account: The account name.
:param rse: The rse name.
:param bytes: The limit in bytes.
:param issuer: The issuer account_core.
"""
kwargs = {'account': account, 'rse': rse, 'bytes': bytes}
if not rucio.api.permission.has_permission(issuer=issuer, action='set_account_limit', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not set account limits.' % (issuer))
if not account_exists(account=account):
raise rucio.common.exception.AccountNotFound('Account %s does not exist' % (account))
rse_id = get_rse_id(rse=rse)
account_limit_core.set_account_limit(account=account, rse_id=rse_id, bytes=bytes)
def test_atlas_archival_policy(self):
""" UNDERTAKER (CORE): Test the atlas archival policy. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
add_rse(rse)
set_account_limit('jdoe', get_rse_id(rse), -1)
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': 'jdoe',
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for i in xrange(nbdatasets)]
add_dids(dids=dsns2, account='root')
replicas = list()
for dsn in dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1L,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for i in xrange(nbfiles)]
def test_undertaker(self):
""" UNDERTAKER (CORE): Test the undertaker. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
rse = 'MOCK'
rse_id = get_rse_id('MOCK')
set_account_limit('jdoe', rse_id, -1)
dsns1 = [{'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1} for i in range(nbdatasets)]
dsns2 = [{'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1,
'rules': [{'account': 'jdoe', 'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'}]} for i in range(nbdatasets)]
add_dids(dids=dsns1 + dsns2, account='root')
replicas = list()
for dsn in dsns1 + dsns2:
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(),
'bytes': 1, 'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)]
attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account='root')
replicas += files
add_rules(dids=dsns1, rules=[{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}])
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert_not_equal(get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'], None)
class TestBinRucio():
def setup(self):
try:
remove('/tmp/.rucio_root/auth_token_root')
except OSError, e:
if e.args[0] != 2:
raise e
self.marker = '$> '
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.user = '******'
self.def_rse = 'MOCK4'
set_account_limit('root', get_rse_id(self.def_rse), -1)
def test_atlas_archival_policy(self):
""" UNDERTAKER (CORE): Test the atlas archival policy. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
add_rse(rse)
set_account_limit('jdoe', get_rse_id(rse), -1)
dsns2 = [{'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1,
'rules': [{'account': 'jdoe', 'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'}]} for i in range(nbdatasets)]
add_dids(dids=dsns2, account='root')
replicas = list()
for dsn in dsns2:
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1,
'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)]
attach_dids(scope=tmp_scope, name=dsn['name'], rse=rse, dids=files, account='root')
replicas += files
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert(get_replica(scope=replica['scope'], name=replica['name'], rse=rse)['tombstone'] is None)
for dsn in dsns2:
assert(get_did(scope='archive', name=dsn['name'])['name'] == dsn['name'])
assert(len([x for x in list_rules(filters={'scope': 'archive', 'name': dsn['name']})]) == 1)
def setUpClass(cls):
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse3 = 'MOCK3'
cls.rse4 = 'MOCK4'
cls.rse5 = 'MOCK5'
cls.rse1_id = get_rse_id(rse=cls.rse1)
cls.rse3_id = get_rse_id(rse=cls.rse3)
cls.rse4_id = get_rse_id(rse=cls.rse4)
cls.rse5_id = get_rse_id(rse=cls.rse5)
# Add Tags
cls.T1 = tag_generator()
cls.T2 = tag_generator()
add_rse_attribute(cls.rse1_id, cls.T1, True)
add_rse_attribute(cls.rse3_id, cls.T1, True)
add_rse_attribute(cls.rse4_id, cls.T2, True)
add_rse_attribute(cls.rse5_id, cls.T1, True)
# Add fake weights
add_rse_attribute(cls.rse1_id, "fakeweight", 10)
add_rse_attribute(cls.rse3_id, "fakeweight", 0)
add_rse_attribute(cls.rse4_id, "fakeweight", 0)
add_rse_attribute(cls.rse5_id, "fakeweight", 0)
# Add quota
cls.jdoe = InternalAccount('jdoe')
cls.root = InternalAccount('root')
set_account_limit(cls.jdoe, cls.rse1_id, -1)
set_account_limit(cls.jdoe, cls.rse3_id, -1)
set_account_limit(cls.jdoe, cls.rse4_id, -1)
set_account_limit(cls.jdoe, cls.rse5_id, -1)
set_account_limit(cls.root, cls.rse1_id, -1)
set_account_limit(cls.root, cls.rse3_id, -1)
set_account_limit(cls.root, cls.rse4_id, -1)
set_account_limit(cls.root, cls.rse5_id, -1)
def create_scope(scope):
add_scope(scope, 'root', 'root')
set_account_limit('root', get_rse_id('PSDM_DISK'), 100000000000)
def setUpClass(cls):
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse3 = 'MOCK3'
cls.rse4 = 'MOCK4'
cls.rse5 = 'MOCK5'
cls.rse1_id = get_rse(cls.rse1).id
cls.rse3_id = get_rse(cls.rse3).id
cls.rse4_id = get_rse(cls.rse4).id
cls.rse5_id = get_rse(cls.rse5).id
# Add Tags
cls.T1 = tag_generator()
cls.T2 = tag_generator()
add_rse_attribute(cls.rse1, cls.T1, True)
add_rse_attribute(cls.rse3, cls.T1, True)
add_rse_attribute(cls.rse4, cls.T2, True)
add_rse_attribute(cls.rse5, cls.T1, True)
# Add fake weights
add_rse_attribute(cls.rse1, "fakeweight", 10)
add_rse_attribute(cls.rse3, "fakeweight", 0)
add_rse_attribute(cls.rse4, "fakeweight", 0)
add_rse_attribute(cls.rse5, "fakeweight", 0)
# Add quota
set_account_limit('jdoe', cls.rse1_id, -1)
set_account_limit('jdoe', cls.rse3_id, -1)
set_account_limit('jdoe', cls.rse4_id, -1)
set_account_limit('jdoe', cls.rse5_id, -1)
set_account_limit('root', cls.rse1_id, -1)
set_account_limit('root', cls.rse3_id, -1)
set_account_limit('root', cls.rse4_id, -1)
set_account_limit('root', cls.rse5_id, -1)
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': 'tst'}
else:
cls.vo = {}
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse3 = 'MOCK3'
cls.rse4 = 'MOCK4'
cls.rse5 = 'MOCK5'
cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo)
cls.rse3_id = get_rse_id(rse=cls.rse3, **cls.vo)
cls.rse4_id = get_rse_id(rse=cls.rse4, **cls.vo)
cls.rse5_id = get_rse_id(rse=cls.rse5, **cls.vo)
# Add Tags
cls.T1 = tag_generator()
cls.T2 = tag_generator()
add_rse_attribute(cls.rse1_id, cls.T1, True)
add_rse_attribute(cls.rse3_id, cls.T1, True)
add_rse_attribute(cls.rse4_id, cls.T2, True)
add_rse_attribute(cls.rse5_id, cls.T1, True)
# Add fake weights
add_rse_attribute(cls.rse1_id, "fakeweight", 10)
add_rse_attribute(cls.rse3_id, "fakeweight", 0)
add_rse_attribute(cls.rse4_id, "fakeweight", 0)
add_rse_attribute(cls.rse5_id, "fakeweight", 0)
# Add quota
cls.jdoe = InternalAccount('jdoe', **cls.vo)
cls.root = InternalAccount('root', **cls.vo)
set_account_limit(cls.jdoe, cls.rse1_id, -1)
set_account_limit(cls.jdoe, cls.rse3_id, -1)
set_account_limit(cls.jdoe, cls.rse4_id, -1)
set_account_limit(cls.jdoe, cls.rse5_id, -1)
set_account_limit(cls.root, cls.rse1_id, -1)
set_account_limit(cls.root, cls.rse3_id, -1)
set_account_limit(cls.root, cls.rse4_id, -1)
set_account_limit(cls.root, cls.rse5_id, -1)
add_protocol('SITE1_DISK', params)
add_rse_attribute(rse='SITE1_DISK', key='istape', value='False')
params = {
'scheme': 'file',
'prefix': '/tmp/SITE2_DISK/',
'impl': 'rucio.rse.protocols.posix.Default',
'domains': {
"lan": {
"read": 1,
"write": 1,
"delete": 1
},
"wan": {
"read": 1,
"write": 1,
"delete": 1
}
}
}
add_rse('SITE2_DISK', 'root')
add_protocol('SITE2_DISK', params)
add_rse_attribute(rse='SITE2_DISK', key='istape', value='False')
# Now set a quota for root and jdoe on the 2 RSEs
set_account_limit('root', get_rse_id('SITE1_DISK'), 100000000000)
set_account_limit('root', get_rse_id('SITE2_DISK'), 100000000000)
set_account_limit('jdoe', get_rse_id('SITE1_DISK'), 1000000000)
set_account_limit('jdoe', get_rse_id('SITE2_DISK'), 0)
def test_add_did(self):
""" DATA IDENTIFIERS (CLIENT): Add, populate, list did content and create a sample"""
tmp_scope = 'mock'
tmp_rse = 'MOCK'
tmp_dsn = 'dsn_%s' % generate_uuid()
set_account_limit('root', get_rse_id('MOCK'), -1)
set_account_limit('root', get_rse_id('CERN-PROD_TZERO'), -1)
# PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename
dataset_meta = {'project': 'data13_hip',
'run_number': 300000,
'stream_name': 'physics_CosmicCalo',
'prod_step': 'merge',
'datatype': 'NTUP_TRIG',
'version': 'f392_m927',
}
rules = [{'copies': 1, 'rse_expression': 'MOCK', 'account': 'root'}]
with assert_raises(ScopeNotFound):
self.did_client.add_dataset(scope='Nimportnawak', name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules)
files = [{'scope': tmp_scope, 'name': 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()), 'bytes': 724963570, 'adler32': '0cc737eb'}, ]
with assert_raises(DataIdentifierNotFound):
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files)
with assert_raises(DataIdentifierNotFound):
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files)
files = []
for i in range(5):
lfn = 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 10}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2), 'account': 'root'}]
with assert_raises(InvalidPath):
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files, rse=tmp_rse)
files_without_pfn = [{'scope': i['scope'], 'name': i['name'], 'bytes': i['bytes'], 'adler32': i['adler32'], 'meta': i['meta']} for i in files]
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files_without_pfn, rse=tmp_rse)
with assert_raises(DataIdentifierAlreadyExists):
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
files = []
for i in range(5):
lfn = '%(tmp_dsn)s.' % locals() + str(generate_uuid())
pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
# it doesn't work with mock: TBF
# pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta
pfn += '%(tmp_dsn)s/%(lfn)s' % locals()
file_meta = {'guid': str(generate_uuid()), 'events': 100}
files.append({'scope': tmp_scope, 'name': lfn,
'bytes': 724963570, 'adler32': '0cc737eb',
'pfn': pfn, 'meta': file_meta})
rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}]
with assert_raises(InvalidPath):
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse)
files_without_pfn = [{'scope': i['scope'], 'name': i['name'], 'bytes': i['bytes'], 'adler32': i['adler32'], 'meta': i['meta']} for i in files]
self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files_without_pfn, rse=tmp_rse)
self.did_client.close(scope=tmp_scope, name=tmp_dsn)
tmp_dsn_output = 'dsn_%s' % generate_uuid()
self.did_client.create_did_sample(input_scope=tmp_scope, input_name=tmp_dsn, output_scope=tmp_scope, output_name=tmp_dsn_output, nbfiles=2)
files = [f for f in self.did_client.list_files(scope=tmp_scope, name=tmp_dsn_output)]
assert_equal(len(files), 2)
def test_create_rule(self):
"""CLIENT(USER): Rucio add rule"""
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
# add quota
set_account_limit('root', get_rse_id(tmp_rse), -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add quota
set_account_limit('root', get_rse_id(tmp_rse), -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add quota
set_account_limit('root', get_rse_id(tmp_rse), -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rules
cmd = "rucio add-rule {0}:{1} 3 'spacetoken=ATLASSCRATCHDISK'".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
rule = out[:-1] # triming new line character
# check if rule exist for the file
cmd = "rucio list-rules {0}:{1}".format(self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(rule, out), None)
def test_set_account_limit(self):
""" ACCOUNT_LIMIT (CORE): Setting account limit """
account_limit.set_account_limit(account=self.account, rse_id=self.rse1_id, bytes=100000)
assert_equal(account_limit.get_account_limit(account=self.account, rse_id=self.rse1_id), 100000)
assert_equal(account_limit.get_account_limit(account=self.account, rse_id=self.rse2_id), float("Inf"))
