def test_did_hierarchy(self):
""" DATA IDENTIFIERS (CLIENT): Check did hierarchy rule """
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(4)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1, '0cc737eb')
for i in range(4):
self.did_client.add_did(scope, dst[i], 'DATASET', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_did(scope, cnt[i], 'CONTAINER', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': cnt[2]}, {'scope': scope, 'name': cnt[3]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
pass
def test_list_dids(self):
""" DATA IDENTIFIERS (CLIENT): List dids by pattern."""
tmp_scope = scope_name_generator()
tmp_files = []
tmp_files.append('file_a_1%s' % generate_uuid())
tmp_files.append('file_a_2%s' % generate_uuid())
tmp_files.append('file_b_1%s' % generate_uuid())
tmp_rse = 'MOCK'
self.scope_client.add_scope('jdoe', tmp_scope)
for tmp_file in tmp_files:
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L, '0cc737eb')
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\_a\_*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\_a\_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 1)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file\__\_1*'}, type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='file'):
results.append(result)
assert_equal(len(results), 3)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'}):
results.append(result)
assert_equal(len(results), 0)
with assert_raises(UnsupportedOperation):
self.did_client.list_dids(tmp_scope, {'name': 'file*'}, type='whateverytype')
def test_list_scope(rest_client, auth_token):
""" SCOPE (REST): send a GET list all scopes for one account """
tmp_val = account_name_generator()
headers_dict = {'Rucio-Type': 'user', 'X-Rucio-Account': 'root'}
data = {'type': 'USER', 'email': '*****@*****.**'}
response = rest_client.post('/accounts/%s' % tmp_val,
headers=headers(auth(auth_token),
hdrdict(headers_dict)),
json=data)
assert response.status_code == 201
scopes = [scope_name_generator() for _ in range(5)]
for scope in scopes:
response = rest_client.post('/accounts/%s/scopes/%s' %
(tmp_val, scope),
headers=headers(auth(auth_token)),
json={})
assert response.status_code == 201
response = rest_client.get('/accounts/%s/scopes/' % tmp_val,
headers=headers(auth(auth_token)))
assert response.status_code == 200
svr_list = loads(response.get_data(as_text=True))
for scope in scopes:
assert scope in svr_list
def test_scope_success(self):
""" SCOPE (REST): send a POST to create a new account and scope """
mw = []
headers1 = {
'X-Rucio-Account': 'root',
'X-Rucio-Username': '******',
'X-Rucio-Password': '******'
}
headers1.update(self.vo_header)
res1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass',
headers=headers1,
expect_errors=True)
assert res1.status == 200
token = str(res1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
acntusr = account_name_generator()
data = dumps({'type': 'USER', 'email': 'rucio.email.com'})
res2 = TestApp(account_app.wsgifunc(*mw)).post('/' + acntusr,
headers=headers2,
params=data,
expect_errors=True)
assert res2.status == 201
headers3 = {'X-Rucio-Auth-Token': str(token)}
scopeusr = scope_name_generator()
res3 = TestApp(account_app.wsgifunc(*mw)).post('/%s/scopes/%s' %
(acntusr, scopeusr),
headers=headers3,
expect_errors=True)
assert res3.status == 201
def test_update_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers and update the flag new """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe')
dids = []
for i in xrange(0, 5):
d = {
'scope': tmp_scope,
'name': '%s-%i' % (tmp_dsn, i),
'did_type': DIDType.DATASET
}
did.add_did(scope=tmp_scope,
name='%s-%i' % (tmp_dsn, i),
type='DATASET',
issuer='root')
dids.append(d)
st = did.set_new_dids(dids, None)
assert_true(st)
with assert_raises(DataIdentifierNotFound):
did.set_new_dids([{
'scope': 'dummyscope',
'name': 'dummyname',
'did_type': DIDType.DATASET
}], None)
def test_create_scope_duplicate(self):
""" SCOPE (CLIENTS): try to create a duplicate scope."""
account = 'jdoe'
scope = scope_name_generator()
self.scope_client.add_scope(account, scope)
with pytest.raises(Duplicate):
self.scope_client.add_scope(account, scope)
def test_did_hierarchy(self):
""" DATA IDENTIFIERS (CLIENT): Check did hierarchy rule """
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(4)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1, '0cc737eb')
for i in range(4):
self.did_client.add_did(scope, dst[i], 'DATASET', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_did(scope, cnt[i], 'CONTAINER', statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1L, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1L, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': cnt[2]}, {'scope': scope, 'name': cnt[3]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
pass
def test_scope_failure(rest_client, auth_token):
""" SCOPE (REST): send a POST to create a new scope for a not existing account to test the error"""
scopeusr = scope_name_generator()
account_name_generator()
response = rest_client.post('/accounts/%s/scopes/%s' %
(scopeusr, scopeusr),
headers=headers(auth(auth_token)))
assert response.status_code == 404
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
self.vo = {}
self.scopes = [InternalScope(scope_name_generator(), **self.vo) for _ in range(5)]
self.jdoe = InternalAccount('jdoe', **self.vo)
def __init__(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo_header = {'X-Rucio-VO': 'tst'}
else:
self.vo_header = {}
self.scopes = [scope_name_generator() for _ in range(5)]
def test_detach_did(self):
""" DATA IDENTIFIERS (CLIENT): Detach dids from a did"""
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(5)]
cnt = ['cnt_%s' % generate_uuid() for i in range(2)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1, '0cc737eb')
for i in range(5):
self.did_client.add_dataset(scope, dst[i], statuses=None, meta=None, rules=None)
for i in range(2):
self.did_client.add_container(scope, cnt[i], statuses=None, meta=None, rules=None)
for i in range(5):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': dst[2]}, {'scope': scope, 'name': dst[3]}])
with assert_raises(UnsupportedOperation):
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': cnt[1]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
self.did_client.detach_dids(scope, cnt[0], [{'scope': scope, 'name': dst[1]}])
self.did_client.detach_dids(scope, dst[3], [{'scope': scope, 'name': file[6]}, {'scope': scope, 'name': file[7]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
if r['name'] == dst[1]:
assert_equal(r['level'], 0)
if r['type'] == 'file':
if (r['name'] in file[6:9]):
assert_equal(r['level'], 0)
else:
assert_not_equal(r['level'], 0)
with assert_raises(UnsupportedOperation):
self.did_client.detach_dids(scope=scope, name=cnt[0], dids=[{'scope': scope, 'name': cnt[0]}])
self.did_client.close(scope, dst[4])
metadata = self.did_client.get_metadata(scope, dst[4])
i_bytes, i_length = metadata['bytes'], metadata['length']
metadata = self.did_client.get_metadata(scope, file[8])
file1_bytes = metadata['bytes']
metadata = self.did_client.get_metadata(scope, file[9])
file2_bytes = metadata['bytes']
self.did_client.detach_dids(scope, dst[4], [{'scope': scope, 'name': file[8]}, {'scope': scope, 'name': file[9]}])
metadata = self.did_client.get_metadata(scope, dst[4])
f_bytes, f_length = metadata['bytes'], metadata['length']
assert_equal(i_bytes, f_bytes + file1_bytes + file2_bytes)
assert_equal(i_length, f_length + 1 + 1)
def test_create_scope(self):
""" SCOPE (CLIENTS): create a new scope."""
account = 'jdoe'
scope = scope_name_generator()
ret = self.scope_client.add_scope(account, scope)
assert_true(ret)
with assert_raises(InvalidObject):
self.scope_client.add_scope(account, 'tooooolooooongscooooooooooooope')
with assert_raises(InvalidObject):
self.scope_client.add_scope(account, '$?!')
def test_scope_success(rest_client, auth_token):
""" SCOPE (REST): send a POST to create a new account and scope """
acntusr = account_name_generator()
data = {'type': 'USER', 'email': 'rucio.email.com'}
response = rest_client.post('/accounts/' + acntusr, headers=headers(auth(auth_token)), json=data)
assert response.status_code == 201
scopeusr = scope_name_generator()
response = rest_client.post('/accounts/%s/scopes/%s' % (acntusr, scopeusr), headers=headers(auth(auth_token)))
assert response.status_code == 201
def test_create_scope(self):
""" SCOPE (CLIENTS): create a new scope."""
account = 'jdoe'
scope = scope_name_generator()
ret = self.scope_client.add_scope(account, scope)
assert_true(ret)
with assert_raises(InvalidObject):
self.scope_client.add_scope(account, 'tooooolooooongscooooooooooooope')
with assert_raises(InvalidObject):
self.scope_client.add_scope(account, '$?!')
def test_add_scope(self):
"""CLIENT(ADMIN): Add scope"""
tmp_scp = scope_name_generator()
tmp_acc = account_name_generator()
cmd = 'rucio-admin account add %s' % tmp_acc
exitcode, out, err = execute(cmd)
cmd = 'rucio-admin scope add --account %s --scope %s' % (tmp_acc, tmp_scp)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal('Added new scope to account: %s-%s\n' % (tmp_scp, tmp_acc), out)
def test_add_scope(self):
"""CLIENT(ADMIN): Add scope"""
tmp_scp = scope_name_generator()
tmp_acc = account_name_generator()
cmd = 'rucio-admin account add %s' % tmp_acc
exitcode, out, err = execute(cmd)
cmd = 'rucio-admin scope add --account %s --scope %s' % (tmp_acc, tmp_scp)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_equal('Added new scope to account: %s-%s\n' % (tmp_scp, tmp_acc), out)
def test_permission_add_did(self):
""" PERMISSION(CORE): Check permission to add a did"""
scope = scope_name_generator()
add_scope(scope=InternalScope(scope), account=InternalAccount('root'))
assert_true(
has_permission(issuer='panda',
action='add_did',
kwargs={'scope': scope}))
assert_false(
has_permission(issuer='spock',
action='add_did',
kwargs={'scope': scope}))
def test_scope_duplicate(rest_client, auth_token):
""" SCOPE (REST): send a POST to create a already existing scope to test the error"""
acntusr = account_name_generator()
data = {'type': 'USER', 'email': '*****@*****.**'}
response = rest_client.post('/accounts/' + acntusr, headers=headers(auth(auth_token)), json=data)
assert response.status_code == 201
scopeusr = scope_name_generator()
response = rest_client.post('/accounts/%s/scopes/%s' % (acntusr, scopeusr), headers=headers(auth(auth_token)))
assert response.status_code == 201
response = rest_client.post('/accounts/%s/scopes/%s' % (acntusr, scopeusr), headers=headers(auth(auth_token)))
assert response.status_code == 409
def test_list_scopes(self):
""" SCOPE (CLIENTS): try to list scopes for an account."""
account = 'jdoe'
scope_list = [scope_name_generator() for _ in range(5)]
for scope in scope_list:
self.scope_client.add_scope(account, scope)
svr_list = self.scope_client.list_scopes_for_account(account)
for scope in scope_list:
if scope not in svr_list:
assert False
def test_list_scopes(self):
""" SCOPE (CLIENTS): try to list scopes for an account."""
account = 'jdoe'
scope_list = [scope_name_generator() for i in xrange(5)]
for scope in scope_list:
self.scope_client.add_scope(account, scope)
svr_list = self.scope_client.list_scopes_for_account(account)
for scope in scope_list:
if scope not in svr_list:
assert_true(False)
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in range(3)]
self.tmp_scopes = [scope_name_generator() for i in range(3)]
self.tmp_rses = [rse_name_generator() for i in range(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in range(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in range(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in range(3)]
# add dummy data to the catalogue
for i in range(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1, '0cc737eb')
# put files in datasets
for i in range(3):
for j in range(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
# put datasets in containers
for i in range(3):
for j in range(3):
datasets = [{'scope': self.tmp_scopes[j], 'name': self.tmp_datasets[j]}]
self.did_client.add_container(self.tmp_scopes[i], self.tmp_containers[j])
self.did_client.add_datasets_to_container(self.tmp_scopes[i], self.tmp_containers[j], datasets)
# reverse check if everything is in order
for i in range(3):
result = self.did_client.scope_list(self.tmp_scopes[i], recursive=True)
r_topdids = []
r_otherscopedids = []
r_scope = []
for r in result:
if r['level'] == 0:
r_topdids.append(r['scope'] + ':' + r['name'])
r_scope.append(r['scope'])
if r['scope'] != self.tmp_scopes[i]:
r_otherscopedids.append(r['scope'] + ':' + r['name'])
assert_in(r['level'], [1, 2])
for j in range(3):
assert_equal(self.tmp_scopes[i], r_scope[j])
if j != i:
assert_in(self.tmp_scopes[j] + ':' + self.tmp_files[j], r_otherscopedids)
assert_not_in(self.tmp_scopes[i] + ':' + self.tmp_files[i], r_topdids)
def test_list_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe')
for i in xrange(0, 5):
did.add_did(scope=tmp_scope, name='%s-%i' % (tmp_dsn, i), type='DATASET', issuer='root')
for i in did.list_new_dids('DATASET'):
assert_not_equal(i, {})
assert_equal(str(i['did_type']), 'DATASET')
break
for i in did.list_new_dids():
assert_not_equal(i, {})
break
def test_scope_failure(self):
""" SCOPE (REST): send a POST to create a new scope for a not existing account to test the error"""
mw = []
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
res1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(res1.status, 200)
token = str(res1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
scopeusr = scope_name_generator()
account_name_generator()
res2 = TestApp(account_app.wsgifunc(*mw)).post('/%s/scopes/%s' % (scopeusr, scopeusr), headers=headers2, expect_errors=True)
assert_equal(res2.status, 404)
def test_list_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe', **self.vo)
for i in range(0, 5):
did.add_did(scope=tmp_scope, name='%s-%i' % (tmp_dsn, i), type='DATASET', issuer='root', **self.vo)
for i in did.list_new_dids('DATASET', **self.vo):
assert i != {}
assert str(i['did_type']) == 'DATASET'
break
for i in did.list_new_dids(**self.vo):
assert i != {}
break
def test_update_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers and update the flag new """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe')
dids = []
for i in xrange(0, 5):
d = {'scope': tmp_scope, 'name': '%s-%i' % (tmp_dsn, i), 'did_type': DIDType.DATASET}
did.add_did(scope=tmp_scope, name='%s-%i' % (tmp_dsn, i), type='DATASET', issuer='root')
dids.append(d)
st = did.set_new_dids(dids, None)
assert_true(st)
with assert_raises(DataIdentifierNotFound):
did.set_new_dids([{'scope': 'dummyscope', 'name': 'dummyname', 'did_type': DIDType.DATASET}], None)
def test_scope_failure(self):
""" SCOPE (REST): send a POST to create a new scope for a not existing account to test the error"""
mw = []
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(r1.status, 200)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
scopeusr = scope_name_generator() # NOQA
acntusr = account_name_generator() # NOQA
r2 = TestApp(account_app.wsgifunc(*mw)).post('/%(scopeusr)s/scopes/%(scopeusr)s' % locals(), headers=headers2, expect_errors=True)
assert_equal(r2.status, 404)
def test_list_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe')
for i in range(0, 5):
did.add_did(scope=tmp_scope, name='%s-%i' % (tmp_dsn, i), type='DATASET', issuer='root')
for i in did.list_new_dids('DATASET'):
assert_not_equal(i, {})
assert_equal(str(i['did_type']), 'DATASET')
break
for i in did.list_new_dids():
assert_not_equal(i, {})
break
def test_list_dids(self):
""" DATA IDENTIFIERS (CLIENT): List dids by pattern."""
tmp_scope = scope_name_generator()
tmp_files = []
tmp_files.append('file_a_1%s' % generate_uuid())
tmp_files.append('file_a_2%s' % generate_uuid())
tmp_files.append('file_b_1%s' % generate_uuid())
tmp_rse = 'MOCK'
self.scope_client.add_scope('jdoe', tmp_scope)
for tmp_file in tmp_files:
self.replica_client.add_replica(tmp_rse, tmp_scope, tmp_file, 1L,
'0cc737eb')
results = []
for result in self.did_client.list_dids(tmp_scope,
{'name': 'file\_a\_*'},
type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope,
{'name': 'file\_a\_1*'},
type='file'):
results.append(result)
assert_equal(len(results), 1)
results = []
for result in self.did_client.list_dids(tmp_scope,
{'name': 'file\__\_1*'},
type='file'):
results.append(result)
assert_equal(len(results), 2)
results = []
for result in self.did_client.list_dids(tmp_scope, {'name': 'file*'},
type='file'):
results.append(result)
assert_equal(len(results), 3)
results = []
filters = {
'name': 'file*',
'created_after': datetime.utcnow() - timedelta(hours=1)
}
for result in self.did_client.list_dids(tmp_scope, filters):
results.append(result)
assert_equal(len(results), 0)
with assert_raises(UnsupportedOperation):
self.did_client.list_dids(tmp_scope, {'name': 'file*'},
type='whateverytype')
def test_detach_did(self):
""" DATA IDENTIFIERS (CLIENT): Detach dids from a did"""
account = 'jdoe'
rse = 'MOCK'
scope = scope_name_generator()
file = ['file_%s' % generate_uuid() for i in range(10)]
dst = ['dst_%s' % generate_uuid() for i in range(4)]
cnt = ['cnt_%s' % generate_uuid() for i in range(2)]
self.scope_client.add_scope(account, scope)
for i in range(10):
self.replica_client.add_replica(rse, scope, file[i], 1L, '0cc737eb')
for i in range(4):
self.did_client.add_dataset(scope, dst[i], statuses=None, meta=None, rules=None)
for i in range(2):
self.did_client.add_container(scope, cnt[i], statuses=None, meta=None, rules=None)
for i in range(4):
self.did_client.add_files_to_dataset(scope, dst[i], [{'scope': scope, 'name': file[2 * i], 'bytes': 1L, 'adler32': '0cc737eb'},
{'scope': scope, 'name': file[2 * i + 1], 'bytes': 1L, 'adler32': '0cc737eb'}])
self.did_client.add_containers_to_container(scope, cnt[1], [{'scope': scope, 'name': dst[2]}, {'scope': scope, 'name': dst[3]}])
with assert_raises(UnsupportedOperation):
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': cnt[1]}])
self.did_client.add_datasets_to_container(scope, cnt[0], [{'scope': scope, 'name': dst[1]}, {'scope': scope, 'name': dst[2]}])
self.did_client.detach_dids(scope, cnt[0], [{'scope': scope, 'name': dst[1]}])
self.did_client.detach_dids(scope, dst[3], [{'scope': scope, 'name': file[6]}, {'scope': scope, 'name': file[7]}])
result = self.did_client.scope_list(scope, recursive=True)
for r in result:
if r['name'] == dst[1]:
assert_equal(r['level'], 0)
if r['type'] is 'file':
if (r['name'] in file[6:9]):
assert_equal(r['level'], 0)
else:
assert_not_equal(r['level'], 0)
with assert_raises(UnsupportedOperation):
self.did_client.detach_dids(scope=scope, name=cnt[0], dids=[{'scope': scope, 'name': cnt[0]}])
def test_scope_success(self):
""" SCOPE (REST): send a POST to create a new account and scope """
mw = []
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(r1.status, 200)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
acntusr = account_name_generator()
data = dumps({'type': 'USER'})
r2 = TestApp(account_app.wsgifunc(*mw)).post('/' + acntusr, headers=headers2, params=data, expect_errors=True)
assert_equal(r2.status, 201)
headers3 = {'X-Rucio-Auth-Token': str(token)}
scopeusr = scope_name_generator()
r3 = TestApp(account_app.wsgifunc(*mw)).post('/%s/scopes/%s' % (acntusr, scopeusr), headers=headers3, expect_errors=True)
assert_equal(r3.status, 201)
def test_scope_duplicate(self):
""" SCOPE (REST): send a POST to create a already existing scope to test the error"""
mw = []
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
res1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(res1.status, 200)
token = str(res1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
acntusr = account_name_generator()
data = dumps({'type': 'USER', 'email': '*****@*****.**'})
res2 = TestApp(account_app.wsgifunc(*mw)).post('/' + acntusr, headers=headers2, params=data, expect_errors=True)
assert_equal(res2.status, 201)
headers3 = {'X-Rucio-Auth-Token': str(token)}
scopeusr = scope_name_generator()
res3 = TestApp(account_app.wsgifunc(*mw)).post('/%s/scopes/%s' % (acntusr, scopeusr), headers=headers3, expect_errors=True)
assert_equal(res3.status, 201)
res3 = TestApp(account_app.wsgifunc(*mw)).post('/%s/scopes/%s' % (acntusr, scopeusr), headers=headers3, expect_errors=True)
assert_equal(res3.status, 409)
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in xrange(3)]
self.tmp_scopes = [scope_name_generator() for i in xrange(3)]
self.tmp_rses = [rse_name_generator() for i in xrange(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in xrange(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in xrange(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in xrange(3)]
# add dummy data to the catalogue
for i in xrange(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1L, '0cc737eb')
# put files in datasets
for i in xrange(3):
for j in xrange(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1L, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in xrange(3)]
self.tmp_scopes = [scope_name_generator() for i in xrange(3)]
self.tmp_rses = [rse_name_generator() for i in xrange(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in xrange(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in xrange(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in xrange(3)]
# add dummy data to the catalogue
for i in xrange(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1L, '0cc737eb')
# put files in datasets
for i in xrange(3):
for j in xrange(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1L, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
def test_overlapping_did_names(rse_factory, did_factory, download_client,
root_account, mock_scope, vo):
"""
Downloading two different did with different scope but same name to the same directory must fail
"""
rse, _ = rse_factory.make_posix_rse()
scope1 = mock_scope
scope2 = InternalScope(scope_name_generator(), vo=vo)
scope_core.add_scope(scope2, root_account)
did1 = did_factory.upload_test_file(rse, scope=scope1)
did2 = did_factory.upload_test_file(rse, scope=scope2, name=did1['name'])
dataset = did_factory.make_dataset()
did_core.attach_dids(dids=[did1, did2], account=root_account, **dataset)
did1_str = '%s:%s' % (did1['scope'], did1['name'])
did2_str = '%s:%s' % (did2['scope'], did2['name'])
dataset1_did_str = '%s:%s' % (dataset['scope'], dataset['name'])
with TemporaryDirectory() as tmp_dir:
with pytest.raises(RucioException):
download_client.download_dids([{
'did': dataset1_did_str,
'base_dir': tmp_dir,
'no_subdir': True
}])
with TemporaryDirectory() as tmp_dir:
with pytest.raises(RucioException):
download_client.download_dids([{
'did': did1_str,
'base_dir': tmp_dir,
'no_subdir': True
}, {
'did': did2_str,
'base_dir': tmp_dir,
'no_subdir': True
}])
def test_is_scope_owner(self):
""" SCOPE (CORE): Is scope owner """
scope = scope_name_generator()
add_scope(scope=scope, account='jdoe')
anwser = is_scope_owner(scope=scope, account='jdoe')
assert_equal(anwser, True)
def test_create_scope_duplicate(self):
""" SCOPE (CLIENTS): try to create a duplicate scope."""
account = 'jdoe'
scope = scope_name_generator()
self.scope_client.add_scope(account, scope)
self.scope_client.add_scope(account, scope)
def test_create_scope_no_account(self):
""" SCOPE (CLIENTS): try to create scope for not existing account."""
account = str(uuid()).lower()[:30]
scope = scope_name_generator()
self.scope_client.add_scope(account, scope)
def test_permission_add_did(self):
""" PERMISSION(CORE): Check permission to add a did"""
scope = scope_name_generator()
add_scope(scope=scope, account='root')
assert_true(has_permission(issuer='panda', action='add_did', kwargs={'scope': scope}))
assert_false(has_permission(issuer='spock', action='add_did', kwargs={'scope': scope}))
def test_is_scope_owner(self):
""" SCOPE (CORE): Is scope owner """
scope = scope_name_generator()
add_scope(scope=scope, account='jdoe')
anwser = is_scope_owner(scope=scope, account='jdoe')
assert_equal(anwser, True)
def __init__(self):
self.scopes = [scope_name_generator() for _ in range(5)]
def test_is_scope_owner(self):
""" SCOPE (CORE): Is scope owner """
scope = InternalScope(scope_name_generator(), **self.vo)
add_scope(scope=scope, account=self.jdoe)
anwser = is_scope_owner(scope=scope, account=self.jdoe)
assert anwser is True
def test_create_scope_no_account(self):
""" SCOPE (CLIENTS): try to create scope for not existing account."""
account = str(uuid()).lower()[:30]
scope = scope_name_generator()
with pytest.raises(AccountNotFound):
self.scope_client.add_scope(account, scope)
def setup(self):
self.scopes = [scope_name_generator() for i in range(5)]
def __init__(self):
self.scopes = [InternalScope(scope_name_generator()) for _ in range(5)]
self.jdoe = InternalAccount('jdoe')
