def test_create_rule(self):
"""CLIENT(USER): Rucio add rule"""
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rules
cmd = "rucio add-rule {0}:{1} 3 'spacetoken=ATLASSCRATCHDISK'".format(self.user, tmp_file1[5:])
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
rule = out[:-1] # triming new line character
# check if rule exist for the file
cmd = "rucio list-rules --did {0}:{1}".format(self.user, tmp_file1[5:])
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_not_equal(re.search(rule, out), None)
def test_add_files_to_dataset(self):
"""CLIENT(USER): Rucio add files to dataset"""
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_dataset = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# add files
cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_file2)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# create dataset
cmd = 'rucio add-dataset ' + tmp_dataset
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add files to dataset
cmd = 'rucio add-files-to-dataset --to {0} mock:{1} mock:{2}'.format(tmp_dataset, tmp_file1[5:], tmp_file2[5:]) # triming '/tmp/' from filename
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# find the added files
cmd = 'rucio list-files ' + tmp_dataset
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
def test_download_dataset(self):
"""CLIENT(USER): Rucio download dataset"""
tmp_file1 = file_generator()
tmp_dataset = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# add files
cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# create dataset
cmd = 'rucio add-dataset ' + tmp_dataset
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add files to dataset
cmd = 'rucio add-files-to-dataset --to {0} {1}:{2}'.format(tmp_dataset, self.user, tmp_file1[5:]) # triming '/tmp/' from filename
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# download dataset
cmd = 'rucio download --dir /tmp {0}'.format(tmp_dataset) # triming '/tmp/' from filename
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
search = 'Getting file {0}:{1}'.format(self.user, tmp_file1[5:])
nose.tools.assert_not_equal(re.search(search, err), None)
search = 'File validated'
nose.tools.assert_not_equal(re.search(search, out), None)
search = 'DID ' + tmp_dataset
nose.tools.assert_not_equal(re.search(search, out), None)
def test_detach_files_dataset(self):
"""CLIENT(USER): Rucio detach files to dataset"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3} {4} --did {5}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# detach the files to the dataset
cmd = 'rucio detach --from {0} {1}:{2} {1}:{3}'.format(tmp_dsn, self.user, tmp_file2[5:], tmp_file3[5:]) # triming '/tmp/' from filenames
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
# tmp_file1 must be in the dataset
nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_equal(re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
def test_upload_repeated_file_dataset(self):
"""CLIENT(USER): Rucio upload repeated files to dataset"""
# One of the files to upload is already in the dataset
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} --files {2} --did {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
# upload the files to the dataset
cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3} {4} --did {5}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
# tmp_file1 must be in the dataset
nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
def test_add_rse(self):
"""CLIENT(ADMIN): Add RSE"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
nose.tools.assert_equal('Added new RSE: %s\n' % tmp_val, out)
def test_create_dataset(self):
"""CLIENT(USER): Rucio add dataset"""
tmp_name = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
cmd = 'rucio add-dataset ' + tmp_name
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_not_equal(re.search('Added ' + tmp_name, out), None)
def test_upload(self):
"""CLIENT(USER): Upload"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
exitcode, out, err = execute(cmd)
cmd = 'rucio upload'
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
def test_create_dataset(self):
"""CLIENT(USER): Rucio add dataset"""
tmp_name = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
cmd = 'rucio add-dataset ' + tmp_name
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search('Added ' + tmp_name, out), None)
def test_add_rse_nondet(self):
"""CLIENT(ADMIN): Add non-deterministic RSE"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add --non-deterministic %s' % tmp_val
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_equal(
'Added new non-deterministic RSE: %s\n' % tmp_val, out)
def test_upload(self):
"""CLIENT(USER): Upload"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
exitcode, out, err = execute(cmd)
cmd = 'rucio upload'
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
def test_reaper_bulk_delete(vo, caches_mock):
""" REAPER (DAEMON): Mock test the reaper daemon on async bulk delete request."""
[cache_region] = caches_mock
scope = InternalScope('data13_hip', vo=vo)
nb_files = 250
file_size = 200 # 2G
rse_name, rse_id, dids = __add_test_rse_and_replicas(
vo=vo,
scope=scope,
rse_name=rse_name_generator(),
names=['lfn' + generate_uuid() for _ in range(nb_files)],
file_size=file_size)
rse_core.set_rse_limits(rse_id=rse_id,
name='MinFreeSpace',
value=50 * file_size)
assert len(
list(replica_core.list_replicas(dids=dids,
rse_expression=rse_name))) == nb_files
# Check first if the reaper does not delete anything if no space is needed
cache_region.invalidate()
rse_core.set_rse_usage(rse_id=rse_id,
source='storage',
used=nb_files * file_size,
free=323000000000)
reaper(once=True,
rses=[],
include_rses=rse_name,
exclude_rses=None,
chunk_size=1000,
scheme='MOCK')
assert len(
list(replica_core.list_replicas(dids=dids,
rse_expression=rse_name))) == nb_files
# Now put it over threshold and delete
cache_region.invalidate()
rse_core.set_rse_usage(rse_id=rse_id,
source='storage',
used=nb_files * file_size,
free=1)
reaper(once=True,
rses=[],
include_rses=rse_name,
exclude_rses=None,
chunk_size=1000,
scheme='MOCK')
reaper(once=True,
rses=[],
include_rses=rse_name,
exclude_rses=None,
chunk_size=1000,
scheme='MOCK')
assert len(list(replica_core.list_replicas(
dids, rse_expression=rse_name))) == 200
def test_list_rses(self):
"""CLIENT(ADMIN): List RSEs"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
exitcode, out, err = execute(cmd)
cmd = 'rucio-admin rse list'
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_regexp_matches(out, re.compile('.*%s.*' % tmp_val))
def setUp(self):
self.account_client = AccountClient()
self.rse_client = RSEClient()
self.account = generate_uuid()[:10]
self.rse = rse_name_generator()
self.account_client.add_account(self.account, 'SERVICE',
'*****@*****.**')
self.rse_client.add_rse(self.rse)
def test_list_rses(self):
"""CLIENT(ADMIN): List RSEs"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
exitcode, out, err = execute(cmd)
cmd = 'rucio-admin rse list'
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
nose.tools.assert_regexp_matches(out, re.compile('.*%s.*' % tmp_val))
def test_delete_rule(self):
"""CLIENT(USER): rule deletion"""
set_account_limit('root', get_rse_id(self.def_rse), -1)
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
set_account_limit('root', get_rse_id(tmp_rse), -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rules
cmd = "rucio add-rule {0}:{1} 1 'spacetoken=ATLASSCRATCHDISK'".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(err)
print(out)
# get the rules for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
(rule1, rule2) = out.split()
# delete the rules for the file
cmd = "rucio delete-rule {0}".format(rule1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = "rucio delete-rule {0}".format(rule2)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the file
cmd = "rucio list-dids {0}:{1}".format(self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(5, len(out.splitlines()))
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
cls.rse_client = RSEClient()
cls.tmp_rse_name = rse_name_generator()
cls.rse_client.add_rse(cls.tmp_rse_name, vo=cls.vo)
cls.tmp_rse = cls.rse_client.get_rse(cls.tmp_rse_name)['id']
def __init__(
self,
vo: str,
db_session: "Session",
setup_func: "Optional[Callable]" = None,
teardown_func: "Optional[Callable]" = None,
):
self.vo = vo
self.db_session = db_session
self.setup = setup_func
self.teardown = teardown_func
self.name = rse_name_generator()
self.rse_id: "Optional[str]" = None
def test_post_rse(self):
"""RSE (CURL): add RSE"""
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" --cert %s --key %s -X GET %s/auth/x509 | tr -d \'\r\' | grep X-Rucio-Auth-Token:' % (
self.cacert, self.usercert, self.userkey, self.auth_host)
exitcode, out, err = execute(cmd)
nose.tools.assert_in('X-Rucio-Auth-Token', out)
os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):].rstrip()
cmd = '''curl -s -i --cacert %s -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -X POST %s/rses/%s''' % (
self.cacert, self.host, rse_name_generator())
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
nose.tools.assert_in('201 Created', out)
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in range(3)]
self.tmp_scopes = [scope_name_generator() for i in range(3)]
self.tmp_rses = [rse_name_generator() for i in range(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in range(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in range(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in range(3)]
# add dummy data to the catalogue
for i in range(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1, '0cc737eb')
# put files in datasets
for i in range(3):
for j in range(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
# put datasets in containers
for i in range(3):
for j in range(3):
datasets = [{'scope': self.tmp_scopes[j], 'name': self.tmp_datasets[j]}]
self.did_client.add_container(self.tmp_scopes[i], self.tmp_containers[j])
self.did_client.add_datasets_to_container(self.tmp_scopes[i], self.tmp_containers[j], datasets)
# reverse check if everything is in order
for i in range(3):
result = self.did_client.scope_list(self.tmp_scopes[i], recursive=True)
r_topdids = []
r_otherscopedids = []
r_scope = []
for r in result:
if r['level'] == 0:
r_topdids.append(r['scope'] + ':' + r['name'])
r_scope.append(r['scope'])
if r['scope'] != self.tmp_scopes[i]:
r_otherscopedids.append(r['scope'] + ':' + r['name'])
assert_in(r['level'], [1, 2])
for j in range(3):
assert_equal(self.tmp_scopes[i], r_scope[j])
if j != i:
assert_in(self.tmp_scopes[j] + ':' + self.tmp_files[j], r_otherscopedids)
assert_not_in(self.tmp_scopes[i] + ':' + self.tmp_files[i], r_topdids)
def test_to_repair_a_rule_with_only_1_rse_whose_site_is_blacklisted(self):
""" JUDGE REPAIRER: Test to repair a rule with only 1 rse whose site is blacklisted"""
rse = rse_name_generator()
rse_id = add_rse(rse, **self.vo)
update_rse(rse_id, {'availability_write': False})
set_local_account_limit(self.jdoe, rse_id, -1)
rule_repairer(once=True) # Clean out the repairer
scope = InternalScope('mock', **self.vo)
files = create_files(4, scope, self.rse4_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
rule_id = add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account=self.jdoe,
copies=1,
rse_expression=rse,
grouping='DATASET',
weight=None,
lifetime=None,
locked=False,
subscription_id=None,
ignore_availability=True,
activity='DebugJudge')[0]
assert (RuleState.STUCK == get_rule(rule_id)['state'])
rule_repairer(once=True)
# Stil assert STUCK because of ignore_availability:
assert (RuleState.STUCK == get_rule(rule_id)['state'])
region = make_region().configure('dogpile.cache.memcached',
expiration_time=3600,
arguments={
'url':
config_get(
'cache', 'url', False,
'127.0.0.1:11211'),
'distributed_lock':
True
})
region.delete(sha256(rse.encode()).hexdigest())
update_rse(rse_id, {'availability_write': True})
rule_repairer(once=True)
assert (RuleState.REPLICATING == get_rule(rule_id)['state'])
def setup(self):
self.rc = ReplicaClient()
self.client_location_without_proxy = {
'ip': '192.168.0.1',
'fqdn': 'anomalous-materials.blackmesa.com',
'site': 'BLACKMESA'
}
self.rse_without_proxy = rse_name_generator()
add_rse(self.rse_without_proxy)
add_rse_attribute(rse=self.rse_without_proxy,
key='site',
value='BLACKMESA')
self.client_location_with_proxy = {
'ip': '10.0.1.1',
'fqdn': 'test-chamber.aperture.com',
'site': 'APERTURE'
}
self.rse_with_proxy = rse_name_generator()
add_rse(self.rse_with_proxy)
add_rse_attribute(rse=self.rse_with_proxy,
key='root-proxy-internal',
value='root://proxy.aperture.com:1094')
add_rse_attribute(rse=self.rse_with_proxy,
key='site',
value='APERTURE')
self.files = [{
'scope': 'mock',
'name': 'half-life_%s' % i,
'bytes': 1234L,
'adler32': 'deadbeef',
'meta': {
'events': 666
}
} for i in xrange(1, 4)]
def test_list_archive_contents_transparently(self):
""" ARCHIVE (CORE): Transparent archive listing """
scope = 'mock'
rse = 'APERTURE_%s' % rse_name_generator()
add_rse(rse)
add_protocol(rse, {'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 1, 'write': 1, 'delete': 1}}})
# register archive
archive = {'scope': scope, 'name': 'weighted.storage.cube.zip', 'type': 'FILE',
'bytes': 2596, 'adler32': 'beefdead'}
add_replicas(rse=rse, files=[archive], account='root')
# archived files with replicas
files_with_replicas = [{'scope': scope, 'name': 'witrep-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE',
'bytes': 1234, 'adler32': 'deadbeef'} for i in xrange(2)]
add_replicas(rse=rse, files=files_with_replicas, account='root')
self.dc.add_files_to_archive(scope=scope, name=archive['name'], files=files_with_replicas)
res = [r['pfns'] for r in self.rc.list_replicas(dids=[{'scope': scope, 'name': f['name']} for f in files_with_replicas],
resolve_archives=True)]
assert_equal(len(res), 2)
assert_equal(len(res[0]), 2)
assert_equal(len(res[1]), 2)
for r in res:
for p in r:
if r[p]['domain'] == 'zip':
assert_in('weighted.storage.cube.zip?xrdcl.unzip=witrep-', p)
else:
assert_not_in('weighted.storage.cube.zip?xrdcl.unzip=witrep-', p)
# archived files without replicas
files = [{'scope': scope, 'name': 'norep-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE',
'bytes': 1234, 'adler32': 'deadbeef'} for i in xrange(2)]
self.dc.add_files_to_archive(scope=scope, name=archive['name'], files=files)
res = [r['pfns'] for r in self.rc.list_replicas(dids=[{'scope': scope, 'name': f['name']} for f in files],
resolve_archives=True)]
assert_equal(len(res), 2)
for r in res:
assert_in('weighted.storage.cube.zip?xrdcl.unzip=norep-', r.keys()[0])
del_rse(rse)
def test_reaper():
""" REAPER2 (DAEMON): Test the reaper2 daemon."""
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
vo = {}
rse_name = rse_name_generator()
rse_id = rse_core.add_rse(rse_name, **vo)
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/reaper',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
nb_files = 30
file_size = 2147483648 # 2G
file_names = []
for i in range(nb_files):
file_name = 'lfn' + generate_uuid()
file_names.append(file_name)
replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo),
name=file_name, bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **vo), adler32=None, md5=None)
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800)
rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240)
rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
if vo:
reaper(once=True, rses=[], include_rses='vo=%s&(%s)' % (vo['vo'], rse_name), exclude_rses=[])
reaper(once=True, rses=[], include_rses='vo=%s&(%s)' % (vo['vo'], rse_name), exclude_rses=[])
else:
reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[])
reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[])
assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in file_names], rse_expression=rse_name))) == nb_files - 5
def test_add_rule_with_ignore_availability(self):
""" REPLICATION RULE (CORE): Add a replication rule with ignore_availability setting"""
rse = rse_name_generator()
add_rse(rse)
update_rse(rse, {'availability_write': False})
scope = 'mock'
files = create_files(3, scope, self.rse1)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
with assert_raises(InvalidRSEExpression):
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=rse, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=rse, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=True)[0]
def test_delete_rule(self):
"""CLIENT(USER): rule deletion"""
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rules
cmd = "rucio add-rule {0}:{1} 1 'spacetoken=ATLASSCRATCHDISK'".format(self.user, tmp_file1[5:])
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
# get the rules for the file
cmd = "rucio list-rules --did {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(self.user, tmp_file1[5:])
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
(rule1, rule2) = out.split()
# delete the rules for the file
cmd = "rucio delete-rule {0}".format(rule1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
cmd = "rucio delete-rule {0}".format(rule2)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# search for the file
cmd = "rucio list-dids {0}:{1}".format(self.user, tmp_file1[5:])
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_equal('', out)
def test_attach_files_dataset(self):
"""CLIENT(USER): Rucio attach files to dataset"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# upload the files
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file2, tmp_file3)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# attach the files to the dataset
cmd = 'rucio attach {0} {1}:{2} {1}:{3}'.format(
tmp_dsn, self.user, tmp_file2[5:],
tmp_file3[5:]) # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# tmp_file2 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file2[5:]), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
def setup(self):
self.rc = client.ReplicaClient()
self.rse = rse_name_generator()
add_rse(self.rse)
add_protocol(
self.rse, {
'scheme': 'https',
'hostname': 'storage.googleapis.com',
'port': 443,
'prefix': '/atlas-europe-west1/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1,
'third_party_copy': 1
}
}
})
# register some files there
self.files = [{
'scope': 'mock',
'name': 'file-on-gcs_%s' % i,
'bytes': 1234,
'adler32': 'deadbeef',
'meta': {
'events': 666
}
} for i in range(0, 3)]
add_replicas(rse=self.rse,
files=self.files,
account='root',
ignore_availability=True)
def tearDown(self):
delete_replicas(rse=self.rse, files=self.files)
del_rse(self.rse)
def test_detach_non_existing_file(self):
"""CLIENT(USER): Rucio detach a non existing file"""
tmp_file1 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
# attach the files to the dataset
cmd = 'rucio detach {0} {1}:{2}'.format(tmp_dsn, self.user, 'file_ghost') # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(re.search("Data identifier not found.", err), None)
def test_detach_non_existing_file(self):
"""CLIENT(USER): Rucio detach a non existing file"""
tmp_file1 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} --files {2} --did {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
remove(tmp_file1)
# attach the files to the dataset
cmd = 'rucio detach --from {0} {1}:{2}'.format(tmp_dsn, self.user, 'file_ghost') # triming '/tmp/' from filenames
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
nose.tools.assert_not_equal(re.search("Failed to detach data identifier", err), None)
def test_attach_file_twice(self):
"""CLIENT(USER): Rucio attach a file twice"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
# attach the files to the dataset
cmd = 'rucio attach {0} {1}:{2}'.format(tmp_dsn, self.user, tmp_file1[5:]) # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(re.search("The file already exists", err), None)
def setupClass(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
self.vo = {}
self.rse_client = RSEClient()
self.tmp_rse_name = rse_name_generator()
self.rse_client.add_rse(self.tmp_rse_name, vo=self.vo)
self.tmp_rse = self.rse_client.get_rse(self.tmp_rse_name)['id']
def test_atlas_archival_policy(self):
""" UNDERTAKER (CORE): Test the atlas archival policy. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
add_rse(rse)
set_account_limit('jdoe', get_rse_id(rse), -1)
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': 'jdoe',
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for i in xrange(nbdatasets)]
add_dids(dids=dsns2, account='root')
replicas = list()
for dsn in dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1L,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for i in xrange(nbfiles)]
def _make_rse(self, scheme, protocol_impl):
rse_name = rse_name_generator()
rse_id = rse_core.add_rse(rse_name, vo=self.vo)
rse_core.add_protocol(rse_id=rse_id, parameter={
'scheme': scheme,
'hostname': 'host%d' % len(self.created_rses),
'port': 0,
'prefix': '/test',
'impl': protocol_impl,
'domains': {
'wan': {
'read': 1,
'write': 1,
'delete': 1,
'third_party_copy': 1
}
}
})
self.created_rses.append(rse_id)
return rse_name, rse_id
def test_export_import(self):
""" IMPORT/EXPORT (REST): Test the export and import of data together to check same syntax."""
# Setup new RSE, distance, attribute, limits
new_rse = rse_name_generator()
add_rse(new_rse)
# Get token
mw = []
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)}
# Export data
r2 = TestApp(export_app.wsgifunc(*mw)).get('/', headers=headers2, expect_errors=True)
exported_data = parse_response(r2.body)
# Import data
r3 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**exported_data))
assert_equal(r3.status, 201)
def test_list_replica_with_domain(self):
""" REPLICA (CORE): Add and list file replicas forcing domain"""
tmp_rse = rse_name_generator()
add_rse(tmp_rse)
protocols = [{'scheme': 'MOCK',
'hostname': 'localhost',
'port': 17,
'prefix': '/i/prefer/the/lan',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 2,
'write': 2,
'delete': 2}}},
{'scheme': 'MOCK',
'hostname': 'localhost',
'port': 18,
'prefix': '/i/prefer/the/wan',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 2,
'write': 2,
'delete': 2},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}, ]
for p in protocols:
add_protocol(tmp_rse, p)
nbfiles = 3
files = [{'scope': 'mock',
'name': 'file_%s' % generate_uuid(),
'bytes': 1234L,
'adler32': '01234567',
'meta': {'events': 1234}} for i in xrange(nbfiles)]
def test_upload_repeated_file_dataset(self):
"""CLIENT(USER): Rucio upload repeated files to dataset"""
# One of the files to upload is already in the dataset
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_file1_name = path.basename(tmp_file1)
tmp_file3_name = path.basename(tmp_file3)
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# upload the files to the dataset
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# tmp_file1 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file1_name), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file3_name), out), None)
def _make_rse(self,
scheme,
protocol_impl,
parameters=None,
add_rse_kwargs=None):
rse_name = rse_name_generator()
if add_rse_kwargs and 'vo' in add_rse_kwargs:
rse_id = rse_core.add_rse(rse_name, **add_rse_kwargs)
else:
rse_id = rse_core.add_rse(rse_name,
vo=self.vo,
**(add_rse_kwargs or {}))
if scheme and protocol_impl:
protocol_parameters = {
'scheme': scheme,
'hostname': '%s.cern.ch' % rse_id,
'port': 0,
'prefix': '/test_%s/' % rse_id,
'impl': protocol_impl,
'domains': {
'wan': {
'read': 1,
'write': 1,
'delete': 1,
'third_party_copy_read': 1,
'third_party_copy_write': 1,
},
'lan': {
'read': 1,
'write': 1,
'delete': 1,
}
}
}
protocol_parameters.update(parameters or {})
rse_core.add_protocol(rse_id=rse_id, parameter=protocol_parameters)
self.created_rses.append(rse_id)
return rse_name, rse_id
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in xrange(3)]
self.tmp_scopes = [scope_name_generator() for i in xrange(3)]
self.tmp_rses = [rse_name_generator() for i in xrange(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in xrange(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in xrange(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in xrange(3)]
# add dummy data to the catalogue
for i in xrange(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1L, '0cc737eb')
# put files in datasets
for i in xrange(3):
for j in xrange(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1L, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
def test_scope_list(self):
""" DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """
# create some dummy data
self.tmp_accounts = ['jdoe' for i in xrange(3)]
self.tmp_scopes = [scope_name_generator() for i in xrange(3)]
self.tmp_rses = [rse_name_generator() for i in xrange(3)]
self.tmp_files = ['file_%s' % generate_uuid() for i in xrange(3)]
self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in xrange(3)]
self.tmp_containers = ['container_%s' % generate_uuid() for i in xrange(3)]
# add dummy data to the catalogue
for i in xrange(3):
self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
self.rse_client.add_rse(self.tmp_rses[i])
self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1L, '0cc737eb')
# put files in datasets
for i in xrange(3):
for j in xrange(3):
files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1L, 'adler32': '0cc737eb'}]
self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
def test_atlas_archival_policy(self):
""" UNDERTAKER (CORE): Test the atlas archival policy. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
add_rse(rse)
set_account_limit('jdoe', get_rse_id(rse), -1)
dsns2 = [{'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1,
'rules': [{'account': 'jdoe', 'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'}]} for i in range(nbdatasets)]
add_dids(dids=dsns2, account='root')
replicas = list()
for dsn in dsns2:
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1,
'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)]
attach_dids(scope=tmp_scope, name=dsn['name'], rse=rse, dids=files, account='root')
replicas += files
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert(get_replica(scope=replica['scope'], name=replica['name'], rse=rse)['tombstone'] is None)
for dsn in dsns2:
assert(get_did(scope='archive', name=dsn['name'])['name'] == dsn['name'])
assert(len([x for x in list_rules(filters={'scope': 'archive', 'name': dsn['name']})]) == 1)
def setup(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo_header = {'X-Rucio-VO': 'tst'}
self.vo = {'vo': 'tst'}
else:
self.vo_header = {}
self.vo = {}
# New RSE
self.new_rse = rse_name_generator()
# RSE 1 that already exists
self.old_rse_1 = rse_name_generator()
self.old_rse_id_1 = add_rse(self.old_rse_1,
availability=1,
region_code='DE',
country_name='DE',
deterministic=True,
volatile=True,
staging_area=True,
time_zone='Europe',
latitude='1',
longitude='2',
**self.vo)
add_protocol(
self.old_rse_id_1, {
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'impl': 'TODO'
})
add_protocol(
self.old_rse_id_1, {
'scheme': 'scheme3',
'hostname': 'hostname3',
'port': 1000,
'impl': 'TODO'
})
set_rse_limits(rse_id=self.old_rse_id_1,
name='MaxBeingDeletedFiles',
value='10')
set_rse_limits(rse_id=self.old_rse_id_1,
name='MinFreeSpace',
value='10')
add_rse_attribute(rse_id=self.old_rse_id_1,
key='attr1',
value='test10')
add_rse_attribute(rse_id=self.old_rse_id_1,
key='lfn2pfn_algorithm',
value='test10')
add_rse_attribute(rse_id=self.old_rse_id_1,
key='verify_checksum',
value=True)
# RSE 2 that already exists
self.old_rse_2 = rse_name_generator()
self.old_rse_id_2 = add_rse(self.old_rse_2, **self.vo)
# RSE 3 that already exists
self.old_rse_3 = rse_name_generator()
self.old_rse_id_3 = add_rse(self.old_rse_3, **self.vo)
# RSE 4 that already exists
self.old_rse_4 = rse_name_generator()
self.old_rse_id_4 = add_rse(self.old_rse_4, **self.vo)
# Distance that already exists
add_distance(self.old_rse_id_1, self.old_rse_id_2)
self.data1 = {
'rses': {
self.new_rse: {
'rse_type':
RSEType.TAPE,
'availability':
3,
'city':
'NewCity',
'region_code':
'CH',
'country_name':
'switzerland',
'staging_area':
False,
'time_zone':
'Europe',
'latitude':
1,
'longitude':
2,
'deterministic':
True,
'volatile':
False,
'protocols': [{
'scheme': 'scheme',
'hostname': 'hostname',
'port': 1000,
'impl': 'impl'
}],
'attributes': {
'attr1': 'test'
},
'MinFreeSpace':
20000,
'lfn2pfn_algorithm':
'hash2',
'verify_checksum':
False,
'availability_delete':
True,
'availability_read':
False,
'availability_write':
True
},
self.old_rse_1: {
'rse_type':
RSEType.TAPE,
'deterministic':
False,
'volatile':
False,
'region_code':
'US',
'country_name':
'US',
'staging_area':
False,
'time_zone':
'Asia',
'longitude':
5,
'city':
'City',
'availability':
2,
'latitude':
10,
'protocols': [{
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'prefix': 'prefix',
'impl': 'impl1'
}, {
'scheme': 'scheme2',
'hostname': 'hostname2',
'port': 1001,
'impl': 'impl'
}],
'attributes': {
'attr1': 'test1',
'attr2': 'test2'
},
'MinFreeSpace':
10000,
'MaxBeingDeletedFiles':
1000,
'verify_checksum':
False,
'lfn2pfn_algorithm':
'hash3',
'availability_delete':
False,
'availability_read':
False,
'availability_write':
True
},
self.old_rse_2: {},
self.old_rse_3: {}
},
'distances': {
self.old_rse_1: {
self.old_rse_2: {
'src_rse': self.old_rse_1,
'dest_rse': self.old_rse_2,
'ranking': 10
},
self.old_rse_3: {
'src_rse': self.old_rse_1,
'dest_rse': self.old_rse_3,
'ranking': 4
}
}
}
}
self.data2 = {'rses': {self.new_rse: {'rse': self.new_rse}}}
self.data3 = {'distances': {}}
def protocols_setup(vo):
rse_info = copy.deepcopy(base_rse_info)
files = [{
'scope': InternalScope('mock', vo=vo),
'name': 'element_0',
'bytes': 1234,
'adler32': 'deadbeef'
}]
root = InternalAccount('root', vo=vo)
for idx in range(len(rse_info)):
rse_info[idx]['name'] = '%s_%s' % (rse_info[idx]['site'],
rse_name_generator())
rse_info[idx]['id'] = add_rse(rse_info[idx]['name'], vo=vo)
add_rse_attribute(rse_id=rse_info[idx]['id'],
key='site',
value=base_rse_info[idx]['site'])
add_replicas(rse_id=rse_info[idx]['id'], files=files, account=root)
# invalidate cache for parse_expression('site=…')
rse_expression_parser.REGION.invalidate()
# check sites
for idx in range(len(rse_info)):
site_rses = rse_expression_parser.parse_expression(
'site=' + base_rse_info[idx]['site'])
assert len(site_rses) > 0
assert rse_info[idx]['id'] in [rse['id'] for rse in site_rses]
add_protocol(
rse_info[0]['id'], {
'scheme': schemes[0],
'hostname': ('root.%s' % base_rse_info[0]['address']),
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
add_protocol(
rse_info[0]['id'], {
'scheme': schemes[2],
'hostname': ('davs.%s' % base_rse_info[0]['address']),
'port': 443,
'prefix': '/test/chamber/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {
'read': 2,
'write': 2,
'delete': 2
},
'wan': {
'read': 2,
'write': 2,
'delete': 2
}
}
})
add_protocol(
rse_info[0]['id'], {
'scheme': schemes[1],
'hostname': ('gsiftp.%s' % base_rse_info[0]['address']),
'port': 8446,
'prefix': '/test/chamber/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {
'read': 0,
'write': 0,
'delete': 0
},
'wan': {
'read': 3,
'write': 3,
'delete': 3
}
}
})
add_protocol(
rse_info[1]['id'], {
'scheme': schemes[1],
'hostname': ('gsiftp.%s' % base_rse_info[1]['address']),
'port': 8446,
'prefix': '/lambda/complex/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {
'read': 2,
'write': 2,
'delete': 2
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
add_protocol(
rse_info[1]['id'], {
'scheme': schemes[2],
'hostname': ('davs.%s' % base_rse_info[1]['address']),
'port': 443,
'prefix': '/lambda/complex/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {
'read': 0,
'write': 0,
'delete': 0
},
'wan': {
'read': 2,
'write': 2,
'delete': 2
}
}
})
add_protocol(
rse_info[1]['id'], {
'scheme': schemes[0],
'hostname': ('root.%s' % base_rse_info[1]['address']),
'port': 1409,
'prefix': '//lambda/complex/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 3,
'write': 3,
'delete': 3
}
}
})
yield {'files': files, 'rse_info': rse_info}
for info in rse_info:
delete_replicas(rse_id=info['id'], files=files)
del_rse_attribute(rse_id=info['id'], key='site')
del_rse(info['id'])
def test_post_rse(self):
"""RSE (CURL): add RSE"""
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token:' % (self.cacert, self.usercert, self.auth_host)
exitcode, out, err = execute(cmd)
nose.tools.assert_in('X-Rucio-Auth-Token', out)
os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
cmd = '''curl -s -i --cacert %s -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -X POST %s/rses/%s''' % (self.cacert, self.host, rse_name_generator())
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
nose.tools.assert_in('201 Created', out)