class TestReplicaMetalinkRedirection:
def __init__(self):
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.marker = '$> '
# get auth token
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def test_replica_meta_redirection(self):
""" REDIRECT: metalink to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" -X GET %s/redirect/%s/%s' '' % (
self.cacert, self.token, self.host, tmp_scope, tmp_name)
_, out, _ = execute(cmd)
assert_in('404 Not Found', out)
# add replicas
self.replica_client.add_replicas(rse='MOCK',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1L,
'adler32': '0cc737eb'
}])
class TestDIDClients(object):
def __init__(self):
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
def test_add_and_list_archive(self):
""" ARCHIVE (CLIENT): Add files to archive and list the content."""
scope, rse = 'mock', 'MOCK'
archive_file = 'file_' + generate_uuid() + '.zip'
files = []
for i in xrange(10):
files.append({
'scope': scope,
'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570L,
'adler32': '0cc737eb',
'type': 'FILE',
'meta': {
'guid': str(generate_uuid())
}
})
self.replica_client.add_replicas(rse=rse,
files=[{
'scope': scope,
'name': archive_file,
'bytes': 1L,
'adler32': '0cc737eb'
}])
def reg_file_list(dids):
""" add files for a run to rucio.
attach the files to a run data set:
filenames: <scope>:xtc.file.<fn>
dataset: <scope>:xtc.runNNNNN
"""
print("Add files to RUCIO")
if len(dids) == 0:
print("No files to add")
return 0
dids_to_register = []
scopes = set()
for did in dids:
nd = {
'pfn': "file:{}".format(did['pfn']),
'bytes': os.path.getsize(did['pfn']),
'adler32': adler32(did['pfn']),
'name': did['name'],
'scope': did['scope']
}
dids_to_register.append(nd)
scopes.add(did['scope'])
print(nd)
print(scopes)
if len(scopes) != 1:
print("Wrong number of scopes", len(scopes))
return 2
# register files to xtc.files
client = ReplicaClient()
client.add_replicas('LCLS_REGD', dids_to_register)
# add files to run dataset
known_run_ds = {}
client = DIDClient()
for did in dids:
run = did['run']
scope = did['scope']
if run not in known_run_ds:
run_ds = "xtc.run%05d" % run
try:
client.get_did(scope, run_ds)
except DataIdentifierNotFound:
print("Create new dataset", run_ds)
client.add_dataset(scope, run_ds)
client.add_datasets_to_container(scope, 'xtc', [{'scope': scope, 'name': run_ds},])
known_run_ds[run] = run_ds
ds = known_run_ds[run]
try:
client.attach_dids(scope, run_ds, [{'scope': scope, 'name': did['name']}])
except FileAlreadyExists:
print("File already exists", did['name'])
else:
print("attached", ds, did['name'])
class TestReplicaMetalinkRedirection:
def __init__(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo_header = '-H "X-Rucio-VO: %s"' % config_get(
'client', 'vo', raise_exception=False, default='tst')
else:
self.vo_header = ''
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def test_replica_meta_redirection(self):
""" REDIRECT: metalink to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s' % (
self.cacert, self.token, self.vo_header, self.host, tmp_scope,
tmp_name)
_, out, _ = execute(cmd)
assert_in('404 Not Found', out)
self.replica_client.add_replicas(rse='MOCK',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'
}])
self.replica_client.add_replicas(rse='MOCK3',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'
}])
_, out, _ = execute(cmd)
assert_in('303 See Other', out)
assert_in('Link: </redirect/%s/%s/metalink' % (tmp_scope, tmp_name),
out)
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s/metalink' % (
self.cacert, self.token, self.vo_header, self.host, tmp_scope,
tmp_name)
_, out, _ = execute(cmd)
assert_in('200 OK', out)
assert_in('<?xml', out)
assert_in('<metalink', out)
assert_in('<url location="MOCK"', out)
assert_in('<url location="MOCK3"', out)
class TestReplicaHeaderRedirection(unittest.TestCase):
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo_header = '-H "X-Rucio-VO: %s"' % config_get(
'client', 'vo', raise_exception=False, default='tst')
else:
self.vo_header = ''
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def test_replica_header_redirection(self):
""" REDIRECT: header to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s' '' % (
self.cacert, self.token, self.vo_header, self.host, tmp_scope,
tmp_name)
_, out, _ = execute(cmd)
assert '404 Not Found'.lower() in out.lower()
self.replica_client.add_replicas(rse='MOCK',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'
}])
self.replica_client.add_replicas(rse='MOCK3',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'
}])
_, out, _ = execute(cmd)
assert '303 See Other'.lower() in out.lower()
assert 'Location: https://mock' in out
def rucio_register(self, filenames):
files = []
dids = []
for filename in filenames:
size = os.stat(str(filename)).st_size
adler = adler32(str(filename))
files.append({
'scope': self.scope,
'name': str(filename.parts[-1]),
'bytes': size,
'adler32': adler,
'pfn': self.pfn + str(filename.parts[-1])
})
replica_client = ReplicaClient()
replica_client.add_replicas(rse=self.rse, files=files)
didclient = DIDClient()
didclient.add_files_to_dataset(self.scope, self.dataset, files)
class TestDIDClients(object):
def __init__(self):
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
def test_add_and_list_archive(self):
""" ARCHIVE (CLIENT): Add files to archive and list the content."""
scope, rse = 'mock', 'MOCK'
archive_files = ['file_' + generate_uuid() + '.zip' for _ in range(2)]
files = []
for i in range(10):
files.append({
'scope': scope,
'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570,
'adler32': '0cc737eb',
'type': 'FILE',
'meta': {
'guid': str(generate_uuid())
}
})
for archive_file in archive_files:
self.replica_client.add_replicas(rse=rse,
files=[{
'scope': scope,
'name': archive_file,
'bytes': 1,
'adler32': '0cc737eb'
}])
self.did_client.add_files_to_archive(scope=scope,
name=archive_file,
files=files)
content = [
fil for fil in self.did_client.list_archive_content(
scope=scope, name=archive_file)
]
assert_equal(len(content), 10)
class TestReplicaHttpRedirection:
def setup(self):
self.cacert = config_get('test', 'cacert')
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.marker = '$> '
# get auth token
self.base_client = BaseClient()
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.replica_client = ReplicaClient()
def test_replica_http_redirection(self):
""" REPLICA (redirection): http redirection to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" -X GET %s/redirect/%s/%s''' % (self.cacert, self.token, self.host, tmp_scope, tmp_name)
exitcode, out, err = execute(cmd)
assert_in('404 Not Found', out)
# add replicas
self.replica_client.add_replicas(rse='MOCK', files=[{'scope': tmp_scope, 'name': tmp_name, 'bytes': 1L, 'adler32': '0cc737eb'}])
class TestBinRucio():
def setup(self):
try:
remove('/tmp/.rucio_root/auth_token_root')
except OSError as e:
if e.args[0] != 2:
raise e
self.marker = '$> '
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.user = '******'
self.def_rse = 'MOCK4'
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.account_client = AccountLimitClient()
self.account_client.set_account_limit('root', self.def_rse, -1)
def test_rucio_version(self):
"""CLIENT(USER): Rucio version"""
cmd = 'bin/rucio --version'
exitcode, out, err = execute(cmd)
nose.tools.assert_true('rucio' in err)
def test_rucio_ping(self):
"""CLIENT(USER): Rucio ping"""
cmd = 'rucio --host %s ping' % self.host
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
def test_add_account(self):
"""CLIENT(ADMIN): Add account"""
tmp_val = account_name_generator()
cmd = 'rucio-admin account add %s' % tmp_val
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_equal('Added new account: %s\n' % tmp_val, out)
def test_whoami(self):
"""CLIENT(USER): Rucio whoami"""
cmd = 'rucio whoami'
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_regexp_matches(out, re.compile('.*account.*'))
def test_add_identity(self):
"""CLIENT(ADMIN): Add identity"""
tmp_val = account_name_generator()
cmd = 'rucio-admin account add %s' % tmp_val
exitcode, out, err = execute(cmd)
nose.tools.assert_equal('Added new account: %s\n' % tmp_val, out)
cmd = 'rucio-admin identity add --account %s --type GSS --id [email protected] --email [email protected]' % tmp_val
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_equal(
'Added new identity to account: [email protected]%s\n' % tmp_val, out)
def test_del_identity(self):
"""CLIENT(ADMIN): Test del identity"""
tmp_acc = account_name_generator()
# create account
cmd = 'rucio-admin account add %s' % tmp_acc
exitcode, out, err = execute(cmd)
# add identity to account
cmd = 'rucio-admin identity add --account %s --type GSS --id [email protected] --email [email protected]' % tmp_acc
exitcode, out, err = execute(cmd)
# delete identity from account
cmd = 'rucio-admin identity delete --account %s --type GSS --id [email protected]' % tmp_acc
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal('Deleted identity: [email protected]\n', out)
# list identities for account
cmd = 'rucio-admin account list-identities %s' % (tmp_acc)
print(self.marker + cmd)
print(cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal('', out)
def test_attributes(self):
"""CLIENT(ADMIN): Add/List/Delete attributes"""
tmp_acc = account_name_generator()
# create account
cmd = 'rucio-admin account add %s' % tmp_acc
exitcode, out, err = execute(cmd)
# add attribute to the account
cmd = 'rucio-admin account add-attribute {0} --key test_attribute_key --value true'.format(
tmp_acc)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_equal(0, exitcode)
# list attributes
cmd = 'rucio-admin account list-attributes {0}'.format(tmp_acc)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_equal(0, exitcode)
# delete attribute to the account
cmd = 'rucio-admin account delete-attribute {0} --key test_attribute_key'.format(
tmp_acc)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_equal(0, exitcode)
def test_add_scope(self):
"""CLIENT(ADMIN): Add scope"""
tmp_scp = scope_name_generator()
tmp_acc = account_name_generator()
cmd = 'rucio-admin account add %s' % tmp_acc
exitcode, out, err = execute(cmd)
cmd = 'rucio-admin scope add --account %s --scope %s' % (tmp_acc,
tmp_scp)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(
'Added new scope to account: %s-%s\n' % (tmp_scp, tmp_acc), out)
def test_add_rse(self):
"""CLIENT(ADMIN): Add RSE"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_equal('Added new RSE: %s\n' % tmp_val, out)
def test_list_rses(self):
"""CLIENT(ADMIN): List RSEs"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
exitcode, out, err = execute(cmd)
cmd = 'rucio-admin rse list'
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_regexp_matches(out, re.compile('.*%s.*' % tmp_val))
def test_upload(self):
"""CLIENT(USER): Upload"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
exitcode, out, err = execute(cmd)
cmd = 'rucio upload'
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
def test_download(self):
"""CLIENT(USER): Download"""
cmd = 'rucio download'
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
def test_upload_file(self):
"""CLIENT(USER): Rucio upload files"""
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
cmd = 'rucio upload --rse {0} --scope {1} {2} {3} {4}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
nose.tools.assert_true(
"File %s:%s successfully uploaded on the storage" %
(self.user, tmp_file1[5:]) in out)
def test_upload_file_guid(self):
"""CLIENT(USER): Rucio upload file with guid"""
tmp_file1 = file_generator()
tmp_guid = generate_uuid()
cmd = 'rucio upload --rse {0} --guid {1} --scope {2} {3}'.format(
self.def_rse, tmp_guid, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
nose.tools.assert_true(
"File %s:%s successfully uploaded on the storage" %
(self.user, tmp_file1[5:]) in out)
def test_upload_repeated_file(self):
"""CLIENT(USER): Rucio upload repeated files"""
# One of the files to upload is already catalogued but was removed
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# get the rule for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
rule = out
# delete the file from the catalog
cmd = "rucio delete-rule {0}".format(rule)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# delete the fisical file
cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = 'rucio upload --rse {0} --scope {1} {2} {3} {4}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
nose.tools.assert_not_equal(
"File %s:%s successfully uploaded on the storage" %
(self.user, tmp_file1[5:]) in out, None)
def test_upload_repeated_file_dataset(self):
"""CLIENT(USER): Rucio upload repeated files to dataset"""
# One of the files to upload is already in the dataset
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# upload the files to the dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3} {4} {5}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# tmp_file1 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
def test_upload_file_dataset(self):
"""CLIENT(USER): Rucio upload files to dataset"""
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3} {4} {5}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
def test_upload_adds_md5digest(self):
"""CLIENT(USER): Upload Checksums"""
# user has a file to upload
filename = file_generator()
file_md5 = md5(filename)
# user uploads file
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, filename)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# When inspecting the metadata of the new file the user finds the md5 checksum
meta = self.did_client.get_metadata(scope=self.user, name=filename[5:])
nose.tools.assert_in('md5', meta)
nose.tools.assert_equal(meta['md5'], file_md5)
remove(filename)
def test_create_dataset(self):
"""CLIENT(USER): Rucio add dataset"""
tmp_name = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
cmd = 'rucio add-dataset ' + tmp_name
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search('Added ' + tmp_name, out), None)
def test_add_files_to_dataset(self):
"""CLIENT(USER): Rucio add files to dataset"""
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_dataset = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# create dataset
cmd = 'rucio add-dataset ' + tmp_dataset
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add files to dataset
cmd = 'rucio attach {0} {3}:{1} {3}:{2}'.format(
tmp_dataset, tmp_file1[5:], tmp_file2[5:],
self.user) # triming '/tmp/' from filename
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# find the added files
cmd = 'rucio list-files ' + tmp_dataset
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
def test_download_file(self):
"""CLIENT(USER): Rucio download files"""
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# download files
cmd = 'rucio download --dir /tmp {0}:{1}'.format(
self.user, tmp_file1[5:]) # triming '/tmp/' from filename
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the files with ls
cmd = 'ls /tmp/' # search in /tmp/
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
try:
for i in listdir('data13_hip'):
unlink('data13_hip/%s' % i)
rmdir('data13_hip')
except Exception:
pass
def test_download_succeeds_md5only(self):
"""CLIENT(USER): Rucio download succeeds MD5 only"""
# user has a file to upload
filename = file_generator()
file_md5 = md5(filename)
filesize = stat(filename).st_size
lfn = {
'name': filename[5:],
'scope': self.user,
'bytes': filesize,
'md5': file_md5
}
# user uploads file
self.replica_client.add_replicas(files=[lfn], rse=self.def_rse)
rse_settings = rsemgr.get_rse_info(self.def_rse)
protocol = rsemgr.create_protocol(rse_settings, 'write')
protocol.connect()
pfn = protocol.lfns2pfns(lfn).values()[0]
protocol.put(filename[5:], pfn, filename[:5])
protocol.close()
remove(filename)
# download files
cmd = 'rucio download --dir /tmp {0}:{1}'.format(
self.user, filename[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the files with ls
cmd = 'ls /tmp/{0}'.format(self.user) # search in /tmp/
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(filename[5:], out), None)
try:
for i in listdir('data13_hip'):
unlink('data13_hip/%s' % i)
rmdir('data13_hip')
except Exception:
pass
def test_download_fails_badmd5(self):
"""CLIENT(USER): Rucio download fails on MD5 mismatch"""
# user has a file to upload
filename = file_generator()
file_md5 = md5(filename)
filesize = stat(filename).st_size
lfn = {
'name': filename[5:],
'scope': self.user,
'bytes': filesize,
'md5': '0123456789abcdef0123456789abcdef'
}
# user uploads file
self.replica_client.add_replicas(files=[lfn], rse=self.def_rse)
rse_settings = rsemgr.get_rse_info(self.def_rse)
protocol = rsemgr.create_protocol(rse_settings, 'write')
protocol.connect()
pfn = protocol.lfns2pfns(lfn).values()[0]
protocol.put(filename[5:], pfn, filename[:5])
protocol.close()
remove(filename)
# download file
cmd = 'rucio download --dir /tmp {0}:{1}'.format(
self.user, filename[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# a failure message 'Checksum mismatch : local _____ vs storage _____' appears
report = 'Checksum\ mismatch\ \:\ local\ {0}\ vs\ recorded\ 0123456789abcdef0123456789abcdef'.format(
file_md5)
print('searching', report, 'in', err)
nose.tools.assert_not_equal(re.search(report, err), None)
# The file should not exist
cmd = 'ls /tmp/' # search in /tmp/
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(re.search(filename[5:], out), None)
try:
for i in listdir('data13_hip'):
unlink('data13_hip/%s' % i)
rmdir('data13_hip')
except Exception:
pass
def test_download_dataset(self):
"""CLIENT(USER): Rucio download dataset"""
tmp_file1 = file_generator()
tmp_dataset = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# create dataset
cmd = 'rucio add-dataset ' + tmp_dataset
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add files to dataset
cmd = 'rucio attach {0} {1}:{2}'.format(
tmp_dataset, self.user,
tmp_file1[5:]) # triming '/tmp/' from filename
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# download dataset
cmd = 'rucio download --dir /tmp {0}'.format(
tmp_dataset) # triming '/tmp/' from filename
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
search = '{0} successfully downloaded'.format(
tmp_file1[5:]) # triming '/tmp/' from filename
nose.tools.assert_not_equal(re.search(search, err), None)
def test_create_rule(self):
"""CLIENT(USER): Rucio add rule"""
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
# add quota
self.account_client.set_account_limit('root', tmp_rse, -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add quota
self.account_client.set_account_limit('root', tmp_rse, -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add quota
self.account_client.set_account_limit('root', tmp_rse, -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rules
cmd = "rucio add-rule {0}:{1} 3 'spacetoken=ATLASSCRATCHDISK'".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
rule = out[:-1] # triming new line character
# check if rule exist for the file
cmd = "rucio list-rules {0}:{1}".format(self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(rule, out), None)
def test_delete_rule(self):
"""CLIENT(USER): rule deletion"""
self.account_client.set_account_limit('root', self.def_rse, -1)
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
self.account_client.set_account_limit('root', tmp_rse, -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rules
cmd = "rucio add-rule {0}:{1} 1 'spacetoken=ATLASSCRATCHDISK'".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(err)
print(out)
# get the rules for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
(rule1, rule2) = out.split()
# delete the rules for the file
cmd = "rucio delete-rule {0}".format(rule1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = "rucio delete-rule {0}".format(rule2)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the file
cmd = "rucio list-dids {0}:{1}".format(self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(5, len(out.splitlines()))
def test_add_file_twice(self):
"""CLIENT(USER): Add file twice"""
tmp_file1 = file_generator()
# add file twice
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
nose.tools.assert_equal(
re.search(
"File {0}:{1} successfully uploaded on the storage".format(
self.user, tmp_file1[5:]), out), None)
def test_add_delete_add_file(self):
"""CLIENT(USER): Add/Delete/Add"""
tmp_file1 = file_generator()
# add file
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# get the rule for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
rule = out
# delete the file from the catalog
cmd = "rucio delete-rule {0}".format(rule)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# delete the fisical file
cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# modify the file to avoid same checksum
cmd = "echo 'delta' >> {0}".format(tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add the same file
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(
re.search(
"File {0}:{1} successfully uploaded on the storage".format(
self.user, tmp_file1[5:]), out), None)
def test_attach_files_dataset(self):
"""CLIENT(USER): Rucio attach files to dataset"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# upload the files
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file2, tmp_file3)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# attach the files to the dataset
cmd = 'rucio attach {0} {1}:{2} {1}:{3}'.format(
tmp_dsn, self.user, tmp_file2[5:],
tmp_file3[5:]) # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# tmp_file2 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file2[5:]), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
def test_detach_files_dataset(self):
"""CLIENT(USER): Rucio detach files to dataset"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3} {4} {5}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# detach the files to the dataset
cmd = 'rucio detach {0} {1}:{2} {1}:{3}'.format(
tmp_dsn, self.user, tmp_file2[5:],
tmp_file3[5:]) # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# tmp_file1 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_equal(
re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
def test_attach_file_twice(self):
"""CLIENT(USER): Rucio attach a file twice"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
# attach the files to the dataset
cmd = 'rucio attach {0} {1}:{2}'.format(
tmp_dsn, self.user,
tmp_file1[5:]) # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(re.search("The file already exists", err),
None)
def test_detach_non_existing_file(self):
"""CLIENT(USER): Rucio detach a non existing file"""
tmp_file1 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
# attach the files to the dataset
cmd = 'rucio detach {0} {1}:{2}'.format(
tmp_dsn, self.user, 'file_ghost') # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(
re.search("Data identifier not found.", err), None)
class TestReplicaClients:
def setup(self):
self.replica_client = ReplicaClient()
self.did_client = DIDClient()
def test_add_list_bad_replicas(self):
""" REPLICA (CLIENT): Add bad replicas"""
tmp_scope = 'mock'
nbfiles = 5
# Adding replicas to deterministic RSE
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
rse_info = rsemgr.get_rse_info('MOCK')
rse_id1 = rse_info['id']
self.replica_client.add_replicas(rse='MOCK', files=files)
# Listing replicas on deterministic RSE
replicas, list_rep = [], []
for replica in self.replica_client.list_replicas(dids=[{
'scope':
f['scope'],
'name':
f['name']
} for f in files],
schemes=['srm'],
unavailable=True):
replicas.extend(replica['rses']['MOCK'])
list_rep.append(replica)
r = self.replica_client.declare_bad_file_replicas(
replicas, 'This is a good reason')
assert_equal(r, {})
bad_replicas = list_bad_replicas()
nbbadrep = 0
for rep in list_rep:
for badrep in bad_replicas:
if badrep['rse_id'] == rse_id1:
if badrep['scope'] == rep['scope'] and badrep[
'name'] == rep['name']:
nbbadrep += 1
assert_equal(len(replicas), nbbadrep)
# Run necromancer once
necromancer_run(threads=1, bulk=10000, once=True)
# Try to attach a lost file
tmp_dsn = 'dataset_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn)
with assert_raises(UnsupportedOperation):
self.did_client.add_files_to_dataset(tmp_scope,
name=tmp_dsn,
files=files,
rse='MOCK')
# Adding replicas to non-deterministic RSE
files = [{
'scope':
tmp_scope,
'name':
'file_%s' % generate_uuid(),
'bytes':
1,
'adler32':
'0cc737eb',
'pfn':
'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s'
% (tmp_scope, generate_uuid()),
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
rse_info = rsemgr.get_rse_info('MOCK2')
rse_id2 = rse_info['id']
self.replica_client.add_replicas(rse='MOCK2', files=files)
# Listing replicas on non-deterministic RSE
replicas, list_rep = [], []
for replica in self.replica_client.list_replicas(dids=[{
'scope':
f['scope'],
'name':
f['name']
} for f in files],
schemes=['srm'],
unavailable=True):
replicas.extend(replica['rses']['MOCK2'])
list_rep.append(replica)
print(replicas, list_rep)
r = self.replica_client.declare_bad_file_replicas(
replicas, 'This is a good reason')
print(r)
assert_equal(r, {})
bad_replicas = list_bad_replicas()
nbbadrep = 0
for rep in list_rep:
for badrep in bad_replicas:
if badrep['rse_id'] == rse_id2:
if badrep['scope'] == rep['scope'] and badrep[
'name'] == rep['name']:
nbbadrep += 1
assert_equal(len(replicas), nbbadrep)
# Now adding non-existing bad replicas
files = [
'srm://mock2.com/rucio/tmpdisk/rucio_tests/%s/%s' %
(tmp_scope, generate_uuid()),
]
r = self.replica_client.declare_bad_file_replicas(
files, 'This is a good reason')
output = ['%s Unknown replica' % rep for rep in files]
assert_equal(r, {'MOCK2': output})
def test_add_suspicious_replicas(self):
""" REPLICA (CLIENT): Add suspicious replicas"""
tmp_scope = 'mock'
nbfiles = 5
# Adding replicas to deterministic RSE
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK', files=files)
# Listing replicas on deterministic RSE
replicas = []
list_rep = []
for replica in self.replica_client.list_replicas(dids=[{
'scope':
f['scope'],
'name':
f['name']
} for f in files],
schemes=['srm'],
unavailable=True):
replicas.extend(replica['rses']['MOCK'])
list_rep.append(replica)
r = self.replica_client.declare_suspicious_file_replicas(
replicas, 'This is a good reason')
assert_equal(r, {})
# Adding replicas to non-deterministic RSE
files = [{
'scope':
tmp_scope,
'name':
'file_%s' % generate_uuid(),
'bytes':
1,
'adler32':
'0cc737eb',
'pfn':
'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s'
% (tmp_scope, generate_uuid()),
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK2', files=files)
# Listing replicas on non-deterministic RSE
replicas = []
list_rep = []
for replica in self.replica_client.list_replicas(dids=[{
'scope':
f['scope'],
'name':
f['name']
} for f in files],
schemes=['srm'],
unavailable=True):
replicas.extend(replica['rses']['MOCK2'])
list_rep.append(replica)
r = self.replica_client.declare_suspicious_file_replicas(
replicas, 'This is a good reason')
assert_equal(r, {})
# Now adding non-existing bad replicas
files = [
'srm://mock2.com/rucio/tmpdisk/rucio_tests/%s/%s' %
(tmp_scope, generate_uuid()),
]
r = self.replica_client.declare_suspicious_file_replicas(
files, 'This is a good reason')
output = ['%s Unknown replica' % rep for rep in files]
assert_equal(r, {'MOCK2': output})
def test_bad_replica_methods_for_UI(self):
""" REPLICA (REST): Test the listing of bad and suspicious replicas """
mw = []
headers1 = {
'X-Rucio-Account': 'root',
'X-Rucio-Username': '******',
'X-Rucio-Password': '******'
}
result = TestApp(auth_app.wsgifunc(*mw)).get('/userpass',
headers=headers1,
expect_errors=True)
assert_equal(result.status, 200)
token = str(result.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
data = dumps({})
result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states',
headers=headers2,
params=data,
expect_errors=True)
assert_equal(result.status, 200)
tot_files = []
for line in result.body.split('\n'):
if line != '':
tot_files.append(dumps(line))
nb_tot_files = len(tot_files)
data = dumps({'state': 'B'})
result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states',
headers=headers2,
params=data,
expect_errors=True)
assert_equal(result.status, 200)
tot_bad_files = []
for line in result.body.split('\n'):
if line != '':
tot_bad_files.append(dumps(line))
nb_tot_bad_files1 = len(tot_bad_files)
data = dumps({'state': 'S', 'list_pfns': 'True'})
result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states',
headers=headers2,
params=data,
expect_errors=True)
assert_equal(result.status, 200)
tot_suspicious_files = []
for line in result.body.split('\n'):
if line != '':
tot_suspicious_files.append(dumps(line))
nb_tot_suspicious_files = len(tot_suspicious_files)
data = dumps({'state': 'T', 'list_pfns': 'True'})
result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states',
headers=headers2,
params=data,
expect_errors=True)
assert_equal(result.status, 200)
tot_temporary_unavailable_files = []
for line in result.body.split('\n'):
if line != '':
tot_temporary_unavailable_files.append(dumps(line))
nb_tot_temporary_unavailable_files = len(
tot_temporary_unavailable_files)
assert_equal(
nb_tot_files, nb_tot_bad_files1 + nb_tot_suspicious_files +
nb_tot_temporary_unavailable_files)
tomorrow = datetime.utcnow() + timedelta(days=1)
data = dumps({'state': 'B', 'younger_than': tomorrow.isoformat()})
result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states',
headers=headers2,
params=data,
expect_errors=True)
assert_equal(result.status, 200)
tot_bad_files = []
for line in result.body.split('\n'):
if line != '':
tot_bad_files.append(dumps(line))
nb_tot_bad_files = len(tot_bad_files)
assert_equal(nb_tot_bad_files, 0)
data = dumps({})
result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/summary',
headers=headers2,
params=data,
expect_errors=True)
assert_equal(result.status, 200)
nb_tot_bad_files2 = 0
for line in result.body.split('\n'):
if line != '':
line = loads(line)
nb_tot_bad_files2 += int(line.get('BAD', 0))
assert_equal(nb_tot_bad_files1, nb_tot_bad_files2)
def test_add_list_replicas(self):
""" REPLICA (CLIENT): Add, change state and list file replicas """
tmp_scope = 'mock'
nbfiles = 5
files1 = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK', files=files1)
files2 = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK3', files=files2)
replicas = [
r for r in self.replica_client.list_replicas(
dids=[{
'scope': i['scope'],
'name': i['name']
} for i in files1])
]
assert_equal(len(replicas), len(files1))
replicas = [
r for r in self.replica_client.list_replicas(
dids=[{
'scope': i['scope'],
'name': i['name']
} for i in files2],
schemes=['file'])
]
assert_equal(len(replicas), 5)
replicas = [
r for r in self.replica_client.list_replicas(
dids=[{
'scope': i['scope'],
'name': i['name']
} for i in files2],
schemes=['srm'])
]
assert_equal(len(replicas), 5)
files3 = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'state': 'U',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK3', files=files3)
replicas = [
r for r in self.replica_client.list_replicas(
dids=[{
'scope': i['scope'],
'name': i['name']
} for i in files3],
schemes=['file'])
]
for i in range(nbfiles):
assert_equal(replicas[i]['rses'], {})
files4 = []
for file in files3:
file['state'] = 'A'
files4.append(file)
self.replica_client.update_replicas_states('MOCK3', files=files4)
replicas = [
r for r in self.replica_client.list_replicas(
dids=[{
'scope': i['scope'],
'name': i['name']
} for i in files3],
schemes=['file'],
unavailable=True)
]
assert_equal(len(replicas), 5)
for i in range(nbfiles):
assert_in('MOCK3', replicas[i]['rses'])
def test_delete_replicas(self):
""" REPLICA (CLIENT): Add and delete file replicas """
tmp_scope = 'mock'
nbfiles = 5
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK', files=files)
with assert_raises(AccessDenied):
self.replica_client.delete_replicas(rse='MOCK', files=files)
# replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files])]
# assert_equal(len(replicas), 0)
def test_add_temporary_unavailable_pfns(self):
""" REPLICA (CLIENT): Add temporary unavailable PFNs"""
tmp_scope = 'mock'
nbfiles = 5
# Adding replicas to deterministic RSE
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK', files=files)
# Listing replicas on deterministic RSE
list_rep = []
for replica in self.replica_client.list_replicas(dids=[{
'scope':
f['scope'],
'name':
f['name']
} for f in files],
schemes=['srm'],
unavailable=True):
pfn = replica['pfns'].keys()[0]
list_rep.append(pfn)
# Submit bad PFNs
now = datetime.utcnow()
reason_str = generate_uuid()
self.replica_client.add_bad_pfns(pfns=list_rep,
reason=str(reason_str),
state='TEMPORARY_UNAVAILABLE',
expires_at=now.isoformat())
result = get_bad_pfns(limit=10000,
thread=None,
total_threads=None,
session=None)
bad_pfns = {}
for res in result:
bad_pfns[res['pfn']] = (res['state'], res['reason'],
res['expires_at'])
for pfn in list_rep:
pfn = str(clean_surls([pfn])[0])
assert_in(pfn, bad_pfns)
assert_equal(str(bad_pfns[pfn][0]), 'TEMPORARY_UNAVAILABLE')
assert_equal(bad_pfns[pfn][1], reason_str)
# Submit with wrong state
with assert_raises(RucioException):
self.replica_client.add_bad_pfns(pfns=list_rep,
reason=str(reason_str),
state='BADSTATE',
expires_at=now.isoformat())
# Run minos once
minos_run(threads=1, bulk=10000, once=True)
result = get_bad_pfns(limit=10000,
thread=None,
total_threads=None,
session=None)
pfns = [res['pfn'] for res in result]
res_pfns = []
for replica in list_rep:
if replica in pfns:
res_pfns.append(replica)
assert_equal(res_pfns, [])
# Check the state in the replica table
for did in files:
rep = get_replicas_state(scope=did['scope'], name=did['name'])
assert_equal(str(rep.keys()[0]), 'TEMPORARY_UNAVAILABLE')
rep = []
for did in files:
did['state'] = ReplicaState.from_sym('TEMPORARY_UNAVAILABLE')
rep.append(did)
# Run the minos expiration
minos_temp_run(threads=1, once=True)
# Check the state in the replica table
for did in files:
rep = get_replicas_state(scope=did['scope'], name=did['name'])
assert_equal(str(rep.keys()[0]), 'AVAILABLE')
def test_set_tombstone(self):
""" REPLICA (CLIENT): set tombstone on replica """
# Set tombstone on one replica
rse = 'MOCK4'
scope = 'mock'
user = '******'
name = generate_uuid()
add_replica(rse, scope, name, 4, user)
assert_equal(get_replica(rse, scope, name)['tombstone'], None)
self.replica_client.set_tombstone([{
'rse': rse,
'scope': scope,
'name': name
}])
assert_equal(get_replica(rse, scope, name)['tombstone'], OBSOLETE)
# Set tombstone on locked replica
name = generate_uuid()
add_replica(rse, scope, name, 4, user)
RuleClient().add_replication_rule([{
'name': name,
'scope': scope
}],
1,
rse,
locked=True)
with assert_raises(ReplicaIsLocked):
self.replica_client.set_tombstone([{
'rse': rse,
'scope': scope,
'name': name
}])
# Set tombstone on not found replica
name = generate_uuid()
with assert_raises(ReplicaNotFound):
self.replica_client.set_tombstone([{
'rse': rse,
'scope': scope,
'name': name
}])
class TestArchive(object):
def __init__(self):
self.dc = DIDClient()
self.rc = ReplicaClient()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': 'tst'}
else:
self.vo = {}
def test_add_and_list_archive(self):
""" ARCHIVE (CLIENT): Add files to archive and list the content """
scope, rse = 'mock', 'MOCK'
archive_files = ['file_' + generate_uuid() + '.zip' for _ in range(2)]
files = []
for i in range(10):
files.append({
'scope': scope,
'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570,
'adler32': '0cc737eb',
'type': 'FILE',
'meta': {
'guid': str(generate_uuid())
}
})
for archive_file in archive_files:
self.rc.add_replicas(rse=rse,
files=[{
'scope': scope,
'name': archive_file,
'bytes': 1,
'adler32': '0cc737eb'
}])
self.dc.add_files_to_archive(scope=scope,
name=archive_file,
files=files)
content = [
f for f in self.dc.list_archive_content(scope=scope,
name=archive_file)
]
assert_equal(len(content), 10)
def test_list_archive_contents_transparently(self):
""" ARCHIVE (CORE): Transparent archive listing """
scope = InternalScope('mock', **self.vo)
rse = 'APERTURE_%s' % rse_name_generator()
rse_id = add_rse(rse, **self.vo)
root = InternalAccount('root', **self.vo)
add_protocol(
rse_id, {
'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
# register archive
archive = {
'scope': scope,
'name': 'weighted.storage.cube.zip',
'type': 'FILE',
'bytes': 2596,
'adler32': 'beefdead'
}
archive_client = archive.copy()
archive_client['scope'] = archive_client['scope'].external
add_replicas(rse_id=rse_id, files=[archive], account=root)
# archived files with replicas
files_with_replicas = [{
'scope':
scope,
'name':
'witrep-%i-%s' % (i, str(generate_uuid())),
'type':
'FILE',
'bytes':
1234,
'adler32':
'deadbeef'
} for i in range(2)]
files_with_replicas_client = []
for f in files_with_replicas:
new_file = f.copy()
new_file['scope'] = new_file['scope'].external
files_with_replicas_client.append(new_file)
add_replicas(rse_id=rse_id, files=files_with_replicas, account=root)
self.dc.add_files_to_archive(scope=scope.external,
name=archive_client['name'],
files=files_with_replicas_client)
res = [
r['pfns'] for r in self.rc.list_replicas(dids=[{
'scope': scope.external,
'name': f['name']
} for f in files_with_replicas_client],
resolve_archives=True)
]
assert_equal(len(res), 2)
assert_equal(len(res[0]), 2)
assert_equal(len(res[1]), 2)
for r in res:
for p in r:
if r[p]['domain'] == 'zip':
assert_in('weighted.storage.cube.zip?xrdcl.unzip=witrep-',
p)
else:
assert_not_in(
'weighted.storage.cube.zip?xrdcl.unzip=witrep-', p)
# archived files without replicas
files = [{
'scope': scope.external,
'name': 'norep-%i-%s' % (i, str(generate_uuid())),
'type': 'FILE',
'bytes': 1234,
'adler32': 'deadbeef'
} for i in range(2)]
self.dc.add_files_to_archive(scope=scope.external,
name=archive_client['name'],
files=files)
res = [
r['pfns'] for r in self.rc.list_replicas(dids=[{
'scope': scope.external,
'name': f['name']
} for f in files],
resolve_archives=True)
]
assert_equal(len(res), 2)
for r in res:
assert_in('weighted.storage.cube.zip?xrdcl.unzip=norep-',
r.keys()[0])
def test_list_archive_contents_at_rse(self):
""" ARCHIVE (CORE): Transparent archive listing at RSE """
scope = InternalScope('mock', **self.vo)
root = InternalAccount('root', **self.vo)
rse1 = 'APERTURE_%s' % rse_name_generator()
rse1_id = add_rse(rse1, **self.vo)
add_protocol(
rse1_id, {
'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
rse2 = 'BLACKMESA_%s' % rse_name_generator()
rse2_id = add_rse(rse2, **self.vo)
add_protocol(
rse2_id, {
'scheme': 'root',
'hostname': 'root.blackmesa.com',
'port': 1409,
'prefix': '//lambda/complex/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
# register archive
archive1 = {
'scope': scope,
'name': 'cube.1.zip',
'type': 'FILE',
'bytes': 2596,
'adler32': 'beefdead'
}
archive2 = {
'scope': scope,
'name': 'cube.2.zip',
'type': 'FILE',
'bytes': 5432,
'adler32': 'deadbeef'
}
add_replicas(rse_id=rse1_id, files=[archive1], account=root)
add_replicas(rse_id=rse2_id, files=[archive2], account=root)
# archived files with replicas
archived_file = [{
'scope': scope.external,
'name': 'zippedfile-%i-%s' % (i, str(generate_uuid())),
'type': 'FILE',
'bytes': 4322,
'adler32': 'beefbeef'
} for i in range(2)]
self.dc.add_files_to_archive(scope=scope.external,
name=archive1['name'],
files=archived_file)
self.dc.add_files_to_archive(scope=scope.external,
name=archive2['name'],
files=archived_file)
res = [
r['pfns'] for r in self.rc.list_replicas(dids=[{
'scope': f['scope'],
'name': f['name']
} for f in archived_file],
rse_expression=rse1,
resolve_archives=True)
]
res = self.rc.list_replicas(dids=[{
'scope': f['scope'],
'name': f['name']
} for f in archived_file],
metalink=True,
rse_expression=rse1,
resolve_archives=True)
assert_in('APERTURE', res)
assert_not_in('BLACKMESA', res)
res = self.rc.list_replicas(dids=[{
'scope': f['scope'],
'name': f['name']
} for f in archived_file],
metalink=True,
rse_expression=rse2,
resolve_archives=True)
assert_in('BLACKMESA', res)
assert_not_in('APERTURE', res)
class TestReplicaClients:
def setup(self):
self.replica_client = ReplicaClient()
self.did_client = DIDClient()
def test_add_list_bad_replicas(self):
""" REPLICA (CLIENT): Add bad replicas"""
tmp_scope = 'mock'
nbfiles = 5
# Adding replicas to deterministic RSE
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in range(nbfiles)]
rse_info = rsemgr.get_rse_info('MOCK')
rse_id1 = rse_info['id']
self.replica_client.add_replicas(rse='MOCK', files=files)
# Listing replicas on deterministic RSE
replicas, list_rep = [], []
for replica in self.replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], unavailable=True):
replicas.extend(replica['rses']['MOCK'])
list_rep.append(replica)
r = self.replica_client.declare_bad_file_replicas(replicas, 'This is a good reason')
assert_equal(r, {})
bad_replicas = list_bad_replicas()
nbbadrep = 0
for rep in list_rep:
for badrep in bad_replicas:
if badrep['rse_id'] == rse_id1:
if badrep['scope'] == rep['scope'] and badrep['name'] == rep['name']:
nbbadrep += 1
assert_equal(len(replicas), nbbadrep)
# Run necromancer once
run(threads=1, bulk=10000, once=True)
# Try to attach a lost file
tmp_dsn = 'dataset_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn)
with assert_raises(UnsupportedOperation):
self.did_client.add_files_to_dataset(tmp_scope, name=tmp_dsn, files=files, rse='MOCK')
# Adding replicas to non-deterministic RSE
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb',
'pfn': 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), 'meta': {'events': 10}} for i in range(nbfiles)]
rse_info = rsemgr.get_rse_info('MOCK2')
rse_id2 = rse_info['id']
self.replica_client.add_replicas(rse='MOCK2', files=files)
# Listing replicas on non-deterministic RSE
replicas, list_rep = [], []
for replica in self.replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], unavailable=True):
replicas.extend(replica['rses']['MOCK2'])
list_rep.append(replica)
print(replicas, list_rep)
r = self.replica_client.declare_bad_file_replicas(replicas, 'This is a good reason')
print(r)
assert_equal(r, {})
bad_replicas = list_bad_replicas()
nbbadrep = 0
for rep in list_rep:
for badrep in bad_replicas:
if badrep['rse_id'] == rse_id2:
if badrep['scope'] == rep['scope'] and badrep['name'] == rep['name']:
nbbadrep += 1
assert_equal(len(replicas), nbbadrep)
# Now adding non-existing bad replicas
files = ['srm://mock2.com/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), ]
r = self.replica_client.declare_bad_file_replicas(files, 'This is a good reason')
output = ['%s Unknown replica' % rep for rep in files]
assert_equal(r, {'MOCK2': output})
def test_add_suspicious_replicas(self):
""" REPLICA (CLIENT): Add suspicious replicas"""
tmp_scope = 'mock'
nbfiles = 5
# Adding replicas to deterministic RSE
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK', files=files)
# Listing replicas on deterministic RSE
replicas = []
list_rep = []
for replica in self.replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], unavailable=True):
replicas.extend(replica['rses']['MOCK'])
list_rep.append(replica)
r = self.replica_client.declare_suspicious_file_replicas(replicas, 'This is a good reason')
assert_equal(r, {})
# Adding replicas to non-deterministic RSE
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb',
'pfn': 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), 'meta': {'events': 10}} for i in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK2', files=files)
# Listing replicas on non-deterministic RSE
replicas = []
list_rep = []
for replica in self.replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], unavailable=True):
replicas.extend(replica['rses']['MOCK2'])
list_rep.append(replica)
r = self.replica_client.declare_suspicious_file_replicas(replicas, 'This is a good reason')
assert_equal(r, {})
# Now adding non-existing bad replicas
files = ['srm://mock2.com/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), ]
r = self.replica_client.declare_suspicious_file_replicas(files, 'This is a good reason')
output = ['%s Unknown replica' % rep for rep in files]
assert_equal(r, {'MOCK2': output})
def test_bad_replica_methods_for_UI(self):
""" REPLICA (REST): Test the listing of bad and suspicious replicas """
mw = []
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(r1.status, 200)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
data = dumps({})
r2 = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True)
assert_equal(r2.status, 200)
tot_files = []
for line in r2.body.split('\n'):
if line != '':
tot_files.append(dumps(line))
nb_tot_files = len(tot_files)
data = dumps({'state': 'B'})
r2 = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True)
assert_equal(r2.status, 200)
tot_bad_files = []
for line in r2.body.split('\n'):
if line != '':
tot_bad_files.append(dumps(line))
nb_tot_bad_files1 = len(tot_bad_files)
data = dumps({'state': 'S', 'list_pfns': 'True'})
r2 = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True)
assert_equal(r2.status, 200)
tot_suspicious_files = []
for line in r2.body.split('\n'):
if line != '':
tot_suspicious_files.append(dumps(line))
nb_tot_suspicious_files = len(tot_suspicious_files)
assert_equal(nb_tot_files, nb_tot_bad_files1 + nb_tot_suspicious_files)
tomorrow = datetime.utcnow() + timedelta(days=1)
data = dumps({'state': 'B', 'younger_than': tomorrow.isoformat()})
r2 = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True)
assert_equal(r2.status, 200)
tot_bad_files = []
for line in r2.body.split('\n'):
if line != '':
tot_bad_files.append(dumps(line))
nb_tot_bad_files = len(tot_bad_files)
assert_equal(nb_tot_bad_files, 0)
data = dumps({})
r2 = TestApp(rep_app.wsgifunc(*mw)).get('/bad/summary', headers=headers2, params=data, expect_errors=True)
assert_equal(r2.status, 200)
nb_tot_bad_files2 = 0
for line in r2.body.split('\n'):
if line != '':
line = loads(line)
nb_tot_bad_files2 += int(line['BAD'])
assert_equal(nb_tot_bad_files1, nb_tot_bad_files2)
def test_add_list_replicas(self):
""" REPLICA (CLIENT): Add, change state and list file replicas """
tmp_scope = 'mock'
nbfiles = 5
files1 = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK', files=files1)
files2 = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK3', files=files2)
replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files1])]
assert_equal(len(replicas), len(files1))
replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files2], schemes=['file'])]
assert_equal(len(replicas), 5)
replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files2], schemes=['srm'])]
assert_equal(len(replicas), 5)
files3 = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'state': 'U', 'meta': {'events': 10}} for i in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK3', files=files3)
replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files3], schemes=['file'])]
for i in range(nbfiles):
assert_equal(replicas[i]['rses'], {})
files4 = []
for file in files3:
file['state'] = 'A'
files4.append(file)
self.replica_client.update_replicas_states('MOCK3', files=files4)
replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files3], schemes=['file'], unavailable=True)]
assert_equal(len(replicas), 5)
for i in range(nbfiles):
assert_in('MOCK3', replicas[i]['rses'])
def test_delete_replicas(self):
""" REPLICA (CLIENT): Add and delete file replicas """
tmp_scope = 'mock'
nbfiles = 5
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in range(nbfiles)]
self.replica_client.add_replicas(rse='MOCK', files=files)
with assert_raises(AccessDenied):
self.replica_client.delete_replicas(rse='MOCK', files=files)
class TestReplicaMetalink:
def setup(self):
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.base_client = BaseClient(account='root',
ca_cert=config_get('client', 'ca_cert'),
auth_type='x509')
self.token = self.base_client.headers['X-Rucio-Auth-Token']
self.fname = generate_uuid()
rses = ['MOCK', 'MOCK3', 'MOCK4']
dsn = generate_uuid()
self.files = [{
'scope': 'mock',
'name': self.fname,
'bytes': 1,
'adler32': '0cc737eb'
}]
self.did_client.add_dataset(scope='mock', name=dsn)
self.did_client.add_files_to_dataset('mock',
name=dsn,
files=self.files,
rse='MOCK')
for r in rses:
self.replica_client.add_replicas(r, self.files)
def test_list_replicas_metalink_4(self):
""" REPLICA (METALINK): List replicas as metalink version 4 """
ml = xmltodict.parse(self.replica_client.list_replicas(
self.files,
metalink=4,
unavailable=True,
schemes=['https', 'sftp', 'file']),
xml_attribs=False)
assert_equal(3, len(ml['metalink']['file']['url']))
def test_get_did_from_pfns_nondeterministic(self):
""" REPLICA (CLIENT): Get list of DIDs associated to PFNs for non-deterministic sites"""
rse = 'MOCK2'
tmp_scope = 'mock'
nbfiles = 3
pfns = []
input = {}
rse_info = rsemgr.get_rse_info(rse)
assert_equal(rse_info['deterministic'], False)
files = [{
'scope':
tmp_scope,
'name':
'file_%s' % generate_uuid(),
'bytes':
1,
'adler32':
'0cc737eb',
'pfn':
'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s'
% (tmp_scope, generate_uuid()),
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
for f in files:
input[f['pfn']] = {'scope': f['scope'], 'name': f['name']}
add_replicas(rse=rse,
files=files,
account='root',
ignore_availability=True)
for replica in list_replicas(dids=[{
'scope': f['scope'],
'name': f['name'],
'type': DIDType.FILE
} for f in files],
schemes=['srm'],
ignore_availability=True):
for rse in replica['rses']:
pfns.extend(replica['rses'][rse])
for result in self.replica_client.get_did_from_pfns(pfns, rse):
pfn = result.keys()[0]
assert_equal(input[pfn], result.values()[0])
def test_get_did_from_pfns_deterministic(self):
""" REPLICA (CLIENT): Get list of DIDs associated to PFNs for deterministic sites"""
tmp_scope = 'mock'
rse = 'MOCK3'
nbfiles = 3
pfns = []
input = {}
rse_info = rsemgr.get_rse_info(rse)
assert_equal(rse_info['deterministic'], True)
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
p = rsemgr.create_protocol(rse_info, 'read', scheme='srm')
for f in files:
pfn = p.lfns2pfns(lfns={
'scope': f['scope'],
'name': f['name']
}).values()[0]
pfns.append(pfn)
input[pfn] = {'scope': f['scope'], 'name': f['name']}
add_replicas(rse=rse,
files=files,
account='root',
ignore_availability=True)
for result in self.replica_client.get_did_from_pfns(pfns, rse):
pfn = result.keys()[0]
assert_equal(input[pfn], result.values()[0])
class TestArchive(object):
def __init__(self):
self.dc = DIDClient()
self.rc = ReplicaClient()
def test_add_and_list_archive(self):
""" ARCHIVE (CLIENT): Add files to archive and list the content """
scope, rse = 'mock', 'MOCK'
archive_files = ['file_' + generate_uuid() + '.zip' for _ in range(2)]
files = []
for i in range(10):
files.append({
'scope': scope,
'name': 'lfn.%s' % str(generate_uuid()),
'bytes': 724963570,
'adler32': '0cc737eb',
'type': 'FILE',
'meta': {
'guid': str(generate_uuid())
}
})
for archive_file in archive_files:
self.rc.add_replicas(rse=rse,
files=[{
'scope': scope,
'name': archive_file,
'bytes': 1,
'adler32': '0cc737eb'
}])
self.dc.add_files_to_archive(scope=scope,
name=archive_file,
files=files)
content = [
f for f in self.dc.list_archive_content(scope=scope,
name=archive_file)
]
assert_equal(len(content), 10)
def test_list_archive_contents_transparently(self):
""" ARCHIVE (CORE): Transparent archive listing """
scope = 'mock'
rse = 'APERTURE_%s' % rse_name_generator()
add_rse(rse)
add_protocol(
rse, {
'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
# register archive
archive = {
'scope': scope,
'name': 'weighted.storage.cube.zip',
'type': 'FILE',
'bytes': 2596,
'adler32': 'beefdead'
}
add_replicas(rse=rse, files=[archive], account='root')
# archived files with replicas
files_with_replicas = [{
'scope':
scope,
'name':
'witrep-%i-%s' % (i, str(generate_uuid())),
'type':
'FILE',
'bytes':
1234,
'adler32':
'deadbeef'
} for i in xrange(2)]
add_replicas(rse=rse, files=files_with_replicas, account='root')
self.dc.add_files_to_archive(scope=scope,
name=archive['name'],
files=files_with_replicas)
res = [
r['pfns'] for r in self.rc.list_replicas(dids=[{
'scope': scope,
'name': f['name']
} for f in files_with_replicas])
]
assert_equal(len(res), 2)
assert_equal(len(res[0]), 2)
assert_equal(len(res[1]), 2)
for r in res:
for p in r:
if r[p]['domain'] == 'zip':
assert_in('weighted.storage.cube.zip?xrdcl.unzip=witrep-',
p)
else:
assert_not_in(
'weighted.storage.cube.zip?xrdcl.unzip=witrep-', p)
# archived files without replicas
files = [{
'scope': scope,
'name': 'norep-%i-%s' % (i, str(generate_uuid())),
'type': 'FILE',
'bytes': 1234,
'adler32': 'deadbeef'
} for i in xrange(2)]
self.dc.add_files_to_archive(scope=scope,
name=archive['name'],
files=files)
res = [
r['pfns'] for r in self.rc.list_replicas(dids=[{
'scope': scope,
'name': f['name']
} for f in files])
]
assert_equal(len(res), 2)
for r in res:
assert_in('weighted.storage.cube.zip?xrdcl.unzip=norep-',
r.keys()[0])
del_rse(rse)
# SIZE = GFAL.stat(str(URL)).st_size
# CHECKSUM = GFAL.checksum(str(URL), 'adler32')
print("Registering file: pfn=%s size=%s checksum=%s" % (URL, SIZE, CHECKSUM))
#except GError:
except:
print("no file found at %s" % URL)
exit()
R = ReplicaClient()
REPLICAS = list(R.list_replicas([{'scope': OPTIONS.scope, 'name': OPTIONS.name}]))
if REPLICAS:
REPLICAS = REPLICAS[0]
if 'rses' in REPLICAS:
if OPTIONS.rse in REPLICAS['rses']:
print("file %s with scope %s has already a replica at %s" %
(OPTIONS.name, OPTIONS.scope, OPTIONS.rse))
exit()
REPLICA = [{
'scope': OPTIONS.scope,
'name' : OPTIONS.name,
'adler32': CHECKSUM,
'bytes': SIZE,
'pfn': URL
}]
R.add_replicas(rse=OPTIONS.rse, files=REPLICA)
if adler32 != dbs_adler32 or phedex_bytes != dbs_bytes:
raise RuntimeError('Checksums or size do not match')
replica = {'scope': 'cms',
'name': fileDict['file'][0]['name'],
'bytes': phedex_bytes,
'meta': {'guid': str(uuid.uuid4()).upper(),
'events': dbs_events,
}, # Should be able to remove later (client cares now)
'adler32': adler32.rjust(8, '0'),
}
print(" Adding file to block: %s" % replica['name'])
replicas.append(replica)
# Mark the replicas as at a site
status = rClient.add_replicas(rse=DUMMY_RSE, files=replicas)
print(' Status for add_replicas', status)
# Attach the files to the dataset representing the block and the single dataset
try:
status = dClient.attach_dids(scope='cms', name=rucio_block_ds, dids=replicas)
# status = dClient.attach_dids(scope='cms', name=RUCIO_DS, dids=replicas)
print('Status for attach', status)
except RucioException:
print("Attach failed, probabably already done.")
# status = ruleClient.add_replication_rule(dids=block_datasets, copies=2, rse_expression='tier=2',
# lifetime=DAYS_TO_LIVE//2, account='ewv')
class DatasetInjector(object):
"""
General Class for injecting a cms dataset in rucio
"""
def __init__(self,
dataset,
site,
rse=None,
scope=DEFAULT_SCOPE,
uuid=None,
check=True,
lifetime=None,
dry_run=False):
self.dataset = dataset
self.site = site
if rse is None:
rse = site
self.rse = rse
self.scope = scope
self.uuid = uuid
self.check = check
self.lifetime = lifetime
self.dry_run = dry_run
self.blocks = []
self.url = ''
self.getmetadata()
self.get_global_url()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.gfal = Gfal2Context()
def get_file_url(self, lfn):
"""
Return the rucio url of a file.
"""
return self.url + '/' + lfn
def get_global_url(self):
"""
Return the base path of the rucio url
"""
print("Getting parameters for rse %s" % self.rse)
rse = rsemgr.get_rse_info(self.rse)
proto = rse['protocols'][0]
schema = proto['scheme']
prefix = proto['prefix'] + '/' + self.scope.replace('.', '/')
if schema == 'srm':
prefix = proto['extended_attributes']['web_service_path'] + prefix
url = schema + '://' + proto['hostname']
if proto['port'] != 0:
url = url + ':' + str(proto['port'])
self.url = url + prefix
print("Determined base url %s" % self.url)
def getmetadata(self):
"""
Gets the list of blocks at a site, their files and their metadata
"""
print("Initializing... getting the list of blocks and files")
blocks = das_go_client("block dataset=%s site=%s system=phedex" %
(self.dataset, self.site))
for item in blocks:
uuid = item['block'][0]['name'].split('#')[1]
if (self.uuid is None) or (uuid == self.uuid):
block = {'name': item['block'][0]['name'], 'files': []}
files = das_go_client("file block=%s site=%s system=phedex" %
(block['name'], self.site))
for item2 in files:
cksum = re.match(r"adler32:([^,]+)",
item2['file'][0]['checksum'])
cksum = cksum.group(0).split(':')[1]
cksum = "{0:0{1}x}".format(int(cksum, 16), 8)
block['files'].append({
'name': item2['file'][0]['name'],
'checksum': cksum,
'size': item2['file'][0]['size']
})
self.blocks.append(block)
print("Initalization done.")
def register(self):
"""
Create the container, the datasets and attach them to the container.
"""
print("Registering...")
self.register_container()
for block in self.blocks:
self.register_dataset(block['name'])
for filemd in block['files']:
self.register_replica(filemd)
self.attach_file(filemd['name'], block['name'])
print("All datasets, blocks and files registered")
def register_container(self):
"""
Create the container.
"""
print("registering container %s" % self.dataset)
if self.dry_run:
print(' Dry run only. Not creating container.')
return
try:
self.didc.add_container(scope=self.scope,
name=self.dataset,
lifetime=self.lifetime)
except DataIdentifierAlreadyExists:
print(" Container %s already exists" % self.dataset)
def register_dataset(self, block):
"""
Create the dataset and attach them to teh container
"""
print("registering dataset %s" % block)
if self.dry_run:
print(' Dry run only. Not creating dataset.')
return
try:
self.didc.add_dataset(scope=self.scope,
name=block,
lifetime=self.lifetime)
except DataIdentifierAlreadyExists:
print(" Dataset %s already exists" % block)
try:
print("attaching dataset %s to container %s" %
(block, self.dataset))
self.didc.attach_dids(scope=self.scope,
name=self.dataset,
dids=[{
'scope': self.scope,
'name': block
}])
except RucioException:
print(" Dataset already attached")
def attach_file(self, lfn, block):
"""
Attach the file to the container
"""
if self.dry_run:
print(' Dry run only. Not attaching files.')
return
try:
print("attaching file %s" % lfn)
self.didc.attach_dids(scope=self.scope,
name=block,
dids=[{
'scope': self.scope,
'name': lfn
}])
except FileAlreadyExists:
print("File already attached")
def register_replica(self, filemd):
"""
Register file replica.
"""
print("registering file %s" % filemd['name'])
if self.dry_run:
print(' Dry run only. Not registering files.')
return
if self.check:
self.check_storage(filemd)
if not self.check_replica(filemd['name']):
self.repc.add_replicas(rse=self.rse,
files=[{
'scope':
self.scope,
'name':
filemd['name'],
'adler32':
filemd['checksum'],
'bytes':
filemd['size'],
'pfn':
self.get_file_url(filemd['name'])
}])
def check_storage(self, filemd):
"""
Check size and checksum of a file on storage
"""
url = self.get_file_url(filemd['name'])
print("checking url %s" % url)
try:
size = self.gfal.stat(str(url)).st_size
checksum = self.gfal.checksum(str(url), 'adler32')
print("got size and checksum of file: pfn=%s size=%s checksum=%s" %
(url, size, checksum))
except GError:
print("no file found at %s" % url)
return False
if str(size) != str(filemd['size']):
print("wrong size for file %s. Expected %s got %s" %
(filemd['name'], filemd['size'], size))
return False
if str(checksum) != str(filemd['checksum']):
print("wrong checksum for file %s. Expected %s git %s" %
(filemd['name'], filemd['checksum'], checksum))
return False
print("size and checksum are ok")
return True
def check_replica(self, lfn):
"""
Check if a replica of the given file at the site already exists.
"""
print("checking if file %s with scope %s has already a replica at %s" %
(lfn, self.scope, self.rse))
replicas = list(
self.repc.list_replicas([{
'scope': self.scope,
'name': lfn
}]))
if replicas:
replicas = replicas[0]
if 'rses' in replicas:
if self.rse in replicas['rses']:
print("file %s with scope %s has already a replica at %s" %
(lfn, self.scope, self.rse))
return True
print("no existing replicas")
return False
def test_list_dataset_replicas_archive(self):
""" REPLICA (CLIENT): List dataset replicas with archives. """
replica_client = ReplicaClient()
did_client = DIDClient()
rule_client = RuleClient()
scope = 'mock'
rse = 'APERTURE_%s' % rse_name_generator()
rse_id = add_rse(rse, **self.vo)
add_protocol(rse_id=rse_id,
parameter={
'scheme': 'root',
'hostname': 'root.aperture.com',
'port': 1409,
'prefix': '//test/chamber/',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
rse2 = 'BLACKMESA_%s' % rse_name_generator()
rse2_id = add_rse(rse2, **self.vo)
add_protocol(rse_id=rse2_id,
parameter={
'scheme': 'root',
'hostname': 'root.blackmesa.com',
'port': 1409,
'prefix': '//underground/facility',
'impl': 'rucio.rse.protocols.xrootd.Default',
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
})
# register archive
archive = {
'scope': scope,
'name': 'another.%s.zip' % generate_uuid(),
'type': 'FILE',
'bytes': 2596,
'adler32': 'deedbeaf'
}
replica_client.add_replicas(rse=rse, files=[archive])
replica_client.add_replicas(rse=rse2, files=[archive])
archived_files = [{
'scope':
scope,
'name':
'zippedfile-%i-%s' % (i, str(generate_uuid())),
'type':
'FILE',
'bytes':
4322,
'adler32':
'deaddead'
} for i in range(2)]
replica_client.add_replicas(rse=rse2, files=archived_files)
did_client.add_files_to_archive(scope=scope,
name=archive['name'],
files=archived_files)
dataset_name = 'find_me.' + str(generate_uuid())
did_client.add_dataset(scope=scope, name=dataset_name)
did_client.attach_dids(scope=scope,
name=dataset_name,
dids=archived_files)
rule_client.add_replication_rule(dids=[{
'scope': scope,
'name': dataset_name
}],
account='root',
copies=1,
rse_expression=rse,
grouping='DATASET')
res = [
r for r in replica_client.list_dataset_replicas(scope=scope,
name=dataset_name)
]
assert len(res) == 1
assert res[0]['state'] == 'UNAVAILABLE'
res = [
r for r in replica_client.list_dataset_replicas(
scope=scope, name=dataset_name, deep=True)
]
assert len(res) == 3
assert res[0]['state'] == 'AVAILABLE'
assert res[1]['state'] == 'AVAILABLE'
assert res[2]['state'] == 'AVAILABLE'
del_rse(rse_id)
class RunSync(object):
"""
Synchronize the replica of a given run at WIPAC-ORIG
the corresponding Rucio site.
"""
def __init__(self,
run,
originrse=DEFAULT_ORIGIN_RSE,
destrse=None,
scope=DEFAULT_SCOPE,
check=True,
lifetime=None,
dry_run=False,
container=None):
"""
:param dataset: Name of the PhEDEx dataset to synchronize with Rucio.
:param pnn: PhEDEx node name to filter on for replica information.
"""
self.run = run
self.originrse = originrse
self.destrse = destrse
self.scope = scope
self.check = check
self.lifetime = lifetime
self.dry_run = dry_run
self.container = container
self.rucio_datasets = {}
self.run_files = {}
self.existent_replica_files = {}
self.url = ''
self.gfal = Gfal2Context()
self.run_Number = None
self.get_run_Number()
self.files_storage = {}
self.get_global_url()
self.didc = DIDClient()
self.repc = ReplicaClient()
self.rulesClient = RuleClient()
# Right now obtaining the Metadata from the storage at WIPAC
# Hopefully in the future from JADE # TODO
self.get_run_Files()
self.get_rucio_metadata()
self.update_run_Files()
self.get_files_metadata()
def update_run_Files(self):
"""
Updating the run files wiht only the files that have not been registered
"""
for f in self.existent_replica_files:
file_name = f.split('/')[-1:][0]
if file_name in self.run_files:
print("File: %s already registered. Skipping it" % file_name)
self.run_files.pop(file_name)
def get_files_metadata(self):
for f in self.run_files:
if self.run + '/' + f not in self.existent_replica_files:
self.obtain_metadata(f)
print("Metadat initialization done")
def obtain_metadata(self, filename):
"""
Get the size and checksum for every file in the run from the gftp server
"""
url = self.get_file_url(filename)
print("checking metadata for url %s" % url)
try:
size = self.gfal.stat(str(url)).st_size
adler32 = self.gfal.checksum(str(url), 'adler32')
print(
"got size and adler 32checksum of file: pfn=%s size=%s checksum=%s"
% (url, size, adler32))
self.run_files[filename] = {
'size': size,
'adler32': adler32,
'name': self.run + '/' + filename
}
except GError:
print("no file found at %s" % url)
return False
def get_file_url(self, filename):
return self.url + '/' + self.run + '/' + filename
def get_global_url(self):
"""
Return the base path of the rucio url
"""
print("Getting parameters for rse %s" % self.originrse)
rse = rsemgr.get_rse_info(self.originrse)
proto = rse['protocols'][0]
schema = proto['scheme']
prefix = proto['prefix'] + self.scope.replace('.', '/')
if schema == 'srm':
prefix = proto['extended_attributes']['web_service_path'] + prefix
url = schema + '://' + proto['hostname']
if proto['port'] != 0:
url = url + ':' + str(proto['port'])
self.url = url + prefix
print("Determined base url %s" % self.url)
def get_run_Number(self):
"""
Obtain the run number out of whole run IceCube/2016/filtered/level2pass2/0101/Run00127347
"""
print("Obtaining run number out of run(dataset): %s" % self.run)
self.run_Number = self.run.split("/")[-1]
print("Run number (dataset): %s" % self.run_Number)
def get_run_Files(self):
"""
Gets the list of files for a given run and their checksums from the storage
"""
self.run_url = self.url + '/' + self.run
print("Listin files from url : %s" % self.run_url)
run_files = []
try:
run_files = self.gfal.listdir(str(self.run_url))
except GError:
print("No files found at %s" % str(self.run_url))
print("Files found in storage:")
count = 0
for f in run_files:
if len(f) > 3:
if count < 5000:
self.run_files[f] = {}
count = count + 1
else:
break
def get_rucio_metadata(self):
"""
Gets the list of datasets at the Rucio RSE, the files, and the metadata.
"""
print(
"Initializing Rucio... getting the list of blocks and files at %s"
% self.originrse)
registered_datasets = self.repc.list_datasets_per_rse(self.originrse)
for dataset in registered_datasets:
self.rucio_datasets[dataset] = {}
replica_info = self.repc.list_replicas([{
"scope": self.scope,
"name": '/' + self.run_Number
}],
rse_expression="rse=%s" %
self.originrse)
replica_files = set()
for file_info in replica_info:
name = file_info['name']
if self.originrse in file_info['rses']:
replica_files.add(name)
self.existent_replica_files = replica_files
print("Rucio initialization done.")
def register(self):
"""
Create the container, the datasets and attach them to the container.
"""
print("Registering...")
self.register_dataset(self.run_Number)
self.register_replicas(self.run_files)
self.register_container(self.container)
self.attach_dataset_to_container(self.run_Number, self.container)
self.add_replica_rule(dataset=self.run_Number, destRSE=self.destrse)
def register_container(self, container):
"""
Registering the container
"""
print("Registering the container %s with scope: %s" %
(container, self.scope))
if container is None:
print('No container added, not registering any container')
return
if self.dry_run:
print('Dry run only, not registering the container')
return
try:
self.didc.add_container(scope=self.scope,
name=container,
lifetime=self.lifetime)
except DataIdentifierAlreadyExists:
print("Container %s already exists" % container)
except InvalidObject:
print("Problem with container name: %s" % container)
def attach_dataset_to_container(self, dataset, container):
"""
Attaching the dataset to a container
"""
print("Attaching dataset %s, to container: %s" % (dataset, container))
if container is None:
print('No container added, not registering dataset in container')
return
if self.dry_run:
print('Dry run only, not attaching dataset container')
return
try:
self.didc.attach_dids(scope=self.scope,
name=container,
dids=[{
'scope': self.scope,
'name': '/' + dataset
}])
except RucioException:
print("dataset already attached to container")
return
def register_dataset(self, run):
"""
Registering a dataset in the rucio database
"""
print("registering dataset %s" % run)
if self.dry_run:
print(' Dry run only. Not creating dataset.')
return
try:
self.didc.add_dataset(scope=self.scope,
name=run,
lifetime=self.lifetime)
except DataIdentifierAlreadyExists:
print(" Dataset %s already exists" % run)
def register_replicas(self, replicas):
"""
Register file replica.
"""
if not replicas:
return
print("registering files in Rucio: %s" %
", ".join([replicas[filemd]['name'] for filemd in replicas]))
if self.dry_run:
print(' Dry run only. Not registering files.')
return
try:
self.repc.add_replicas(rse=self.originrse,
files=[{
'scope':
self.scope,
'name':
replicas[filemd]['name'],
'adler32':
replicas[filemd]['adler32'],
'bytes':
replicas[filemd]['size'],
} for filemd in replicas])
print("Adding files to dataset: %s" % self.run_Number)
except InvalidObject:
print("Problem with file name does not match pattern")
for filemd in replicas:
try:
self.didc.attach_dids(scope=self.scope,
name=self.run_Number,
dids=[{
'scope': self.scope,
'name': replicas[filemd]['name']
}])
except FileAlreadyExists:
print("File already attached")
def add_replica_rule(self, destRSE, dataset):
"""
Create a replication rule for one dataset "Run" at an RSE
"""
print("Creating replica rule for dataset %s at rse: %s" %
(dataset, destRSE))
if self.dry_run:
print(' Dry run only. Not creating rules')
return
if destRSE:
try:
self.rulesClient.add_replication_rule([{
"scope": self.scope,
"name": "/" + dataset
}],
copies=1,
rse_expression=destRSE)
except DuplicateRule:
print('Rule already exists')
def _rucio_register(beamline, uid, filenames):
"""
Register the file in rucio for replication to SDCC.
"""
scope = beamline
container = uid
replica_client = ReplicaClient()
didclient = DIDClient()
scopeclient = ScopeClient()
ruleclient = RuleClient()
for root, ending, filename in filenames:
#size = os.stat(str(filename)).st_size
#adler = adler32(str(filename))
files = [{
'scope': scope,
'name': filename.split('/')[-1],
'bytes': 1000,
#'adler32': "unknown",
'pfn': pfn + filename
}]
dataset = os.path.join(root, ending)
dataset = '.'.join(dataset.split('/')[1:-1])
print("DATASET", dataset)
breakpoint()
try:
scopeclient.add_scope(account='nsls2data', scope=scope)
except rucio.common.exception.Duplicate:
pass
replica_client.add_replicas(rse=rse, files=files)
# Create a new container if it doesn't exist.
try:
didclient.add_did(scope=scope, name=uid, type='container')
except rucio.common.exception.DataIdentifierAlreadyExists:
pass
# Create a replication rule.
try:
dids = [{'scope': scope, 'name': container}]
ruleclient.add_replication_rule(
dids=dids,
copies=1,
rse_expression='SDCC',
lifetime=86400, # Seconds
account='nsls2data',
source_replica_expression='NSLS2',
purge_replicas=True,
comment='purge_replicas in 24 hours')
except rucio.common.exception.DuplicateRule:
pass
# Create a new dataset if it doesn't exist.
try:
didclient.add_did(scope=scope, name=dataset, type='dataset')
except rucio.common.exception.DataIdentifierAlreadyExists:
pass
attachment = {
'scope': scope,
'name': uid,
'dids': [{
'scope': scope,
'name': dataset
}]
}
try:
didclient.add_files_to_dataset(scope, dataset, files)
except rucio.common.exception.FileAlreadyExists:
pass
try:
didclient.add_datasets_to_containers([attachment])
except rucio.common.exception.DuplicateContent:
pass
class TestBinRucio():
def setup(self):
try:
remove('/tmp/.rucio_root/auth_token_root')
except OSError as e:
if e.args[0] != 2:
raise e
self.marker = '$> '
self.host = config_get('client', 'rucio_host')
self.auth_host = config_get('client', 'auth_host')
self.user = '******'
self.def_rse = 'MOCK4'
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.account_client = AccountLimitClient()
self.account_client.set_account_limit('root', self.def_rse, -1)
add_rse_attribute(self.def_rse, 'istape', 'False')
self.upload_success_str = 'Successfully uploaded file %s'
def test_rucio_version(self):
"""CLIENT(USER): Rucio version"""
cmd = 'bin/rucio --version'
exitcode, out, err = execute(cmd)
nose.tools.assert_true('rucio' in err)
def test_rucio_ping(self):
"""CLIENT(USER): Rucio ping"""
cmd = 'rucio --host %s ping' % self.host
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
def test_add_account(self):
"""CLIENT(ADMIN): Add account"""
tmp_val = account_name_generator()
cmd = 'rucio-admin account add %s' % tmp_val
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_equal('Added new account: %s\n' % tmp_val, out)
def test_whoami(self):
"""CLIENT(USER): Rucio whoami"""
cmd = 'rucio whoami'
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_regexp_matches(out, re.compile('.*account.*'))
def test_add_identity(self):
"""CLIENT(ADMIN): Add identity"""
tmp_val = account_name_generator()
cmd = 'rucio-admin account add %s' % tmp_val
exitcode, out, err = execute(cmd)
nose.tools.assert_equal('Added new account: %s\n' % tmp_val, out)
cmd = 'rucio-admin identity add --account %s --type GSS --id [email protected] --email [email protected]' % tmp_val
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_equal(
'Added new identity to account: [email protected]%s\n' % tmp_val, out)
def test_del_identity(self):
"""CLIENT(ADMIN): Test del identity"""
tmp_acc = account_name_generator()
# create account
cmd = 'rucio-admin account add %s' % tmp_acc
exitcode, out, err = execute(cmd)
# add identity to account
cmd = 'rucio-admin identity add --account %s --type GSS --id [email protected] --email [email protected]' % tmp_acc
exitcode, out, err = execute(cmd)
# delete identity from account
cmd = 'rucio-admin identity delete --account %s --type GSS --id [email protected]' % tmp_acc
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal('Deleted identity: [email protected]\n', out)
# list identities for account
cmd = 'rucio-admin account list-identities %s' % (tmp_acc)
print(self.marker + cmd)
print(cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal('', out)
def test_attributes(self):
"""CLIENT(ADMIN): Add/List/Delete attributes"""
tmp_acc = account_name_generator()
# create account
cmd = 'rucio-admin account add %s' % tmp_acc
exitcode, out, err = execute(cmd)
# add attribute to the account
cmd = 'rucio-admin account add-attribute {0} --key test_attribute_key --value true'.format(
tmp_acc)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_equal(0, exitcode)
# list attributes
cmd = 'rucio-admin account list-attributes {0}'.format(tmp_acc)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_equal(0, exitcode)
# delete attribute to the account
cmd = 'rucio-admin account delete-attribute {0} --key test_attribute_key'.format(
tmp_acc)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_equal(0, exitcode)
def test_add_scope(self):
"""CLIENT(ADMIN): Add scope"""
tmp_scp = scope_name_generator()
tmp_acc = account_name_generator()
cmd = 'rucio-admin account add %s' % tmp_acc
exitcode, out, err = execute(cmd)
cmd = 'rucio-admin scope add --account %s --scope %s' % (tmp_acc,
tmp_scp)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(
'Added new scope to account: %s-%s\n' % (tmp_scp, tmp_acc), out)
def test_add_rse(self):
"""CLIENT(ADMIN): Add RSE"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_equal('Added new deterministic RSE: %s\n' % tmp_val,
out)
def test_add_rse_nondet(self):
"""CLIENT(ADMIN): Add non-deterministic RSE"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add --non-deterministic %s' % tmp_val
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_equal(
'Added new non-deterministic RSE: %s\n' % tmp_val, out)
def test_list_rses(self):
"""CLIENT(ADMIN): List RSEs"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
exitcode, out, err = execute(cmd)
cmd = 'rucio-admin rse list'
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_regexp_matches(out, re.compile('.*%s.*' % tmp_val))
def test_upload(self):
"""CLIENT(USER): Upload"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
exitcode, out, err = execute(cmd)
cmd = 'rucio upload'
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
def test_download(self):
"""CLIENT(USER): Download"""
cmd = 'rucio download'
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
def test_upload_file(self):
"""CLIENT(USER): Rucio upload files"""
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
nose.tools.assert_true((self.upload_success_str %
path.basename(tmp_file1)) in out)
nose.tools.assert_true((self.upload_success_str %
path.basename(tmp_file2)) in out)
nose.tools.assert_true((self.upload_success_str %
path.basename(tmp_file3)) in out)
def test_upload_file_guid(self):
"""CLIENT(USER): Rucio upload file with guid"""
tmp_file1 = file_generator()
tmp_guid = generate_uuid()
cmd = 'rucio -v upload --rse {0} --guid {1} --scope {2} {3}'.format(
self.def_rse, tmp_guid, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
nose.tools.assert_true((self.upload_success_str %
path.basename(tmp_file1)) in out)
def test_upload_repeated_file(self):
"""CLIENT(USER): Rucio upload repeated files"""
# One of the files to upload is already catalogued but was removed
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_file1_name = path.basename(tmp_file1)
cmd = 'rucio -v upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# get the rule for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
self.user, tmp_file1_name)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
rule = out
# delete the file from the catalog
cmd = "rucio delete-rule {0}".format(rule)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# delete the physical file
cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1_name)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
nose.tools.assert_true((self.upload_success_str %
tmp_file1_name) in out)
def test_upload_repeated_file_dataset(self):
"""CLIENT(USER): Rucio upload repeated files to dataset"""
# One of the files to upload is already in the dataset
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_file1_name = path.basename(tmp_file1)
tmp_file3_name = path.basename(tmp_file3)
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# upload the files to the dataset
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# tmp_file1 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file1_name), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file3_name), out), None)
def test_upload_file_dataset(self):
"""CLIENT(USER): Rucio upload files to dataset"""
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_file1_name = path.basename(tmp_file1)
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file1_name), out), None)
def test_upload_adds_md5digest(self):
"""CLIENT(USER): Upload Checksums"""
# user has a file to upload
filename = file_generator()
tmp_file1_name = path.basename(filename)
file_md5 = md5(filename)
# user uploads file
cmd = 'rucio -v upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, filename)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# When inspecting the metadata of the new file the user finds the md5 checksum
meta = self.did_client.get_metadata(scope=self.user,
name=tmp_file1_name)
nose.tools.assert_in('md5', meta)
nose.tools.assert_equal(meta['md5'], file_md5)
remove(filename)
def test_create_dataset(self):
"""CLIENT(USER): Rucio add dataset"""
tmp_name = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
cmd = 'rucio add-dataset ' + tmp_name
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search('Added ' + tmp_name, out), None)
def test_add_files_to_dataset(self):
"""CLIENT(USER): Rucio add files to dataset"""
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_dataset = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# create dataset
cmd = 'rucio add-dataset ' + tmp_dataset
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add files to dataset
cmd = 'rucio attach {0} {3}:{1} {3}:{2}'.format(
tmp_dataset, tmp_file1[5:], tmp_file2[5:],
self.user) # triming '/tmp/' from filename
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# find the added files
cmd = 'rucio list-files ' + tmp_dataset
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
def test_download_file(self):
"""CLIENT(USER): Rucio download files"""
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# download files
cmd = 'rucio -v download --dir /tmp {0}:{1}'.format(
self.user, tmp_file1[5:]) # triming '/tmp/' from filename
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the files with ls
cmd = 'ls /tmp/' # search in /tmp/
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# download files
cmd = 'rucio -v download --dir /tmp {0}:{1}'.format(
self.user, tmp_file1[5:-2] + '*') # triming '/tmp/' from filename
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the files with ls
cmd = 'ls /tmp/' # search in /tmp/
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
try:
for i in listdir('data13_hip'):
unlink('data13_hip/%s' % i)
rmdir('data13_hip')
except Exception:
pass
def test_download_filter(self):
"""CLIENT(USER): Rucio download with filter options"""
# Use filter option to download file with wildcarded name
tmp_file1 = file_generator()
uuid = generate_uuid()
cmd = 'rucio upload --rse {0} --scope {1} --guid {2} {3}'.format(
self.def_rse, self.user, uuid, tmp_file1)
exitcode, out, err = execute(cmd)
print(out, err)
remove(tmp_file1)
wrong_guid = generate_uuid()
cmd = 'rucio -v download --dir /tmp {0}:{1} --filter guid={2}'.format(
self.user, '*', wrong_guid)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = 'ls /tmp/{0}'.format(self.user)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(re.search(tmp_file1[5:], out), None)
cmd = 'rucio -v download --dir /tmp {0}:{1} --filter guid={2}'.format(
self.user, '*', uuid)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = 'ls /tmp/{0}'.format(self.user)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
# Only use filter option to download file
tmp_file1 = file_generator()
uuid = generate_uuid()
cmd = 'rucio upload --rse {0} --scope {1} --guid {2} {3}'.format(
self.def_rse, self.user, uuid, tmp_file1)
exitcode, out, err = execute(cmd)
print(out, err)
remove(tmp_file1)
wrong_guid = generate_uuid()
cmd = 'rucio -v download --dir /tmp --scope {0} --filter guid={1}'.format(
self.user, wrong_guid)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = 'ls /tmp/{0}'.format(self.user)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(re.search(tmp_file1[5:], out), None)
cmd = 'rucio -v download --dir /tmp --scope {0} --filter guid={1}'.format(
self.user, uuid)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = 'ls /tmp/{0}'.format(self.user)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
# Only use filter option to download dataset
tmp_file1 = file_generator()
dataset_name = 'dataset_%s' % generate_uuid()
cmd = 'rucio upload --rse {0} --scope {1} {2} {1}:{3}'.format(
self.def_rse, self.user, tmp_file1, dataset_name)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
remove(tmp_file1)
db_session = session.get_session()
db_session.query(models.DataIdentifier).filter_by(
scope=self.user, name=dataset_name).one().length = 15
db_session.commit()
cmd = 'rucio download --dir /tmp --scope {0} --filter length=100'.format(
self.user)
exitcode, out, err = execute(cmd)
cmd = 'ls /tmp/{0}'.format(dataset_name)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(re.search(tmp_file1[5:], out), None)
cmd = 'rucio download --dir /tmp --scope {0} --filter length=15'.format(
self.user)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = 'ls /tmp/{0}'.format(dataset_name)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
# Use filter option to download dataset with wildcarded name
tmp_file1 = file_generator()
dataset_name = 'dataset_%s' % generate_uuid()
cmd = 'rucio upload --rse {0} --scope {1} {2} {1}:{3}'.format(
self.def_rse, self.user, tmp_file1, dataset_name)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
remove(tmp_file1)
db_session = session.get_session()
db_session.query(models.DataIdentifier).filter_by(
scope=self.user, name=dataset_name).one().length = 1
db_session.commit()
cmd = 'rucio download --dir /tmp {0}:{1} --filter length=10'.format(
self.user, dataset_name[0:-1] + '*')
exitcode, out, err = execute(cmd)
cmd = 'ls /tmp/{0}'.format(dataset_name)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(re.search(tmp_file1[5:], out), None)
cmd = 'rucio download --dir /tmp {0}:{1} --filter length=1'.format(
self.user, dataset_name[0:-1] + '*')
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = 'ls /tmp/{0}'.format(dataset_name)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
def test_download_succeeds_md5only(self):
"""CLIENT(USER): Rucio download succeeds MD5 only"""
# user has a file to upload
filename = file_generator()
file_md5 = md5(filename)
filesize = stat(filename).st_size
lfn = {
'name': filename[5:],
'scope': self.user,
'bytes': filesize,
'md5': file_md5
}
# user uploads file
self.replica_client.add_replicas(files=[lfn], rse=self.def_rse)
rse_settings = rsemgr.get_rse_info(self.def_rse)
protocol = rsemgr.create_protocol(rse_settings, 'write')
protocol.connect()
pfn = protocol.lfns2pfns(lfn).values()[0]
protocol.put(filename[5:], pfn, filename[:5])
protocol.close()
remove(filename)
# download files
cmd = 'rucio -v download --dir /tmp {0}:{1}'.format(
self.user, filename[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the files with ls
cmd = 'ls /tmp/{0}'.format(self.user) # search in /tmp/
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(filename[5:], out), None)
try:
for i in listdir('data13_hip'):
unlink('data13_hip/%s' % i)
rmdir('data13_hip')
except Exception:
pass
def test_download_fails_badmd5(self):
"""CLIENT(USER): Rucio download fails on MD5 mismatch"""
# user has a file to upload
filename = file_generator()
file_md5 = md5(filename)
filesize = stat(filename).st_size
lfn = {
'name': filename[5:],
'scope': self.user,
'bytes': filesize,
'md5': '0123456789abcdef0123456789abcdef'
}
# user uploads file
self.replica_client.add_replicas(files=[lfn], rse=self.def_rse)
rse_settings = rsemgr.get_rse_info(self.def_rse)
protocol = rsemgr.create_protocol(rse_settings, 'write')
protocol.connect()
pfn = protocol.lfns2pfns(lfn).values()[0]
protocol.put(filename[5:], pfn, filename[:5])
protocol.close()
remove(filename)
# download file
cmd = 'rucio -v download --dir /tmp {0}:{1}'.format(
self.user, filename[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
report = 'Local\ checksum\:\ {0},\ Rucio\ checksum\:\ 0123456789abcdef0123456789abcdef'.format(
file_md5)
print('searching', report, 'in', err)
nose.tools.assert_not_equal(re.search(report, err), None)
# The file should not exist
cmd = 'ls /tmp/' # search in /tmp/
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(re.search(filename[5:], out), None)
try:
for i in listdir('data13_hip'):
unlink('data13_hip/%s' % i)
rmdir('data13_hip')
except Exception:
pass
def test_download_dataset(self):
"""CLIENT(USER): Rucio download dataset"""
tmp_file1 = file_generator()
tmp_dataset = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# create dataset
cmd = 'rucio add-dataset ' + tmp_dataset
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add files to dataset
cmd = 'rucio attach {0} {1}:{2}'.format(
tmp_dataset, self.user,
tmp_file1[5:]) # triming '/tmp/' from filename
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# download dataset
cmd = 'rucio -v download --dir /tmp {0}'.format(
tmp_dataset) # triming '/tmp/' from filename
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
search = '{0} successfully downloaded'.format(
tmp_file1[5:]) # triming '/tmp/' from filename
nose.tools.assert_not_equal(re.search(search, err), None)
def test_create_rule(self):
"""CLIENT(USER): Rucio add rule"""
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
# add quota
self.account_client.set_account_limit('root', tmp_rse, -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add quota
self.account_client.set_account_limit('root', tmp_rse, -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add quota
self.account_client.set_account_limit('root', tmp_rse, -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rules
cmd = "rucio add-rule {0}:{1} 3 'spacetoken=ATLASSCRATCHDISK'".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
rule = out[:-1] # triming new line character
# check if rule exist for the file
cmd = "rucio list-rules {0}:{1}".format(self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(rule, out), None)
def test_delete_rule(self):
"""CLIENT(USER): rule deletion"""
self.account_client.set_account_limit('root', self.def_rse, -1)
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
self.account_client.set_account_limit('root', tmp_rse, -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rules
cmd = "rucio add-rule {0}:{1} 1 'spacetoken=ATLASSCRATCHDISK'".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(err)
print(out)
# get the rules for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
(rule1, rule2) = out.split()
# delete the rules for the file
cmd = "rucio delete-rule {0}".format(rule1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = "rucio delete-rule {0}".format(rule2)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the file
cmd = "rucio list-dids --filter type=all {0}:{1}".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(5, len(out.splitlines()))
def test_add_file_twice(self):
"""CLIENT(USER): Add file twice"""
tmp_file1 = file_generator()
# add file twice
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
nose.tools.assert_equal(
re.search(
"File {0}:{1} successfully uploaded on the storage".format(
self.user, tmp_file1[5:]), out), None)
def test_add_delete_add_file(self):
"""CLIENT(USER): Add/Delete/Add"""
tmp_file1 = file_generator()
# add file
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# get the rule for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
rule = out
# delete the file from the catalog
cmd = "rucio delete-rule {0}".format(rule)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# delete the fisical file
cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# modify the file to avoid same checksum
cmd = "echo 'delta' >> {0}".format(tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add the same file
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(
re.search(
"File {0}:{1} successfully uploaded on the storage".format(
self.user, tmp_file1[5:]), out), None)
def test_attach_files_dataset(self):
"""CLIENT(USER): Rucio attach files to dataset"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# upload the files
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file2, tmp_file3)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# attach the files to the dataset
cmd = 'rucio attach {0} {1}:{2} {1}:{3}'.format(
tmp_dsn, self.user, tmp_file2[5:],
tmp_file3[5:]) # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# tmp_file2 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file2[5:]), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
def test_detach_files_dataset(self):
"""CLIENT(USER): Rucio detach files to dataset"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3} {4} {5}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# detach the files to the dataset
cmd = 'rucio detach {0} {1}:{2} {1}:{3}'.format(
tmp_dsn, self.user, tmp_file2[5:],
tmp_file3[5:]) # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# tmp_file1 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_equal(
re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
def test_attach_file_twice(self):
"""CLIENT(USER): Rucio attach a file twice"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
# attach the files to the dataset
cmd = 'rucio attach {0} {1}:{2}'.format(
tmp_dsn, self.user,
tmp_file1[5:]) # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(re.search("The file already exists", err),
None)
def test_attach_dataset_twice(self):
""" CLIENT(USER): Rucio attach a dataset twice """
container = 'container_%s' % generate_uuid()
dataset = 'dataset_%s' % generate_uuid()
self.did_client.add_container(scope=self.user, name=container)
self.did_client.add_dataset(scope=self.user, name=dataset)
# Attach dataset to container
cmd = 'rucio attach {0}:{1} {0}:{2}'.format(self.user, container,
dataset)
exitcode, out, err = execute(cmd)
# Attach again
cmd = 'rucio attach {0}:{1} {0}:{2}'.format(self.user, container,
dataset)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(
re.search(
"Data identifier already added to the destination content",
err), None)
def test_detach_non_existing_file(self):
"""CLIENT(USER): Rucio detach a non existing file"""
tmp_file1 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
# attach the files to the dataset
cmd = 'rucio detach {0} {1}:{2}'.format(
tmp_dsn, self.user, 'file_ghost') # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(
re.search("Data identifier not found.", err), None)
def test_list_did_recursive(self):
""" CLIENT(USER): List did recursive """
# Setup nested collections
tmp_scope = 'mock'
tmp_container_1 = 'container_%s' % generate_uuid()
cmd = 'rucio add-container {0}:{1}'.format(tmp_scope, tmp_container_1)
exitcode, out, err = execute(cmd)
tmp_container_2 = 'container_%s' % generate_uuid()
cmd = 'rucio add-container {0}:{1}'.format(tmp_scope, tmp_container_2)
exitcode, out, err = execute(cmd)
tmp_container_3 = 'container_%s' % generate_uuid()
cmd = 'rucio add-container {0}:{1}'.format(tmp_scope, tmp_container_3)
exitcode, out, err = execute(cmd)
cmd = 'rucio attach {0}:{1} {0}:{2}'.format(tmp_scope, tmp_container_1,
tmp_container_2)
exitcode, out, err = execute(cmd)
cmd = 'rucio attach {0}:{1} {0}:{2}'.format(tmp_scope, tmp_container_2,
tmp_container_3)
exitcode, out, err = execute(cmd)
# All attached DIDs are expected
cmd = 'rucio list-dids {0}:{1} --recursive'.format(
tmp_scope, tmp_container_1)
exitcode, out, err = execute(cmd)
nose.tools.assert_not_equal(re.search(tmp_container_1, out), None)
nose.tools.assert_not_equal(re.search(tmp_container_2, out), None)
nose.tools.assert_not_equal(re.search(tmp_container_3, out), None)
# Wildcards are not allowed to use with --recursive
cmd = 'rucio list-dids {0}:* --recursive'.format(tmp_scope)
exitcode, out, err = execute(cmd)
nose.tools.assert_not_equal(
re.search("Option recursive cannot be used with wildcards", err),
None)
def test_attach_many_dids(self):
""" CLIENT(USER): Rucio attach many (>1000) DIDs """
# Setup data for CLI check
tmp_dsn_name = 'Container' + rse_name_generator()
tmp_dsn_did = self.user + ':' + tmp_dsn_name
self.did_client.add_did(scope=self.user,
name=tmp_dsn_name,
type='CONTAINER')
files = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': self.user,
'type': 'DATASET'
} for i in range(0, 1500)]
self.did_client.add_dids(files[:1000])
self.did_client.add_dids(files[1000:])
# Attaching over 1000 DIDs with CLI
cmd = 'rucio attach {0}'.format(tmp_dsn_did)
for tmp_file in files:
cmd += ' {0}:{1}'.format(tmp_file['scope'], tmp_file['name'])
exitcode, out, err = execute(cmd)
print(out)
print(err)
# Checking if the execution was successfull and if the DIDs belong together
nose.tools.assert_not_equal(
re.search('DIDs successfully attached', out), None)
cmd = 'rucio list-content {0}'.format(tmp_dsn_did)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
# first dataset must be in the container
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, files[0]['name']), out),
None)
# last dataset must be in the container
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, files[-1]['name']), out),
None)
# Setup data with file
did_file_path = 'list_dids.txt'
files = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': self.user,
'type': 'DATASET'
} for i in range(0, 1500)]
self.did_client.add_dids(files[:1000])
self.did_client.add_dids(files[1000:])
with open(did_file_path, 'w') as did_file:
for file in files:
did_file.write(file['scope'] + ':' + file['name'] + '\n')
did_file.close()
# Attaching over 1000 files per file
cmd = 'rucio attach {0} -f {1}'.format(tmp_dsn_did, did_file_path)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(did_file_path)
# Checking if the execution was successfull and if the DIDs belong together
nose.tools.assert_not_equal(
re.search('DIDs successfully attached', out), None)
cmd = 'rucio list-content {0}'.format(tmp_dsn_did)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
# first file must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, files[0]['name']), out),
None)
# last file must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, files[-1]['name']), out),
None)
def test_attach_many_dids_twice(self):
""" CLIENT(USER) Attach many (>1000) DIDs twice """
# Setup data for CLI check
container_name = 'container' + generate_uuid()
container = self.user + ':' + container_name
self.did_client.add_did(scope=self.user,
name=container_name,
type='CONTAINER')
datasets = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': self.user,
'type': 'DATASET'
} for i in range(0, 1500)]
self.did_client.add_dids(datasets[:1000])
self.did_client.add_dids(datasets[1000:])
# Attaching over 1000 DIDs with CLI
cmd = 'rucio attach {0}'.format(container)
for dataset in datasets:
cmd += ' {0}:{1}'.format(dataset['scope'], dataset['name'])
exitcode, out, err = execute(cmd)
# Attaching twice
cmd = 'rucio attach {0}'.format(container)
for dataset in datasets:
cmd += ' {0}:{1}'.format(dataset['scope'], dataset['name'])
exitcode, out, err = execute(cmd)
nose.tools.assert_not_equal(
re.search("DIDs successfully attached", out), None)
# Attaching twice plus one DID that is not already attached
new_dataset = {
'name': 'dsn_%s' % generate_uuid(),
'scope': self.user,
'type': 'DATASET'
}
datasets.append(new_dataset)
self.did_client.add_did(scope=self.user,
name=new_dataset['name'],
type='DATASET')
cmd = 'rucio attach {0}'.format(container)
for dataset in datasets:
cmd += ' {0}:{1}'.format(dataset['scope'], dataset['name'])
exitcode, out, err = execute(cmd)
nose.tools.assert_not_equal(
re.search("DIDs successfully attached", out), None)
cmd = 'rucio list-content {0}'.format(container)
exitcode, out, err = execute(cmd)
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, new_dataset['name']), out),
None)
#!/usr/bin/env python
from rucio.client.replicaclient import ReplicaClient
client = ReplicaClient()
#data = {
# 'pfn': "root://134.79.103.90:2076//psdm/rucio/rte/rte01/xtc/rte01-r0001-s01-c00.xtc",
# 'bytes': 104857600,
# 'md5': 'bfec14dd0fd2e733df4a7d00511f5a0c',
# 'name': "xtc.file.rte01-r0001-s01-c00.xtc",
# 'scope': "rte01",
#}
data = {
'pfn':
"root://134.79.103.90:2076//psdm/rucio/rte/rte01/xtc/rte01-r0002-s02-c00.xtc",
'bytes': 1073741824,
'adler32': '897ea592',
'md5': 'a0c84f769c76989f87a44b59409d73ea',
'name': "rte.rte01.xtc.rte01-r0002-s02-c00.xtc",
'scope': "psdm",
}
client.add_replicas('LCLS_DATA', [data])
