def test_download_dataset(self):
"""CLIENT(USER): Rucio download dataset"""
tmp_file1 = file_generator()
tmp_dataset = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# add files
cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# create dataset
cmd = 'rucio add-dataset ' + tmp_dataset
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add files to dataset
cmd = 'rucio add-files-to-dataset --to {0} {1}:{2}'.format(tmp_dataset, self.user, tmp_file1[5:]) # triming '/tmp/' from filename
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# download dataset
cmd = 'rucio download --dir /tmp {0}'.format(tmp_dataset) # triming '/tmp/' from filename
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
search = 'Getting file {0}:{1}'.format(self.user, tmp_file1[5:])
nose.tools.assert_not_equal(re.search(search, err), None)
search = 'File validated'
nose.tools.assert_not_equal(re.search(search, out), None)
search = 'DID ' + tmp_dataset
nose.tools.assert_not_equal(re.search(search, out), None)
def test_detach_files_dataset(self):
"""CLIENT(USER): Rucio detach files to dataset"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3} {4} --did {5}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# detach the files to the dataset
cmd = 'rucio detach --from {0} {1}:{2} {1}:{3}'.format(tmp_dsn, self.user, tmp_file2[5:], tmp_file3[5:]) # triming '/tmp/' from filenames
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
# tmp_file1 must be in the dataset
nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_equal(re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
def test_add_files_to_dataset(self):
"""CLIENT(USER): Rucio add files to dataset"""
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_dataset = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# add files
cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_file2)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# create dataset
cmd = 'rucio add-dataset ' + tmp_dataset
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add files to dataset
cmd = 'rucio add-files-to-dataset --to {0} mock:{1} mock:{2}'.format(tmp_dataset, tmp_file1[5:], tmp_file2[5:]) # triming '/tmp/' from filename
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# find the added files
cmd = 'rucio list-files ' + tmp_dataset
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
def test_upload_repeated_file_dataset(self):
"""CLIENT(USER): Rucio upload repeated files to dataset"""
# One of the files to upload is already in the dataset
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} --files {2} --did {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
# upload the files to the dataset
cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3} {4} --did {5}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
# tmp_file1 must be in the dataset
nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
def test_upload(self):
"""CLIENT(USER): Upload"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
exitcode, out, err = execute(cmd)
cmd = 'rucio upload'
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
def test_list_rses(self):
"""CLIENT(ADMIN): List RSEs"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
exitcode, out, err = execute(cmd)
cmd = 'rucio-admin rse list'
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
nose.tools.assert_regexp_matches(out, re.compile('.*%s.*' % tmp_val))
def test_get_auth_GSS(self):
"""AUTH (CURL): Test auth token retrieval with via gss"""
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token' % (self.cacert, self.usercert, self.auth_host)
exitcode, out, err = execute(cmd)
nose.tools.assert_in('X-Rucio-Auth-Token', out)
os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" --negotiate -u: -X GET %s/auth/gss' % (self.cacert, self.auth_host)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
def test_get_auth_x509_proxy(self):
"""AUTH (CURL): Test auth token retrieval with via proxy"""
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token' % (self.cacert, self.usercert, self.auth_host)
exitcode, out, err = execute(cmd)
nose.tools.assert_in('X-Rucio-Auth-Token', out)
os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: vgaronne" --cert $X509_USER_PROXY --key $X509_USER_PROXY -X GET %s/auth/x509_proxy' % (self.cacert, self.auth_host)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
def test_add_scope(self):
"""CLIENT(ADMIN): Add scope"""
tmp_scp = scope_name_generator()
tmp_acc = account_name_generator()
cmd = 'rucio-admin account add %s' % tmp_acc
exitcode, out, err = execute(cmd)
cmd = 'rucio-admin scope add --account %s --scope %s' % (tmp_acc, tmp_scp)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_equal('Added new scope to account: %s-%s\n' % (tmp_scp, tmp_acc), out)
def test_add_identity(self):
"""CLIENT(ADMIN): Add identity"""
tmp_val = account_name_generator()
cmd = 'rucio-admin account add %s' % tmp_val
exitcode, out, err = execute(cmd)
nose.tools.assert_equal('Added new account: %s\n' % tmp_val, out)
cmd = 'rucio-admin identity add --account %s --type GSS --id [email protected] --email [email protected]' % tmp_val
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
nose.tools.assert_equal('Added new identity to account: [email protected]%s\n' % tmp_val, out)
def test_get_auth_validate(self):
"""AUTH (CURL): Test if token is valid"""
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token:' % (self.cacert, self.usercert, self.auth_host)
exitcode, out, err = execute(cmd)
nose.tools.assert_in('X-Rucio-Auth-Token', out)
os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -X GET %s/auth/validate' % (self.cacert, self.auth_host)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
nose.tools.assert_in('datetime.datetime', out)
def test_post_account(self):
"""ACCOUNT (CURL): add account"""
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token:' % (self.cacert, self.usercert, self.auth_host)
exitcode, out, err = execute(cmd)
nose.tools.assert_in('X-Rucio-Auth-Token', out)
os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
cmd = '''curl -s -i --cacert %s -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -H "Rucio-Type: user" -d '{"type": "USER"}' -X POST %s/accounts/%s''' % (self.cacert, self.host, account_name_generator())
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
nose.tools.assert_in('201 Created', out)
def test_get_accounts_whoami(self):
"""ACCOUNT (CURL): Test whoami method"""
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token:' % (self.cacert, self.usercert, self.auth_host)
exitcode, out, err = execute(cmd)
nose.tools.assert_in('X-Rucio-Auth-Token', out)
os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
cmd = '''curl -s -i -L --cacert %s -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -X GET %s/accounts/whoami''' % (self.cacert, self.host)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
nose.tools.assert_in('303 See Other', out)
def test_post_rse(self):
"""RSE (CURL): add RSE"""
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token:' % (self.cacert, self.usercert, self.auth_host)
exitcode, out, err = execute(cmd)
nose.tools.assert_in('X-Rucio-Auth-Token', out)
os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
cmd = '''curl -s -i --cacert %s -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -X POST %s/rses/%s''' % (self.cacert, self.host, rse_name_generator())
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
nose.tools.assert_in('201 Created', out)
def test_add_file_twice(self):
"""CLIENT(USER): Add file twice"""
tmp_file1 = file_generator()
# add file twice
cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
nose.tools.assert_equal(re.search("File {0}:{1} successfully uploaded on the storage".format(self.user, tmp_file1[5:]), out), None)
def test_create_rule(self):
"""CLIENT(USER): Rucio add rule"""
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add rules
cmd = "rucio add-rule {0}:{1} 3 'spacetoken=ATLASSCRATCHDISK'".format(self.user, tmp_file1[5:])
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
rule = out[:-1] # triming new line character
# check if rule exist for the file
cmd = "rucio list-rules --did {0}:{1}".format(self.user, tmp_file1[5:])
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_not_equal(re.search(rule, out), None)
def test_get_auth_x509(self):
"""AUTH (CURL): Test auth token retrieval with via x509"""
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509' % (self.cacert, self.usercert, self.auth_host)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
nose.tools.assert_in('X-Rucio-Auth-Token', out)
def test_whoami(self):
"""CLIENT(USER): Rucio whoami"""
cmd = 'rucio whoami'
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
nose.tools.assert_regexp_matches(out, re.compile('.*account.*'))
def test_get_auth_userpass(self):
"""AUTH (CURL): Test auth token retrieval with via username and password"""
cmd = 'curl -s -i --cacert %s -X GET -H "X-Rucio-Account: root" -H "X-Rucio-Username: ddmlab" -H "X-Rucio-Password: secret" %s/auth/userpass' % (self.cacert, self.auth_host)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
nose.tools.assert_in('X-Rucio-Auth-Token', out)
def test_add_account(self):
"""CLIENT(ADMIN): Add account"""
tmp_val = account_name_generator()
cmd = 'rucio-admin account add %s' % tmp_val
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
nose.tools.assert_equal('Added new account: %s\n' % tmp_val, out)
def test_add_rse(self):
"""CLIENT(ADMIN): Add RSE"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add %s' % tmp_val
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
nose.tools.assert_equal('Added new RSE: %s\n' % tmp_val, out)
def test_create_dataset(self):
"""CLIENT(USER): Rucio add dataset"""
tmp_name = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
cmd = 'rucio add-dataset ' + tmp_name
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_not_equal(re.search('Added ' + tmp_name, out), None)
def test_list_did_recursive(self):
""" CLIENT(USER): List did recursive """
# Setup nested collections
tmp_scope = 'mock'
tmp_container_1 = 'container_%s' % generate_uuid()
cmd = 'rucio add-container {0}:{1}'.format(tmp_scope, tmp_container_1)
exitcode, out, err = execute(cmd)
tmp_container_2 = 'container_%s' % generate_uuid()
cmd = 'rucio add-container {0}:{1}'.format(tmp_scope, tmp_container_2)
exitcode, out, err = execute(cmd)
tmp_container_3 = 'container_%s' % generate_uuid()
cmd = 'rucio add-container {0}:{1}'.format(tmp_scope, tmp_container_3)
exitcode, out, err = execute(cmd)
cmd = 'rucio attach {0}:{1} {0}:{2}'.format(tmp_scope, tmp_container_1,
tmp_container_2)
exitcode, out, err = execute(cmd)
cmd = 'rucio attach {0}:{1} {0}:{2}'.format(tmp_scope, tmp_container_2,
tmp_container_3)
exitcode, out, err = execute(cmd)
# All attached DIDs are expected
cmd = 'rucio list-dids {0}:{1} --recursive'.format(
tmp_scope, tmp_container_1)
exitcode, out, err = execute(cmd)
nose.tools.assert_not_equal(re.search(tmp_container_1, out), None)
nose.tools.assert_not_equal(re.search(tmp_container_2, out), None)
nose.tools.assert_not_equal(re.search(tmp_container_3, out), None)
# Wildcards are not allowed to use with --recursive
cmd = 'rucio list-dids {0}:* --recursive'.format(tmp_scope)
exitcode, out, err = execute(cmd)
nose.tools.assert_not_equal(
re.search("Option recursive cannot be used with wildcards", err),
None)
def test_attach_files_dataset(self):
"""CLIENT(USER): Rucio attach files to dataset"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
# upload the files
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file2, tmp_file3)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# attach the files to the dataset
cmd = 'rucio attach {0} {1}:{2} {1}:{3}'.format(
tmp_dsn, self.user, tmp_file2[5:],
tmp_file3[5:]) # triming '/tmp/' from filenames
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
# tmp_file2 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file2[5:]), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
def test_detach_non_existing_file(self):
"""CLIENT(USER): Rucio detach a non existing file"""
tmp_file1 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
# attach the files to the dataset
cmd = 'rucio detach {0} {1}:{2}'.format(tmp_dsn, self.user, 'file_ghost') # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(re.search("Data identifier not found.", err), None)
def test_detach_non_existing_file(self):
"""CLIENT(USER): Rucio detach a non existing file"""
tmp_file1 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} --files {2} --did {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
remove(tmp_file1)
# attach the files to the dataset
cmd = 'rucio detach --from {0} {1}:{2}'.format(tmp_dsn, self.user, 'file_ghost') # triming '/tmp/' from filenames
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
nose.tools.assert_not_equal(re.search("Failed to detach data identifier", err), None)
def test_get_auth_x509(self):
"""AUTH (CURL): Test auth token retrieval with via x509"""
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" %s -cert %s --key %s -X GET %s/auth/x509' % (
self.cacert, self.vo_header, self.usercert, self.userkey,
self.auth_host)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
assert 'X-Rucio-Auth-Token' in out
def test_ping(self):
"""PING (CURL): Get Version"""
cmd = 'curl --cacert %s -s -X GET %s/ping' % (self.cacert, self.host)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
ret = json.loads(out)
nose.tools.assert_true('version' in ret)
nose.tools.assert_is_instance(ret, dict)
def test_get_accounts_whoami(self):
"""ACCOUNT (CURL): Test whoami method"""
cmd = 'curl -s -d -i -k -L --verbose --cacert /etc/grid-security/certificates/CERN-Root-2.pem -H "X-Rucio-Account: root" -E /etc/grid-security/hostcert.pem --key /etc/grid-security/hostkey.pem --key-type PEM --ignore-content-length -X GET https://chenghsi-rucio-server2.cern.ch:443/auth/x509'
exitcode, out, err = execute(cmd)
#nose.tools.assert_in('X-Rucio-Auth-Token', out)
os.environ['RUCIO_TOKEN'] = err[len('X-Rucio-Auth-Token: '):-1]
#print os.environ['RUCIO_TOKEN']
print err[err.find('X-Rucio-Auth-Token'):]
nose.tools.assert_in('X-Rucio-Auth-Token', err)
def test_replica_http_redirection(self):
""" REPLICA (redirection): http redirection to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" -X GET %s/redirect/%s/%s''' % (self.cacert, self.token, self.host, tmp_scope, tmp_name)
exitcode, out, err = execute(cmd)
assert_in('404 Not Found', out)
# add replicas
self.replica_client.add_replicas(rse='MOCK', files=[{'scope': tmp_scope, 'name': tmp_name, 'bytes': 1L, 'adler32': '0cc737eb'}])
def test_create_dataset(self):
"""CLIENT(USER): Rucio add dataset"""
tmp_name = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
cmd = 'rucio add-dataset ' + tmp_name
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search('Added ' + tmp_name, out), None)
def test_ping(self):
"""PING (CURL): Get Version"""
cmd = 'curl --cacert %s -s -X GET %s/ping' % (self.cacert, self.host)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out,
ret = json.loads(out)
nose.tools.assert_true('version' in ret)
nose.tools.assert_is_instance(ret, dict)
def test_ping(self):
"""PING (CURL): Get Version"""
cmd = 'curl --cacert %s -s -X GET %s/ping' % (self.cacert, self.host)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
ret = json.loads(out)
assert 'version' in ret
assert isinstance(ret, dict)
def test_add_rse_nondet(self):
"""CLIENT(ADMIN): Add non-deterministic RSE"""
tmp_val = rse_name_generator()
cmd = 'rucio-admin rse add --non-deterministic %s' % tmp_val
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, )
nose.tools.assert_equal(
'Added new non-deterministic RSE: %s\n' % tmp_val, out)
def test_delete_rule(self):
"""CLIENT(USER): rule deletion"""
set_account_limit('root', get_rse_id(self.def_rse), -1)
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rse
tmp_rse = rse_name_generator()
cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
set_account_limit('root', get_rse_id(tmp_rse), -1)
# add rse atributes
cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
tmp_rse)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add rules
cmd = "rucio add-rule {0}:{1} 1 'spacetoken=ATLASSCRATCHDISK'".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(err)
print(out)
# get the rules for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
(rule1, rule2) = out.split()
# delete the rules for the file
cmd = "rucio delete-rule {0}".format(rule1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = "rucio delete-rule {0}".format(rule2)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the file
cmd = "rucio list-dids {0}:{1}".format(self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(5, len(out.splitlines()))
def test_download_file(self):
"""CLIENT(USER): Rucio download files"""
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# download files
cmd = 'rucio download --dir /tmp {0}:{1}'.format(self.user, tmp_file1[5:]) # triming '/tmp/' from filename
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# search for the files with ls
cmd = 'ls /tmp/{0}'.format(self.user) # search in /tmp/mock
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
def test_attach_file_twice(self):
"""CLIENT(USER): Rucio attach a file twice"""
# Attach files to a dataset using the attach method
tmp_file1 = file_generator()
tmp_dsn = self.user + ':DSet' + rse_name_generator() # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
# attach the files to the dataset
cmd = 'rucio attach {0} {1}:{2}'.format(tmp_dsn, self.user, tmp_file1[5:]) # triming '/tmp/' from filenames
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(re.search("The file already exists", err), None)
def test_upload_repeated_file(self):
"""CLIENT(USER): Rucio upload repeated files"""
# One of the files to upload is already catalogued but was removed
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# get the rule for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
rule = out
# delete the file from the catalog
cmd = "rucio delete-rule {0}".format(rule)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# delete the fisical file
cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = 'rucio upload --rse {0} --scope {1} {2} {3} {4}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
nose.tools.assert_not_equal(
re.search(
"File {0}:{1} successfully uploaded on the storage".format(
self.user, tmp_file2[5:]), out), None)
def test_upload_file_guid(self):
"""CLIENT(USER): Rucio upload file with guid"""
tmp_file1 = file_generator()
tmp_guid = generate_uuid()
cmd = 'rucio -v upload --rse {0} --guid {1} --scope {2} {3}'.format(self.def_rse, tmp_guid, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file1)) in out)
def test_upload_repeated_file(self):
"""CLIENT(USER): Rucio upload repeated files"""
# One of the files to upload is already catalogued but was removed
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_file1_name = path.basename(tmp_file1)
cmd = 'rucio -v upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# get the rule for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
self.user, tmp_file1_name)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
rule = out
# delete the file from the catalog
cmd = "rucio delete-rule {0}".format(rule)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# delete the physical file
cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1_name)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
nose.tools.assert_true((self.upload_success_str %
tmp_file1_name) in out)
def test_add_delete_add_file(self):
"""CLIENT(USER): Add/Delete/Add"""
tmp_file1 = file_generator()
# add file
cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# get the rule for the file
cmd = "rucio list-rules --did {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(self.user, tmp_file1[5:])
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
rule = out
# delete the file from the catalog
cmd = "rucio delete-rule {0}".format(rule)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# delete the fisical file
cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1[5:])
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# modify the file to avoid same checksum
cmd = "echo 'delta' >> {0}".format(tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
# add the same file
cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_equal(re.search("File {0}:{1} successfully uploaded on the storage".format(self.user, tmp_file1[5:]), out), None)
def test_download_fails_badmd5(self):
"""CLIENT(USER): Rucio download fails on MD5 mismatch"""
# user has a file to upload
filename = file_generator()
file_md5 = md5(filename)
filesize = stat(filename).st_size
lfn = {'name': filename[5:], 'scope': self.user, 'bytes': filesize, 'md5': '0123456789abcdef0123456789abcdef'}
# user uploads file
self.replica_client.add_replicas(files=[lfn], rse=self.def_rse)
rse_settings = rsemgr.get_rse_info(self.def_rse)
protocol = rsemgr.create_protocol(rse_settings, 'write')
protocol.connect()
pfn = protocol.lfns2pfns(lfn).values()[0]
protocol.put(filename[5:], pfn, filename[:5])
protocol.close()
remove(filename)
# download file
cmd = 'rucio -v download --dir /tmp {0}:{1}'.format(self.user, filename[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
report = 'Local\ checksum\:\ {0},\ Rucio\ checksum\:\ 0123456789abcdef0123456789abcdef'.format(file_md5)
print('searching', report, 'in', err)
nose.tools.assert_not_equal(re.search(report, err), None)
# The file should not exist
cmd = 'ls /tmp/' # search in /tmp/
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(re.search(filename[5:], out), None)
try:
for i in listdir('data13_hip'):
unlink('data13_hip/%s' % i)
rmdir('data13_hip')
except Exception:
pass
def test_add_delete_add_file(self):
"""CLIENT(USER): Add/Delete/Add"""
tmp_file1 = file_generator()
# add file
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# get the rule for the file
cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(self.user, tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
rule = out
# delete the file from the catalog
cmd = "rucio delete-rule {0}".format(rule)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# delete the fisical file
cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# modify the file to avoid same checksum
cmd = "echo 'delta' >> {0}".format(tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# add the same file
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal(re.search("File {0}:{1} successfully uploaded on the storage".format(self.user, tmp_file1[5:]), out), None)
def test_download_succeeds_md5only(self):
"""CLIENT(USER): Rucio download succeeds MD5 only"""
# user has a file to upload
filename = file_generator()
file_md5 = md5(filename)
filesize = stat(filename).st_size
lfn = {
'name': filename[5:],
'scope': self.user,
'bytes': filesize,
'md5': file_md5
}
# user uploads file
self.replica_client.add_replicas(files=[lfn], rse=self.def_rse)
rse_settings = rsemgr.get_rse_info(self.def_rse)
protocol = rsemgr.create_protocol(rse_settings, 'write')
protocol.connect()
pfn = protocol.lfns2pfns(lfn).values()[0]
protocol.put(filename[5:], pfn, filename[:5])
protocol.close()
remove(filename)
# download files
cmd = 'rucio download --dir /tmp {0}:{1}'.format(
self.user, filename[5:])
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the files with ls
cmd = 'ls /tmp/{0}'.format(self.user) # search in /tmp/
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(filename[5:], out), None)
try:
for i in listdir('data13_hip'):
unlink('data13_hip/%s' % i)
rmdir('data13_hip')
except Exception:
pass
def test_upload_repeated_file_dataset(self):
"""CLIENT(USER): Rucio upload repeated files to dataset"""
# One of the files to upload is already in the dataset
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
tmp_file1_name = path.basename(tmp_file1)
tmp_file3_name = path.basename(tmp_file3)
tmp_dsn = self.user + ':DSet' + rse_name_generator(
) # something like mock:DSetMOCK_S0M37HING
# Adding files to a new dataset
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3}'.format(
self.def_rse, self.user, tmp_file1, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# upload the files to the dataset
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
# searching for the file in the new dataset
cmd = 'rucio list-files {0}'.format(tmp_dsn)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# tmp_file1 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file1_name), out), None)
# tmp_file3 must be in the dataset
nose.tools.assert_not_equal(
re.search("{0}:{1}".format(self.user, tmp_file3_name), out), None)
def test_replica_meta_redirection(self):
""" REDIRECT: metalink to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s' '' % (
self.cacert, self.token, self.vo_header, self.host, tmp_scope,
tmp_name)
_, out, _ = execute(cmd)
assert_in('404 Not Found', out)
self.replica_client.add_replicas(rse='MOCK',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'
}])
self.replica_client.add_replicas(rse='MOCK3',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'
}])
_, out, _ = execute(cmd)
assert_in('303 See Other', out)
assert_in('Link: </redirect/%s/%s/metalink' % (tmp_scope, tmp_name),
out)
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s/metalink' '' % (
self.cacert, self.token, self.vo_header, self.host, tmp_scope,
tmp_name)
_, out, _ = execute(cmd)
assert_in('200 OK', out)
assert_in('<?xml', out)
assert_in('<metalink', out)
assert_in('<url location="MOCK"', out)
assert_in('<url location="MOCK3"', out)
def test_del_identity(self):
"""CLIENT(ADMIN): Test del identity"""
tmp_acc = account_name_generator()
# create account
cmd = 'rucio-admin account add %s' % tmp_acc
exitcode, out, err = execute(cmd)
# add identity to account
cmd = 'rucio-admin identity add --account %s --type GSS --id [email protected] --email [email protected]' % tmp_acc
exitcode, out, err = execute(cmd)
# delete identity from account
cmd = 'rucio-admin identity delete --account %s --type GSS --id [email protected]' % tmp_acc
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_equal('Deleted identity: [email protected]\n', out)
# list identities for account
cmd = 'rucio-admin account list-identities %s' % (tmp_acc)
print self.marker + cmd
print cmd
exitcode, out, err = execute(cmd)
print out, err
nose.tools.assert_equal('', out)
def test_upload_file_guid(self):
"""CLIENT(USER): Rucio upload file with guid"""
tmp_file1 = file_generator()
tmp_guid = generate_uuid()
cmd = 'rucio upload --rse {0} --guid {1} --scope {2} {3}'.format(
self.def_rse, tmp_guid, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
nose.tools.assert_true(
"File %s:%s successfully uploaded on the storage" %
(self.user, tmp_file1[5:]) in out)
def test_del_identity(self):
"""CLIENT(ADMIN): Test del identity"""
tmp_acc = account_name_generator()
# create account
cmd = 'rucio-admin account add %s' % tmp_acc
exitcode, out, err = execute(cmd)
# add identity to account
cmd = 'rucio-admin identity add --account %s --type GSS --id [email protected] --email [email protected]' % tmp_acc
exitcode, out, err = execute(cmd)
# delete identity from account
cmd = 'rucio-admin identity delete --account %s --type GSS --id [email protected]' % tmp_acc
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal('Deleted identity: [email protected]\n', out)
# list identities for account
cmd = 'rucio-admin account list-identities %s' % (tmp_acc)
print(self.marker + cmd)
print(cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_equal('', out)
def test_upload_file(self):
"""CLIENT(USER): Rucio upload files"""
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3} {4}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
nose.tools.assert_not_equal(re.search("File {0}:{1} successfully uploaded on the storage".format(self.user, tmp_file1[5:]), out), None)
def test_attach_dataset_twice(self):
""" CLIENT(USER): Rucio attach a dataset twice """
container = 'container_%s' % generate_uuid()
dataset = 'dataset_%s' % generate_uuid()
self.did_client.add_container(scope=self.user, name=container)
self.did_client.add_dataset(scope=self.user, name=dataset)
# Attach dataset to container
cmd = 'rucio attach {0}:{1} {0}:{2}'.format(self.user, container,
dataset)
exitcode, out, err = execute(cmd)
# Attach again
cmd = 'rucio attach {0}:{1} {0}:{2}'.format(self.user, container,
dataset)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
nose.tools.assert_not_equal(
re.search(
"Data identifier already added to the destination content",
err), None)
def test_downgrade_and_upgrade(self):
""" ALEMBIC (CORE): Test the schema migration """
cmd = 'alembic upgrade head'
exitcode, out, err = execute(cmd)
print(cmd)
print(exitcode, out, err)
assert_equal(exitcode, 0)
cmd = 'alembic downgrade -1'
while exitcode is 0:
exitcode, out, err = execute(cmd)
print(cmd)
print(exitcode, out, err)
if "Relative revision -1 didn't produce 1 migrations" not in out:
assert_equal(exitcode, 0)
else:
assert_equal(exitcode, 255)
cmd = 'alembic upgrade head'
exitcode, out, err = execute(cmd)
print(cmd)
print(exitcode, out, err)
assert_equal(exitcode, 0)
def test_replica_header_redirection(self):
""" REDIRECT: header to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s''' % (self.cacert,
self.token,
self.vo_header,
self.host,
tmp_scope,
tmp_name)
_, out, _ = execute(cmd)
assert '404 Not Found' in out
self.replica_client.add_replicas(rse='MOCK', files=[{'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'}])
self.replica_client.add_replicas(rse='MOCK3', files=[{'scope': tmp_scope,
'name': tmp_name,
'bytes': 1,
'adler32': '0cc737eb'}])
_, out, _ = execute(cmd)
assert '303 See Other' in out
assert 'Location: https://mock' in out
def test_upload_file_guid(self):
"""CLIENT(USER): Rucio upload file with guid"""
tmp_file1 = file_generator()
tmp_guid = generate_uuid()
cmd = 'rucio upload --rse {0} --guid {1} --scope {2} {3}'.format(
self.def_rse, tmp_guid, self.user, tmp_file1)
print self.marker + cmd
exitcode, out, err = execute(cmd)
print out
print err
remove(tmp_file1)
nose.tools.assert_not_equal(
re.search(
"File {0}:{1} successfully uploaded on the storage".format(
self.user, tmp_file1[5:]), out), None)
def test_replica_meta_redirection(self):
""" REDIRECT: metalink to replica"""
tmp_scope = 'mock'
tmp_name = 'file_%s' % generate_uuid()
cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" -X GET %s/redirect/%s/%s' '' % (
self.cacert, self.token, self.host, tmp_scope, tmp_name)
_, out, _ = execute(cmd)
assert_in('404 Not Found', out)
# add replicas
self.replica_client.add_replicas(rse='MOCK',
files=[{
'scope': tmp_scope,
'name': tmp_name,
'bytes': 1L,
'adler32': '0cc737eb'
}])
def test_upload_file(self):
"""CLIENT(USER): Rucio upload files"""
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file1)) in out)
nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file2)) in out)
nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file3)) in out)
def test_upload_adds_md5digest(self):
"""CLIENT(USER): Upload Checksums"""
# user has a file to upload
filename = file_generator()
file_md5 = md5(filename)
# user uploads file
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, filename)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
# When inspecting the metadata of the new file the user finds the md5 checksum
meta = self.did_client.get_metadata(scope=self.user, name=filename[5:])
nose.tools.assert_in('md5', meta)
nose.tools.assert_equal(meta['md5'], file_md5)
remove(filename)
def test_upload_file(self):
"""CLIENT(USER): Rucio upload files"""
tmp_file1 = file_generator()
tmp_file2 = file_generator()
tmp_file3 = file_generator()
cmd = 'rucio upload --rse {0} --scope {1} {2} {3} {4}'.format(
self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out)
print(err)
remove(tmp_file1)
remove(tmp_file2)
remove(tmp_file3)
nose.tools.assert_true(
"File %s:%s successfully uploaded on the storage" %
(self.user, tmp_file1[5:]) in out)
def test_download_item(self):
""" DOWNLOAD (CLIENT): download DIDs. """
download_client = DownloadClient()
tmp_file1 = file_generator()
scope = 'mock'
name = tmp_file1[5:]
uuid = generate_uuid()
cmd = 'rucio upload --rse {0} --scope {1} --guid {2} {3}'.format(
'MOCK4', scope, uuid, tmp_file1)
exitcode, out, err = execute(cmd)
# Download specific DID
result = download_client.download_dids([{
'did': '%s:%s' % (scope, name)
}])
assert_true(result)
# Download with wildcard
result = download_client.download_dids([{
'did':
'%s:%s' % (scope, name[:10] + '*')
}])
assert_true(result)
# Download with filter
result = download_client.download_dids([{
'filters': {
'guid': uuid,
'scope': scope
}
}])
assert_true(result)
# Download with wildcard and name
result = download_client.download_dids([{
'did': '%s:%s' % (scope, '*'),
'filters': {
'guid': uuid
}
}])
assert_true(result)
def test_download_file(self):
"""CLIENT(USER): Rucio download files"""
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# download files
cmd = 'rucio -v download --dir /tmp {0}:{1}'.format(
self.user, tmp_file1[5:]) # triming '/tmp/' from filename
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the files with ls
cmd = 'ls /tmp/' # search in /tmp/
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
tmp_file1 = file_generator()
# add files
cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
self.def_rse, self.user, tmp_file1)
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# download files
cmd = 'rucio -v download --dir /tmp {0}:{1}'.format(
self.user, tmp_file1[5:-2] + '*') # triming '/tmp/' from filename
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
# search for the files with ls
cmd = 'ls /tmp/' # search in /tmp/
print(self.marker + cmd)
exitcode, out, err = execute(cmd)
print(out, err)
nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
try:
for i in listdir('data13_hip'):
unlink('data13_hip/%s' % i)
rmdir('data13_hip')
except Exception:
pass
