Exemple #1
0
 def test_download_dataset(self):
     """CLIENT(USER): Rucio download dataset"""
     tmp_file1 = file_generator()
     tmp_dataset = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # add files
     cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # create dataset
     cmd = 'rucio add-dataset ' + tmp_dataset
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add files to dataset
     cmd = 'rucio add-files-to-dataset --to {0} {1}:{2}'.format(tmp_dataset, self.user, tmp_file1[5:])  # triming '/tmp/' from filename
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # download dataset
     cmd = 'rucio download --dir /tmp {0}'.format(tmp_dataset)  # triming '/tmp/' from filename
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     search = 'Getting file {0}:{1}'.format(self.user, tmp_file1[5:])
     nose.tools.assert_not_equal(re.search(search, err), None)
     search = 'File validated'
     nose.tools.assert_not_equal(re.search(search, out), None)
     search = 'DID ' + tmp_dataset
     nose.tools.assert_not_equal(re.search(search, out), None)
Exemple #2
0
 def test_detach_files_dataset(self):
     """CLIENT(USER): Rucio detach files to dataset"""
     # Attach files to a dataset using the attach method
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3} {4} --did {5}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     # detach the files to the dataset
     cmd = 'rucio detach --from {0} {1}:{2} {1}:{3}'.format(tmp_dsn, self.user, tmp_file2[5:], tmp_file3[5:])  # triming '/tmp/' from filenames
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     # searching for the file in the new dataset
     cmd = 'rucio list-files {0}'.format(tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     # tmp_file1 must be in the dataset
     nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
     # tmp_file3 must be in the dataset
     nose.tools.assert_equal(re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
Exemple #3
0
 def test_add_files_to_dataset(self):
     """CLIENT(USER): Rucio add files to dataset"""
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_dataset = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # add files
     cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_file2)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # create dataset
     cmd = 'rucio add-dataset ' + tmp_dataset
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add files to dataset
     cmd = 'rucio add-files-to-dataset --to {0} mock:{1} mock:{2}'.format(tmp_dataset, tmp_file1[5:], tmp_file2[5:])  # triming '/tmp/' from filename
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # find the added files
     cmd = 'rucio list-files ' + tmp_dataset
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
Exemple #4
0
 def test_upload_repeated_file_dataset(self):
     """CLIENT(USER): Rucio upload repeated files to dataset"""
     # One of the files to upload is already in the dataset
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} --files {2} --did {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     # upload the files to the dataset
     cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3} {4} --did {5}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     # searching for the file in the new dataset
     cmd = 'rucio list-files {0}'.format(tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     # tmp_file1 must be in the dataset
     nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
     # tmp_file3 must be in the dataset
     nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
Exemple #5
0
 def test_upload(self):
     """CLIENT(USER): Upload"""
     tmp_val = rse_name_generator()
     cmd = 'rucio-admin rse add %s' % tmp_val
     exitcode, out, err = execute(cmd)
     cmd = 'rucio upload'
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
Exemple #6
0
 def test_list_rses(self):
     """CLIENT(ADMIN): List RSEs"""
     tmp_val = rse_name_generator()
     cmd = 'rucio-admin rse add %s' % tmp_val
     exitcode, out, err = execute(cmd)
     cmd = 'rucio-admin rse list'
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
     nose.tools.assert_regexp_matches(out, re.compile('.*%s.*' % tmp_val))
Exemple #7
0
 def test_get_auth_GSS(self):
     """AUTH (CURL): Test auth token retrieval with via gss"""
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token' % (self.cacert, self.usercert, self.auth_host)
     exitcode, out, err = execute(cmd)
     nose.tools.assert_in('X-Rucio-Auth-Token', out)
     os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" --negotiate -u: -X GET %s/auth/gss' % (self.cacert, self.auth_host)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
Exemple #8
0
 def test_get_auth_x509_proxy(self):
     """AUTH (CURL): Test auth token retrieval with via proxy"""
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token' % (self.cacert, self.usercert, self.auth_host)
     exitcode, out, err = execute(cmd)
     nose.tools.assert_in('X-Rucio-Auth-Token', out)
     os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: vgaronne" --cert $X509_USER_PROXY --key $X509_USER_PROXY -X GET %s/auth/x509_proxy' % (self.cacert, self.auth_host)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
Exemple #9
0
 def test_add_scope(self):
     """CLIENT(ADMIN): Add scope"""
     tmp_scp = scope_name_generator()
     tmp_acc = account_name_generator()
     cmd = 'rucio-admin account add %s' % tmp_acc
     exitcode, out, err = execute(cmd)
     cmd = 'rucio-admin scope add --account %s --scope %s' % (tmp_acc, tmp_scp)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     nose.tools.assert_equal('Added new scope to account: %s-%s\n' % (tmp_scp, tmp_acc), out)
Exemple #10
0
 def test_add_identity(self):
     """CLIENT(ADMIN): Add identity"""
     tmp_val = account_name_generator()
     cmd = 'rucio-admin account add %s' % tmp_val
     exitcode, out, err = execute(cmd)
     nose.tools.assert_equal('Added new account: %s\n' % tmp_val, out)
     cmd = 'rucio-admin identity add --account %s --type GSS --id [email protected] --email [email protected]' % tmp_val
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
     nose.tools.assert_equal('Added new identity to account: [email protected]%s\n' % tmp_val, out)
Exemple #11
0
 def test_get_auth_validate(self):
     """AUTH (CURL): Test if token is valid"""
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token:' % (self.cacert, self.usercert, self.auth_host)
     exitcode, out, err = execute(cmd)
     nose.tools.assert_in('X-Rucio-Auth-Token', out)
     os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
     cmd = 'curl -s -i --cacert %s  -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -X GET %s/auth/validate' % (self.cacert, self.auth_host)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     nose.tools.assert_in('datetime.datetime', out)
Exemple #12
0
 def test_post_account(self):
     """ACCOUNT (CURL): add account"""
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token:' % (self.cacert, self.usercert, self.auth_host)
     exitcode, out, err = execute(cmd)
     nose.tools.assert_in('X-Rucio-Auth-Token', out)
     os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
     cmd = '''curl -s -i --cacert %s -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -H "Rucio-Type: user" -d '{"type": "USER"}' -X POST %s/accounts/%s''' % (self.cacert, self.host, account_name_generator())
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     nose.tools.assert_in('201 Created', out)
Exemple #13
0
 def test_get_accounts_whoami(self):
     """ACCOUNT (CURL): Test whoami method"""
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token:' % (self.cacert, self.usercert, self.auth_host)
     exitcode, out, err = execute(cmd)
     nose.tools.assert_in('X-Rucio-Auth-Token', out)
     os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
     cmd = '''curl -s -i -L --cacert %s -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -X GET %s/accounts/whoami''' % (self.cacert, self.host)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     nose.tools.assert_in('303 See Other', out)
Exemple #14
0
 def test_post_rse(self):
     """RSE (CURL): add RSE"""
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token:' % (self.cacert, self.usercert, self.auth_host)
     exitcode, out, err = execute(cmd)
     nose.tools.assert_in('X-Rucio-Auth-Token', out)
     os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
     cmd = '''curl -s -i --cacert %s -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -X POST %s/rses/%s''' % (self.cacert, self.host, rse_name_generator())
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     nose.tools.assert_in('201 Created', out)
Exemple #15
0
 def test_add_file_twice(self):
     """CLIENT(USER): Add file twice"""
     tmp_file1 = file_generator()
     # add file twice
     cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     nose.tools.assert_equal(re.search("File {0}:{1} successfully uploaded on the storage".format(self.user, tmp_file1[5:]), out), None)
Exemple #16
0
 def test_create_rule(self):
     """CLIENT(USER): Rucio add rule"""
     tmp_file1 = file_generator()
     # add files
     cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rse
     tmp_rse = rse_name_generator()
     cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     # add rse atributes
     cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rse
     tmp_rse = rse_name_generator()
     cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rse atributes
     cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rse
     tmp_rse = rse_name_generator()
     cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rse atributes
     cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rules
     cmd = "rucio add-rule {0}:{1} 3 'spacetoken=ATLASSCRATCHDISK'".format(self.user, tmp_file1[5:])
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     rule = out[:-1]  # triming new line character
     # check if rule exist for the file
     cmd = "rucio list-rules --did {0}:{1}".format(self.user, tmp_file1[5:])
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     nose.tools.assert_not_equal(re.search(rule, out), None)
Exemple #17
0
 def test_get_auth_x509(self):
     """AUTH  (CURL): Test auth token retrieval with via x509"""
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509' % (self.cacert, self.usercert, self.auth_host)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
     nose.tools.assert_in('X-Rucio-Auth-Token', out)
Exemple #18
0
 def test_whoami(self):
     """CLIENT(USER): Rucio whoami"""
     cmd = 'rucio whoami'
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
     nose.tools.assert_regexp_matches(out, re.compile('.*account.*'))
Exemple #19
0
 def test_get_auth_userpass(self):
     """AUTH (CURL): Test auth token retrieval with via username and password"""
     cmd = 'curl -s -i --cacert %s -X GET -H "X-Rucio-Account: root" -H "X-Rucio-Username: ddmlab" -H "X-Rucio-Password: secret" %s/auth/userpass' % (self.cacert, self.auth_host)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
     nose.tools.assert_in('X-Rucio-Auth-Token', out)
Exemple #20
0
 def test_add_account(self):
     """CLIENT(ADMIN): Add account"""
     tmp_val = account_name_generator()
     cmd = 'rucio-admin account add %s' % tmp_val
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
     nose.tools.assert_equal('Added new account: %s\n' % tmp_val, out)
Exemple #21
0
 def test_add_rse(self):
     """CLIENT(ADMIN): Add RSE"""
     tmp_val = rse_name_generator()
     cmd = 'rucio-admin rse add %s' % tmp_val
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
     nose.tools.assert_equal('Added new RSE: %s\n' % tmp_val, out)
Exemple #22
0
 def test_create_dataset(self):
     """CLIENT(USER): Rucio add dataset"""
     tmp_name = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     cmd = 'rucio add-dataset ' + tmp_name
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     nose.tools.assert_not_equal(re.search('Added ' + tmp_name, out), None)
Exemple #23
0
    def test_list_did_recursive(self):
        """ CLIENT(USER): List did recursive """
        # Setup nested collections
        tmp_scope = 'mock'
        tmp_container_1 = 'container_%s' % generate_uuid()
        cmd = 'rucio add-container {0}:{1}'.format(tmp_scope, tmp_container_1)
        exitcode, out, err = execute(cmd)
        tmp_container_2 = 'container_%s' % generate_uuid()
        cmd = 'rucio add-container {0}:{1}'.format(tmp_scope, tmp_container_2)
        exitcode, out, err = execute(cmd)
        tmp_container_3 = 'container_%s' % generate_uuid()
        cmd = 'rucio add-container {0}:{1}'.format(tmp_scope, tmp_container_3)
        exitcode, out, err = execute(cmd)
        cmd = 'rucio attach {0}:{1} {0}:{2}'.format(tmp_scope, tmp_container_1,
                                                    tmp_container_2)
        exitcode, out, err = execute(cmd)
        cmd = 'rucio attach {0}:{1} {0}:{2}'.format(tmp_scope, tmp_container_2,
                                                    tmp_container_3)
        exitcode, out, err = execute(cmd)

        # All attached DIDs are expected
        cmd = 'rucio list-dids {0}:{1} --recursive'.format(
            tmp_scope, tmp_container_1)
        exitcode, out, err = execute(cmd)
        nose.tools.assert_not_equal(re.search(tmp_container_1, out), None)
        nose.tools.assert_not_equal(re.search(tmp_container_2, out), None)
        nose.tools.assert_not_equal(re.search(tmp_container_3, out), None)

        # Wildcards are not allowed to use with --recursive
        cmd = 'rucio list-dids {0}:* --recursive'.format(tmp_scope)
        exitcode, out, err = execute(cmd)
        nose.tools.assert_not_equal(
            re.search("Option recursive cannot be used with wildcards", err),
            None)
Exemple #24
0
 def test_attach_files_dataset(self):
     """CLIENT(USER): Rucio attach files to dataset"""
     # Attach files to a dataset using the attach method
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator(
     )  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
         self.def_rse, self.user, tmp_file1, tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     # upload the files
     cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
         self.def_rse, self.user, tmp_file2, tmp_file3)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     # attach the files to the dataset
     cmd = 'rucio attach {0} {1}:{2} {1}:{3}'.format(
         tmp_dsn, self.user, tmp_file2[5:],
         tmp_file3[5:])  # triming '/tmp/' from filenames
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     # searching for the file in the new dataset
     cmd = 'rucio list-files {0}'.format(tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     # tmp_file2 must be in the dataset
     nose.tools.assert_not_equal(
         re.search("{0}:{1}".format(self.user, tmp_file2[5:]), out), None)
     # tmp_file3 must be in the dataset
     nose.tools.assert_not_equal(
         re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
Exemple #25
0
 def test_detach_non_existing_file(self):
     """CLIENT(USER): Rucio detach a non existing file"""
     tmp_file1 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     # attach the files to the dataset
     cmd = 'rucio detach {0} {1}:{2}'.format(tmp_dsn, self.user, 'file_ghost')  # triming '/tmp/' from filenames
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     nose.tools.assert_not_equal(re.search("Data identifier not found.", err), None)
Exemple #26
0
 def test_detach_non_existing_file(self):
     """CLIENT(USER): Rucio detach a non existing file"""
     tmp_file1 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} --files {2} --did {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     remove(tmp_file1)
     # attach the files to the dataset
     cmd = 'rucio detach --from {0} {1}:{2}'.format(tmp_dsn, self.user, 'file_ghost')  # triming '/tmp/' from filenames
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     nose.tools.assert_not_equal(re.search("Failed to detach data identifier", err), None)
Exemple #27
0
 def test_get_auth_x509(self):
     """AUTH (CURL): Test auth token retrieval with via x509"""
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" %s -cert %s --key %s -X GET %s/auth/x509' % (
         self.cacert, self.vo_header, self.usercert, self.userkey,
         self.auth_host)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, )
     assert 'X-Rucio-Auth-Token' in out
Exemple #28
0
 def test_ping(self):
     """PING (CURL): Get Version"""
     cmd = 'curl --cacert %s -s -X GET %s/ping' % (self.cacert, self.host)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
     ret = json.loads(out)
     nose.tools.assert_true('version' in ret)
     nose.tools.assert_is_instance(ret, dict)
Exemple #29
0
 def test_get_accounts_whoami(self):
     """ACCOUNT (CURL): Test whoami method"""
     cmd = 'curl -s -d -i -k -L --verbose --cacert /etc/grid-security/certificates/CERN-Root-2.pem -H "X-Rucio-Account: root" -E /etc/grid-security/hostcert.pem --key /etc/grid-security/hostkey.pem --key-type PEM --ignore-content-length -X GET https://chenghsi-rucio-server2.cern.ch:443/auth/x509' 
     exitcode, out, err = execute(cmd)
     #nose.tools.assert_in('X-Rucio-Auth-Token', out)
     os.environ['RUCIO_TOKEN'] = err[len('X-Rucio-Auth-Token: '):-1]
     #print os.environ['RUCIO_TOKEN']
     print err[err.find('X-Rucio-Auth-Token'):]
     nose.tools.assert_in('X-Rucio-Auth-Token', err)
Exemple #30
0
 def test_replica_http_redirection(self):
     """ REPLICA (redirection): http redirection to replica"""
     tmp_scope = 'mock'
     tmp_name = 'file_%s' % generate_uuid()
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" -X GET %s/redirect/%s/%s''' % (self.cacert, self.token, self.host, tmp_scope, tmp_name)
     exitcode, out, err = execute(cmd)
     assert_in('404 Not Found', out)
     # add replicas
     self.replica_client.add_replicas(rse='MOCK', files=[{'scope': tmp_scope, 'name': tmp_name, 'bytes': 1L, 'adler32': '0cc737eb'}])
Exemple #31
0
 def test_create_dataset(self):
     """CLIENT(USER): Rucio add dataset"""
     tmp_name = self.user + ':DSet' + rse_name_generator(
     )  # something like mock:DSetMOCK_S0M37HING
     cmd = 'rucio add-dataset ' + tmp_name
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     nose.tools.assert_not_equal(re.search('Added ' + tmp_name, out), None)
Exemple #32
0
 def test_ping(self):
     """PING (CURL): Get Version"""
     cmd = 'curl --cacert %s -s -X GET %s/ping' % (self.cacert, self.host)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
     ret = json.loads(out)
     nose.tools.assert_true('version' in ret)
     nose.tools.assert_is_instance(ret, dict)
Exemple #33
0
 def test_ping(self):
     """PING (CURL): Get Version"""
     cmd = 'curl --cacert %s -s -X GET %s/ping' % (self.cacert, self.host)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, )
     ret = json.loads(out)
     assert 'version' in ret
     assert isinstance(ret, dict)
Exemple #34
0
 def test_add_rse_nondet(self):
     """CLIENT(ADMIN): Add non-deterministic RSE"""
     tmp_val = rse_name_generator()
     cmd = 'rucio-admin rse add --non-deterministic %s' % tmp_val
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, )
     nose.tools.assert_equal(
         'Added new non-deterministic RSE: %s\n' % tmp_val, out)
Exemple #35
0
    def test_delete_rule(self):
        """CLIENT(USER): rule deletion"""
        set_account_limit('root', get_rse_id(self.def_rse), -1)
        tmp_file1 = file_generator()
        # add files
        cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
            self.def_rse, self.user, tmp_file1)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        # add rse
        tmp_rse = rse_name_generator()
        cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out)

        set_account_limit('root', get_rse_id(tmp_rse), -1)

        # add rse atributes
        cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
            tmp_rse)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        # add rules
        cmd = "rucio add-rule {0}:{1} 1 'spacetoken=ATLASSCRATCHDISK'".format(
            self.user, tmp_file1[5:])
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(err)
        print(out)
        # get the rules for the file
        cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
            self.user, tmp_file1[5:])
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        (rule1, rule2) = out.split()
        # delete the rules for the file
        cmd = "rucio delete-rule {0}".format(rule1)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        cmd = "rucio delete-rule {0}".format(rule2)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        # search for the file
        cmd = "rucio list-dids {0}:{1}".format(self.user, tmp_file1[5:])
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        nose.tools.assert_equal(5, len(out.splitlines()))
Exemple #36
0
 def test_download_file(self):
     """CLIENT(USER): Rucio download files"""
     tmp_file1 = file_generator()
     # add files
     cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # download files
     cmd = 'rucio download --dir /tmp {0}:{1}'.format(self.user, tmp_file1[5:])  # triming '/tmp/' from filename
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # search for the files with ls
     cmd = 'ls /tmp/{0}'.format(self.user)   # search in /tmp/mock
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
Exemple #37
0
 def test_attach_file_twice(self):
     """CLIENT(USER): Rucio attach a file twice"""
     # Attach files to a dataset using the attach method
     tmp_file1 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     # attach the files to the dataset
     cmd = 'rucio attach {0} {1}:{2}'.format(tmp_dsn, self.user, tmp_file1[5:])  # triming '/tmp/' from filenames
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     nose.tools.assert_not_equal(re.search("The file already exists", err), None)
Exemple #38
0
 def test_upload_repeated_file(self):
     """CLIENT(USER): Rucio upload repeated files"""
     # One of the files to upload is already catalogued but was removed
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
         self.def_rse, self.user, tmp_file1)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     # get the rule for the file
     cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
         self.user, tmp_file1[5:])
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     rule = out
     # delete the file from the catalog
     cmd = "rucio delete-rule {0}".format(rule)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     # delete the fisical file
     cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1[5:])
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     cmd = 'rucio upload --rse {0} --scope {1} {2} {3} {4}'.format(
         self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     nose.tools.assert_not_equal(
         re.search(
             "File {0}:{1} successfully uploaded on the storage".format(
                 self.user, tmp_file2[5:]), out), None)
Exemple #39
0
 def test_upload_file_guid(self):
     """CLIENT(USER): Rucio upload file with guid"""
     tmp_file1 = file_generator()
     tmp_guid = generate_uuid()
     cmd = 'rucio -v upload --rse {0} --guid {1} --scope {2} {3}'.format(self.def_rse, tmp_guid, self.user, tmp_file1)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file1)) in out)
Exemple #40
0
 def test_upload_repeated_file(self):
     """CLIENT(USER): Rucio upload repeated files"""
     # One of the files to upload is already catalogued but was removed
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     tmp_file1_name = path.basename(tmp_file1)
     cmd = 'rucio -v upload --rse {0} --scope {1} {2}'.format(
         self.def_rse, self.user, tmp_file1)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     # get the rule for the file
     cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
         self.user, tmp_file1_name)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     rule = out
     # delete the file from the catalog
     cmd = "rucio delete-rule {0}".format(rule)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     # delete the physical file
     cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1_name)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4}'.format(
         self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     nose.tools.assert_true((self.upload_success_str %
                             tmp_file1_name) in out)
Exemple #41
0
 def test_add_delete_add_file(self):
     """CLIENT(USER): Add/Delete/Add"""
     tmp_file1 = file_generator()
     # add file
     cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # get the rule for the file
     cmd = "rucio list-rules --did {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(self.user, tmp_file1[5:])
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     rule = out
     # delete the file from the catalog
     cmd = "rucio delete-rule {0}".format(rule)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # delete the fisical file
     cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1[5:])
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # modify the file to avoid same checksum
     cmd = "echo 'delta' >> {0}".format(tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add the same file
     cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     nose.tools.assert_equal(re.search("File {0}:{1} successfully uploaded on the storage".format(self.user, tmp_file1[5:]), out), None)
Exemple #42
0
    def test_download_fails_badmd5(self):
        """CLIENT(USER): Rucio download fails on MD5 mismatch"""
        # user has a file to upload
        filename = file_generator()
        file_md5 = md5(filename)
        filesize = stat(filename).st_size
        lfn = {'name': filename[5:], 'scope': self.user, 'bytes': filesize, 'md5': '0123456789abcdef0123456789abcdef'}
        # user uploads file
        self.replica_client.add_replicas(files=[lfn], rse=self.def_rse)
        rse_settings = rsemgr.get_rse_info(self.def_rse)
        protocol = rsemgr.create_protocol(rse_settings, 'write')
        protocol.connect()
        pfn = protocol.lfns2pfns(lfn).values()[0]
        protocol.put(filename[5:], pfn, filename[:5])
        protocol.close()
        remove(filename)

        # download file
        cmd = 'rucio -v download --dir /tmp {0}:{1}'.format(self.user, filename[5:])
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)

        report = 'Local\ checksum\:\ {0},\ Rucio\ checksum\:\ 0123456789abcdef0123456789abcdef'.format(file_md5)
        print('searching', report, 'in', err)
        nose.tools.assert_not_equal(re.search(report, err), None)

        # The file should not exist
        cmd = 'ls /tmp/'    # search in /tmp/
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        nose.tools.assert_equal(re.search(filename[5:], out), None)

        try:
            for i in listdir('data13_hip'):
                unlink('data13_hip/%s' % i)
            rmdir('data13_hip')
        except Exception:
            pass
Exemple #43
0
 def test_add_delete_add_file(self):
     """CLIENT(USER): Add/Delete/Add"""
     tmp_file1 = file_generator()
     # add file
     cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(self.def_rse, self.user, tmp_file1)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     # get the rule for the file
     cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(self.user, tmp_file1[5:])
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     rule = out
     # delete the file from the catalog
     cmd = "rucio delete-rule {0}".format(rule)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     # delete the fisical file
     cmd = "find /tmp/rucio_rse/ -name {0} |xargs rm".format(tmp_file1[5:])
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     # modify the file to avoid same checksum
     cmd = "echo 'delta' >> {0}".format(tmp_file1)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     # add the same file
     cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(self.def_rse, self.user, tmp_file1)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     nose.tools.assert_equal(re.search("File {0}:{1} successfully uploaded on the storage".format(self.user, tmp_file1[5:]), out), None)
Exemple #44
0
 def test_download_succeeds_md5only(self):
     """CLIENT(USER): Rucio download succeeds MD5 only"""
     # user has a file to upload
     filename = file_generator()
     file_md5 = md5(filename)
     filesize = stat(filename).st_size
     lfn = {
         'name': filename[5:],
         'scope': self.user,
         'bytes': filesize,
         'md5': file_md5
     }
     # user uploads file
     self.replica_client.add_replicas(files=[lfn], rse=self.def_rse)
     rse_settings = rsemgr.get_rse_info(self.def_rse)
     protocol = rsemgr.create_protocol(rse_settings, 'write')
     protocol.connect()
     pfn = protocol.lfns2pfns(lfn).values()[0]
     protocol.put(filename[5:], pfn, filename[:5])
     protocol.close()
     remove(filename)
     # download files
     cmd = 'rucio download --dir /tmp {0}:{1}'.format(
         self.user, filename[5:])
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     # search for the files with ls
     cmd = 'ls /tmp/{0}'.format(self.user)  # search in /tmp/
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     nose.tools.assert_not_equal(re.search(filename[5:], out), None)
     try:
         for i in listdir('data13_hip'):
             unlink('data13_hip/%s' % i)
         rmdir('data13_hip')
     except Exception:
         pass
Exemple #45
0
 def test_upload_repeated_file_dataset(self):
     """CLIENT(USER): Rucio upload repeated files to dataset"""
     # One of the files to upload is already in the dataset
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     tmp_file1_name = path.basename(tmp_file1)
     tmp_file3_name = path.basename(tmp_file3)
     tmp_dsn = self.user + ':DSet' + rse_name_generator(
     )  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3}'.format(
         self.def_rse, self.user, tmp_file1, tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     # upload the files to the dataset
     cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5}'.format(
         self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     # searching for the file in the new dataset
     cmd = 'rucio list-files {0}'.format(tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     # tmp_file1 must be in the dataset
     nose.tools.assert_not_equal(
         re.search("{0}:{1}".format(self.user, tmp_file1_name), out), None)
     # tmp_file3 must be in the dataset
     nose.tools.assert_not_equal(
         re.search("{0}:{1}".format(self.user, tmp_file3_name), out), None)
Exemple #46
0
    def test_replica_meta_redirection(self):
        """ REDIRECT: metalink to replica"""
        tmp_scope = 'mock'
        tmp_name = 'file_%s' % generate_uuid()
        cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s' '' % (
            self.cacert, self.token, self.vo_header, self.host, tmp_scope,
            tmp_name)
        _, out, _ = execute(cmd)
        assert_in('404 Not Found', out)

        self.replica_client.add_replicas(rse='MOCK',
                                         files=[{
                                             'scope': tmp_scope,
                                             'name': tmp_name,
                                             'bytes': 1,
                                             'adler32': '0cc737eb'
                                         }])
        self.replica_client.add_replicas(rse='MOCK3',
                                         files=[{
                                             'scope': tmp_scope,
                                             'name': tmp_name,
                                             'bytes': 1,
                                             'adler32': '0cc737eb'
                                         }])
        _, out, _ = execute(cmd)
        assert_in('303 See Other', out)
        assert_in('Link: </redirect/%s/%s/metalink' % (tmp_scope, tmp_name),
                  out)

        cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s/metalink' '' % (
            self.cacert, self.token, self.vo_header, self.host, tmp_scope,
            tmp_name)
        _, out, _ = execute(cmd)
        assert_in('200 OK', out)
        assert_in('<?xml', out)
        assert_in('<metalink', out)
        assert_in('<url location="MOCK"', out)
        assert_in('<url location="MOCK3"', out)
Exemple #47
0
    def test_del_identity(self):
        """CLIENT(ADMIN): Test del identity"""
        tmp_acc = account_name_generator()

        # create account
        cmd = 'rucio-admin account add %s' % tmp_acc
        exitcode, out, err = execute(cmd)
        # add identity to account
        cmd = 'rucio-admin identity add --account %s --type GSS --id [email protected] --email [email protected]' % tmp_acc
        exitcode, out, err = execute(cmd)
        # delete identity from account
        cmd = 'rucio-admin identity delete --account %s --type GSS --id [email protected]' % tmp_acc
        print self.marker + cmd
        exitcode, out, err = execute(cmd)
        print out, err
        nose.tools.assert_equal('Deleted identity: [email protected]\n', out)
        # list identities for account
        cmd = 'rucio-admin account list-identities %s' % (tmp_acc)
        print self.marker + cmd
        print cmd
        exitcode, out, err = execute(cmd)
        print out, err
        nose.tools.assert_equal('', out)
Exemple #48
0
 def test_upload_file_guid(self):
     """CLIENT(USER): Rucio upload file with guid"""
     tmp_file1 = file_generator()
     tmp_guid = generate_uuid()
     cmd = 'rucio upload --rse {0} --guid {1} --scope {2} {3}'.format(
         self.def_rse, tmp_guid, self.user, tmp_file1)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     nose.tools.assert_true(
         "File %s:%s successfully uploaded on the storage" %
         (self.user, tmp_file1[5:]) in out)
Exemple #49
0
    def test_del_identity(self):
        """CLIENT(ADMIN): Test del identity"""
        tmp_acc = account_name_generator()

        # create account
        cmd = 'rucio-admin account add %s' % tmp_acc
        exitcode, out, err = execute(cmd)
        # add identity to account
        cmd = 'rucio-admin identity add --account %s --type GSS --id [email protected] --email [email protected]' % tmp_acc
        exitcode, out, err = execute(cmd)
        # delete identity from account
        cmd = 'rucio-admin identity delete --account %s --type GSS --id [email protected]' % tmp_acc
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        nose.tools.assert_equal('Deleted identity: [email protected]\n', out)
        # list identities for account
        cmd = 'rucio-admin account list-identities %s' % (tmp_acc)
        print(self.marker + cmd)
        print(cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        nose.tools.assert_equal('', out)
Exemple #50
0
 def test_upload_file(self):
     """CLIENT(USER): Rucio upload files"""
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3} {4}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     nose.tools.assert_not_equal(re.search("File {0}:{1} successfully uploaded on the storage".format(self.user, tmp_file1[5:]), out), None)
Exemple #51
0
    def test_attach_dataset_twice(self):
        """ CLIENT(USER): Rucio attach a dataset twice """
        container = 'container_%s' % generate_uuid()
        dataset = 'dataset_%s' % generate_uuid()
        self.did_client.add_container(scope=self.user, name=container)
        self.did_client.add_dataset(scope=self.user, name=dataset)

        # Attach dataset to container
        cmd = 'rucio attach {0}:{1} {0}:{2}'.format(self.user, container,
                                                    dataset)
        exitcode, out, err = execute(cmd)

        # Attach again
        cmd = 'rucio attach {0}:{1} {0}:{2}'.format(self.user, container,
                                                    dataset)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out)
        print(err)
        nose.tools.assert_not_equal(
            re.search(
                "Data identifier already added to the destination content",
                err), None)
Exemple #52
0
    def test_downgrade_and_upgrade(self):
        """ ALEMBIC (CORE): Test the schema migration """

        cmd = 'alembic upgrade head'
        exitcode, out, err = execute(cmd)
        print(cmd)
        print(exitcode, out, err)
        assert_equal(exitcode, 0)

        cmd = 'alembic downgrade  -1'
        while exitcode is 0:
            exitcode, out, err = execute(cmd)
            print(cmd)
            print(exitcode, out, err)
            if "Relative revision -1 didn't produce 1 migrations" not in out:
                assert_equal(exitcode, 0)
            else:
                assert_equal(exitcode, 255)

        cmd = 'alembic upgrade head'
        exitcode, out, err = execute(cmd)
        print(cmd)
        print(exitcode, out, err)
        assert_equal(exitcode, 0)
Exemple #53
0
    def test_replica_header_redirection(self):
        """ REDIRECT: header to replica"""
        tmp_scope = 'mock'
        tmp_name = 'file_%s' % generate_uuid()
        cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" %s -X GET %s/redirect/%s/%s''' % (self.cacert,
                                                                                                    self.token,
                                                                                                    self.vo_header,
                                                                                                    self.host,
                                                                                                    tmp_scope,
                                                                                                    tmp_name)
        _, out, _ = execute(cmd)
        assert '404 Not Found' in out

        self.replica_client.add_replicas(rse='MOCK', files=[{'scope': tmp_scope,
                                                             'name': tmp_name,
                                                             'bytes': 1,
                                                             'adler32': '0cc737eb'}])
        self.replica_client.add_replicas(rse='MOCK3', files=[{'scope': tmp_scope,
                                                              'name': tmp_name,
                                                              'bytes': 1,
                                                              'adler32': '0cc737eb'}])
        _, out, _ = execute(cmd)
        assert '303 See Other' in out
        assert 'Location: https://mock' in out
Exemple #54
0
 def test_upload_file_guid(self):
     """CLIENT(USER): Rucio upload file with guid"""
     tmp_file1 = file_generator()
     tmp_guid = generate_uuid()
     cmd = 'rucio upload --rse {0} --guid {1} --scope {2} {3}'.format(
         self.def_rse, tmp_guid, self.user, tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     remove(tmp_file1)
     nose.tools.assert_not_equal(
         re.search(
             "File {0}:{1} successfully uploaded on the storage".format(
                 self.user, tmp_file1[5:]), out), None)
Exemple #55
0
 def test_replica_meta_redirection(self):
     """ REDIRECT: metalink to replica"""
     tmp_scope = 'mock'
     tmp_name = 'file_%s' % generate_uuid()
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Auth-Token: %s" -X GET %s/redirect/%s/%s' '' % (
         self.cacert, self.token, self.host, tmp_scope, tmp_name)
     _, out, _ = execute(cmd)
     assert_in('404 Not Found', out)
     # add replicas
     self.replica_client.add_replicas(rse='MOCK',
                                      files=[{
                                          'scope': tmp_scope,
                                          'name': tmp_name,
                                          'bytes': 1L,
                                          'adler32': '0cc737eb'
                                      }])
Exemple #56
0
 def test_upload_file(self):
     """CLIENT(USER): Rucio upload files"""
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file1)) in out)
     nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file2)) in out)
     nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file3)) in out)
Exemple #57
0
 def test_upload_adds_md5digest(self):
     """CLIENT(USER): Upload Checksums"""
     # user has a file to upload
     filename = file_generator()
     file_md5 = md5(filename)
     # user uploads file
     cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
         self.def_rse, self.user, filename)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     # When inspecting the metadata of the new file the user finds the md5 checksum
     meta = self.did_client.get_metadata(scope=self.user, name=filename[5:])
     nose.tools.assert_in('md5', meta)
     nose.tools.assert_equal(meta['md5'], file_md5)
     remove(filename)
Exemple #58
0
 def test_upload_file(self):
     """CLIENT(USER): Rucio upload files"""
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     cmd = 'rucio upload --rse {0} --scope {1} {2} {3} {4}'.format(
         self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     nose.tools.assert_true(
         "File %s:%s successfully uploaded on the storage" %
         (self.user, tmp_file1[5:]) in out)
Exemple #59
0
    def test_download_item(self):
        """ DOWNLOAD (CLIENT): download DIDs. """
        download_client = DownloadClient()
        tmp_file1 = file_generator()
        scope = 'mock'
        name = tmp_file1[5:]
        uuid = generate_uuid()
        cmd = 'rucio upload --rse {0} --scope {1} --guid {2} {3}'.format(
            'MOCK4', scope, uuid, tmp_file1)
        exitcode, out, err = execute(cmd)

        # Download specific DID
        result = download_client.download_dids([{
            'did': '%s:%s' % (scope, name)
        }])
        assert_true(result)

        # Download with wildcard
        result = download_client.download_dids([{
            'did':
            '%s:%s' % (scope, name[:10] + '*')
        }])
        assert_true(result)

        # Download with filter
        result = download_client.download_dids([{
            'filters': {
                'guid': uuid,
                'scope': scope
            }
        }])
        assert_true(result)

        # Download with wildcard and name
        result = download_client.download_dids([{
            'did': '%s:%s' % (scope, '*'),
            'filters': {
                'guid': uuid
            }
        }])
        assert_true(result)
Exemple #60
0
    def test_download_file(self):
        """CLIENT(USER): Rucio download files"""
        tmp_file1 = file_generator()
        # add files
        cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
            self.def_rse, self.user, tmp_file1)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        # download files
        cmd = 'rucio -v download --dir /tmp {0}:{1}'.format(
            self.user, tmp_file1[5:])  # triming '/tmp/' from filename
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        # search for the files with ls
        cmd = 'ls /tmp/'  # search in /tmp/
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)

        tmp_file1 = file_generator()
        # add files
        cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
            self.def_rse, self.user, tmp_file1)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        # download files
        cmd = 'rucio -v download --dir /tmp {0}:{1}'.format(
            self.user, tmp_file1[5:-2] + '*')  # triming '/tmp/' from filename
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        # search for the files with ls
        cmd = 'ls /tmp/'  # search in /tmp/
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)

        try:
            for i in listdir('data13_hip'):
                unlink('data13_hip/%s' % i)
            rmdir('data13_hip')
        except Exception:
            pass