Beispiel #1
0
 def test_create_rule(self):
     """CLIENT(USER): Rucio add rule"""
     tmp_file1 = file_generator()
     # add files
     cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rse
     tmp_rse = rse_name_generator()
     cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     # add rse atributes
     cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rse
     tmp_rse = rse_name_generator()
     cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rse atributes
     cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rse
     tmp_rse = rse_name_generator()
     cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rse atributes
     cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rules
     cmd = "rucio add-rule {0}:{1} 3 'spacetoken=ATLASSCRATCHDISK'".format(self.user, tmp_file1[5:])
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     rule = out[:-1]  # triming new line character
     # check if rule exist for the file
     cmd = "rucio list-rules --did {0}:{1}".format(self.user, tmp_file1[5:])
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     nose.tools.assert_not_equal(re.search(rule, out), None)
Beispiel #2
0
 def test_add_files_to_dataset(self):
     """CLIENT(USER): Rucio add files to dataset"""
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_dataset = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # add files
     cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_file2)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # create dataset
     cmd = 'rucio add-dataset ' + tmp_dataset
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add files to dataset
     cmd = 'rucio add-files-to-dataset --to {0} mock:{1} mock:{2}'.format(tmp_dataset, tmp_file1[5:], tmp_file2[5:])  # triming '/tmp/' from filename
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # find the added files
     cmd = 'rucio list-files ' + tmp_dataset
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
Beispiel #3
0
 def test_download_dataset(self):
     """CLIENT(USER): Rucio download dataset"""
     tmp_file1 = file_generator()
     tmp_dataset = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # add files
     cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # create dataset
     cmd = 'rucio add-dataset ' + tmp_dataset
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add files to dataset
     cmd = 'rucio add-files-to-dataset --to {0} {1}:{2}'.format(tmp_dataset, self.user, tmp_file1[5:])  # triming '/tmp/' from filename
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # download dataset
     cmd = 'rucio download --dir /tmp {0}'.format(tmp_dataset)  # triming '/tmp/' from filename
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     search = 'Getting file {0}:{1}'.format(self.user, tmp_file1[5:])
     nose.tools.assert_not_equal(re.search(search, err), None)
     search = 'File validated'
     nose.tools.assert_not_equal(re.search(search, out), None)
     search = 'DID ' + tmp_dataset
     nose.tools.assert_not_equal(re.search(search, out), None)
Beispiel #4
0
 def test_detach_files_dataset(self):
     """CLIENT(USER): Rucio detach files to dataset"""
     # Attach files to a dataset using the attach method
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3} {4} --did {5}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     # detach the files to the dataset
     cmd = 'rucio detach --from {0} {1}:{2} {1}:{3}'.format(tmp_dsn, self.user, tmp_file2[5:], tmp_file3[5:])  # triming '/tmp/' from filenames
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     # searching for the file in the new dataset
     cmd = 'rucio list-files {0}'.format(tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     # tmp_file1 must be in the dataset
     nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
     # tmp_file3 must be in the dataset
     nose.tools.assert_equal(re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
Beispiel #5
0
 def test_upload_repeated_file_dataset(self):
     """CLIENT(USER): Rucio upload repeated files to dataset"""
     # One of the files to upload is already in the dataset
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} --files {2} --did {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     # upload the files to the dataset
     cmd = 'rucio upload --rse {0} --scope {1} --files {2} {3} {4} --did {5}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     # searching for the file in the new dataset
     cmd = 'rucio list-files {0}'.format(tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     # tmp_file1 must be in the dataset
     nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file1[5:]), out), None)
     # tmp_file3 must be in the dataset
     nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
Beispiel #6
0
 def test_add_rse(self):
     """CLIENT(ADMIN): Add RSE"""
     tmp_val = rse_name_generator()
     cmd = 'rucio-admin rse add %s' % tmp_val
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
     nose.tools.assert_equal('Added new RSE: %s\n' % tmp_val, out)
Beispiel #7
0
 def test_create_dataset(self):
     """CLIENT(USER): Rucio add dataset"""
     tmp_name = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     cmd = 'rucio add-dataset ' + tmp_name
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     nose.tools.assert_not_equal(re.search('Added ' + tmp_name, out), None)
Beispiel #8
0
 def test_upload(self):
     """CLIENT(USER): Upload"""
     tmp_val = rse_name_generator()
     cmd = 'rucio-admin rse add %s' % tmp_val
     exitcode, out, err = execute(cmd)
     cmd = 'rucio upload'
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
Beispiel #9
0
 def test_create_dataset(self):
     """CLIENT(USER): Rucio add dataset"""
     tmp_name = self.user + ':DSet' + rse_name_generator(
     )  # something like mock:DSetMOCK_S0M37HING
     cmd = 'rucio add-dataset ' + tmp_name
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     nose.tools.assert_not_equal(re.search('Added ' + tmp_name, out), None)
Beispiel #10
0
 def test_add_rse_nondet(self):
     """CLIENT(ADMIN): Add non-deterministic RSE"""
     tmp_val = rse_name_generator()
     cmd = 'rucio-admin rse add --non-deterministic %s' % tmp_val
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, )
     nose.tools.assert_equal(
         'Added new non-deterministic RSE: %s\n' % tmp_val, out)
Beispiel #11
0
 def test_upload(self):
     """CLIENT(USER): Upload"""
     tmp_val = rse_name_generator()
     cmd = 'rucio-admin rse add %s' % tmp_val
     exitcode, out, err = execute(cmd)
     cmd = 'rucio upload'
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, )
Beispiel #12
0
def test_reaper_bulk_delete(vo, caches_mock):
    """ REAPER (DAEMON): Mock test the reaper daemon on async bulk delete request."""
    [cache_region] = caches_mock
    scope = InternalScope('data13_hip', vo=vo)

    nb_files = 250
    file_size = 200  # 2G
    rse_name, rse_id, dids = __add_test_rse_and_replicas(
        vo=vo,
        scope=scope,
        rse_name=rse_name_generator(),
        names=['lfn' + generate_uuid() for _ in range(nb_files)],
        file_size=file_size)

    rse_core.set_rse_limits(rse_id=rse_id,
                            name='MinFreeSpace',
                            value=50 * file_size)
    assert len(
        list(replica_core.list_replicas(dids=dids,
                                        rse_expression=rse_name))) == nb_files

    # Check first if the reaper does not delete anything if no space is needed
    cache_region.invalidate()
    rse_core.set_rse_usage(rse_id=rse_id,
                           source='storage',
                           used=nb_files * file_size,
                           free=323000000000)
    reaper(once=True,
           rses=[],
           include_rses=rse_name,
           exclude_rses=None,
           chunk_size=1000,
           scheme='MOCK')
    assert len(
        list(replica_core.list_replicas(dids=dids,
                                        rse_expression=rse_name))) == nb_files

    # Now put it over threshold and delete
    cache_region.invalidate()
    rse_core.set_rse_usage(rse_id=rse_id,
                           source='storage',
                           used=nb_files * file_size,
                           free=1)
    reaper(once=True,
           rses=[],
           include_rses=rse_name,
           exclude_rses=None,
           chunk_size=1000,
           scheme='MOCK')
    reaper(once=True,
           rses=[],
           include_rses=rse_name,
           exclude_rses=None,
           chunk_size=1000,
           scheme='MOCK')
    assert len(list(replica_core.list_replicas(
        dids, rse_expression=rse_name))) == 200
Beispiel #13
0
 def test_list_rses(self):
     """CLIENT(ADMIN): List RSEs"""
     tmp_val = rse_name_generator()
     cmd = 'rucio-admin rse add %s' % tmp_val
     exitcode, out, err = execute(cmd)
     cmd = 'rucio-admin rse list'
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out, )
     nose.tools.assert_regexp_matches(out, re.compile('.*%s.*' % tmp_val))
Beispiel #14
0
    def setUp(self):
        self.account_client = AccountClient()
        self.rse_client = RSEClient()

        self.account = generate_uuid()[:10]
        self.rse = rse_name_generator()

        self.account_client.add_account(self.account, 'SERVICE',
                                        '*****@*****.**')
        self.rse_client.add_rse(self.rse)
Beispiel #15
0
 def test_list_rses(self):
     """CLIENT(ADMIN): List RSEs"""
     tmp_val = rse_name_generator()
     cmd = 'rucio-admin rse add %s' % tmp_val
     exitcode, out, err = execute(cmd)
     cmd = 'rucio-admin rse list'
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out,
     nose.tools.assert_regexp_matches(out, re.compile('.*%s.*' % tmp_val))
Beispiel #16
0
    def test_delete_rule(self):
        """CLIENT(USER): rule deletion"""
        set_account_limit('root', get_rse_id(self.def_rse), -1)
        tmp_file1 = file_generator()
        # add files
        cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(
            self.def_rse, self.user, tmp_file1)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        # add rse
        tmp_rse = rse_name_generator()
        cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out)

        set_account_limit('root', get_rse_id(tmp_rse), -1)

        # add rse atributes
        cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(
            tmp_rse)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        # add rules
        cmd = "rucio add-rule {0}:{1} 1 'spacetoken=ATLASSCRATCHDISK'".format(
            self.user, tmp_file1[5:])
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(err)
        print(out)
        # get the rules for the file
        cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(
            self.user, tmp_file1[5:])
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        (rule1, rule2) = out.split()
        # delete the rules for the file
        cmd = "rucio delete-rule {0}".format(rule1)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        cmd = "rucio delete-rule {0}".format(rule2)
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        # search for the file
        cmd = "rucio list-dids {0}:{1}".format(self.user, tmp_file1[5:])
        print(self.marker + cmd)
        exitcode, out, err = execute(cmd)
        print(out, err)
        nose.tools.assert_equal(5, len(out.splitlines()))
Beispiel #17
0
    def setUpClass(cls):
        if config_get_bool('common',
                           'multi_vo',
                           raise_exception=False,
                           default=False):
            cls.vo = {'vo': get_vo()}
        else:
            cls.vo = {}

        cls.rse_client = RSEClient()
        cls.tmp_rse_name = rse_name_generator()
        cls.rse_client.add_rse(cls.tmp_rse_name, vo=cls.vo)
        cls.tmp_rse = cls.rse_client.get_rse(cls.tmp_rse_name)['id']
Beispiel #18
0
 def __init__(
     self,
     vo: str,
     db_session: "Session",
     setup_func: "Optional[Callable]" = None,
     teardown_func: "Optional[Callable]" = None,
 ):
     self.vo = vo
     self.db_session = db_session
     self.setup = setup_func
     self.teardown = teardown_func
     self.name = rse_name_generator()
     self.rse_id: "Optional[str]" = None
Beispiel #19
0
 def test_post_rse(self):
     """RSE (CURL): add RSE"""
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" --cert %s --key %s -X GET %s/auth/x509 | tr -d \'\r\' | grep X-Rucio-Auth-Token:' % (
         self.cacert, self.usercert, self.userkey, self.auth_host)
     exitcode, out, err = execute(cmd)
     nose.tools.assert_in('X-Rucio-Auth-Token', out)
     os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):].rstrip()
     cmd = '''curl -s -i --cacert %s -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -X POST %s/rses/%s''' % (
         self.cacert, self.host, rse_name_generator())
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     nose.tools.assert_in('201 Created', out)
Beispiel #20
0
    def test_scope_list(self):
        """ DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """

        # create some dummy data
        self.tmp_accounts = ['jdoe' for i in range(3)]
        self.tmp_scopes = [scope_name_generator() for i in range(3)]
        self.tmp_rses = [rse_name_generator() for i in range(3)]
        self.tmp_files = ['file_%s' % generate_uuid() for i in range(3)]
        self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in range(3)]
        self.tmp_containers = ['container_%s' % generate_uuid() for i in range(3)]

        # add dummy data to the catalogue
        for i in range(3):
            self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
            self.rse_client.add_rse(self.tmp_rses[i])
            self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1, '0cc737eb')

        # put files in datasets
        for i in range(3):
            for j in range(3):
                files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1, 'adler32': '0cc737eb'}]
                self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
                self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)

        # put datasets in containers
        for i in range(3):
            for j in range(3):
                datasets = [{'scope': self.tmp_scopes[j], 'name': self.tmp_datasets[j]}]
                self.did_client.add_container(self.tmp_scopes[i], self.tmp_containers[j])
                self.did_client.add_datasets_to_container(self.tmp_scopes[i], self.tmp_containers[j], datasets)

        # reverse check if everything is in order
        for i in range(3):
            result = self.did_client.scope_list(self.tmp_scopes[i], recursive=True)

            r_topdids = []
            r_otherscopedids = []
            r_scope = []
            for r in result:
                if r['level'] == 0:
                    r_topdids.append(r['scope'] + ':' + r['name'])
                    r_scope.append(r['scope'])
                if r['scope'] != self.tmp_scopes[i]:
                    r_otherscopedids.append(r['scope'] + ':' + r['name'])
                    assert_in(r['level'], [1, 2])

            for j in range(3):
                assert_equal(self.tmp_scopes[i], r_scope[j])
                if j != i:
                    assert_in(self.tmp_scopes[j] + ':' + self.tmp_files[j], r_otherscopedids)
            assert_not_in(self.tmp_scopes[i] + ':' + self.tmp_files[i], r_topdids)
Beispiel #21
0
    def test_to_repair_a_rule_with_only_1_rse_whose_site_is_blacklisted(self):
        """ JUDGE REPAIRER: Test to repair a rule with only 1 rse whose site is blacklisted"""

        rse = rse_name_generator()
        rse_id = add_rse(rse, **self.vo)
        update_rse(rse_id, {'availability_write': False})
        set_local_account_limit(self.jdoe, rse_id, -1)

        rule_repairer(once=True)  # Clean out the repairer
        scope = InternalScope('mock', **self.vo)
        files = create_files(4, scope, self.rse4_id, bytes=100)
        dataset = 'dataset_' + str(uuid())
        add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
        attach_dids(scope, dataset, files, self.jdoe)

        rule_id = add_rule(dids=[{
            'scope': scope,
            'name': dataset
        }],
                           account=self.jdoe,
                           copies=1,
                           rse_expression=rse,
                           grouping='DATASET',
                           weight=None,
                           lifetime=None,
                           locked=False,
                           subscription_id=None,
                           ignore_availability=True,
                           activity='DebugJudge')[0]

        assert (RuleState.STUCK == get_rule(rule_id)['state'])
        rule_repairer(once=True)

        # Stil assert STUCK because of ignore_availability:
        assert (RuleState.STUCK == get_rule(rule_id)['state'])

        region = make_region().configure('dogpile.cache.memcached',
                                         expiration_time=3600,
                                         arguments={
                                             'url':
                                             config_get(
                                                 'cache', 'url', False,
                                                 '127.0.0.1:11211'),
                                             'distributed_lock':
                                             True
                                         })
        region.delete(sha256(rse.encode()).hexdigest())

        update_rse(rse_id, {'availability_write': True})
        rule_repairer(once=True)
        assert (RuleState.REPLICATING == get_rule(rule_id)['state'])
Beispiel #22
0
    def setup(self):

        self.rc = ReplicaClient()

        self.client_location_without_proxy = {
            'ip': '192.168.0.1',
            'fqdn': 'anomalous-materials.blackmesa.com',
            'site': 'BLACKMESA'
        }
        self.rse_without_proxy = rse_name_generator()
        add_rse(self.rse_without_proxy)
        add_rse_attribute(rse=self.rse_without_proxy,
                          key='site',
                          value='BLACKMESA')

        self.client_location_with_proxy = {
            'ip': '10.0.1.1',
            'fqdn': 'test-chamber.aperture.com',
            'site': 'APERTURE'
        }
        self.rse_with_proxy = rse_name_generator()
        add_rse(self.rse_with_proxy)
        add_rse_attribute(rse=self.rse_with_proxy,
                          key='root-proxy-internal',
                          value='root://proxy.aperture.com:1094')
        add_rse_attribute(rse=self.rse_with_proxy,
                          key='site',
                          value='APERTURE')

        self.files = [{
            'scope': 'mock',
            'name': 'half-life_%s' % i,
            'bytes': 1234L,
            'adler32': 'deadbeef',
            'meta': {
                'events': 666
            }
        } for i in xrange(1, 4)]
Beispiel #23
0
    def test_list_archive_contents_transparently(self):
        """ ARCHIVE (CORE): Transparent archive listing """

        scope = 'mock'
        rse = 'APERTURE_%s' % rse_name_generator()
        add_rse(rse)

        add_protocol(rse, {'scheme': 'root',
                           'hostname': 'root.aperture.com',
                           'port': 1409,
                           'prefix': '//test/chamber/',
                           'impl': 'rucio.rse.protocols.xrootd.Default',
                           'domains': {
                               'lan': {'read': 1, 'write': 1, 'delete': 1},
                               'wan': {'read': 1, 'write': 1, 'delete': 1}}})

        # register archive
        archive = {'scope': scope, 'name': 'weighted.storage.cube.zip', 'type': 'FILE',
                   'bytes': 2596, 'adler32': 'beefdead'}
        add_replicas(rse=rse, files=[archive], account='root')

        # archived files with replicas
        files_with_replicas = [{'scope': scope, 'name': 'witrep-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE',
                                'bytes': 1234, 'adler32': 'deadbeef'} for i in xrange(2)]
        add_replicas(rse=rse, files=files_with_replicas, account='root')
        self.dc.add_files_to_archive(scope=scope, name=archive['name'], files=files_with_replicas)

        res = [r['pfns'] for r in self.rc.list_replicas(dids=[{'scope': scope, 'name': f['name']} for f in files_with_replicas],
                                                        resolve_archives=True)]
        assert_equal(len(res), 2)
        assert_equal(len(res[0]), 2)
        assert_equal(len(res[1]), 2)
        for r in res:
            for p in r:
                if r[p]['domain'] == 'zip':
                    assert_in('weighted.storage.cube.zip?xrdcl.unzip=witrep-', p)
                else:
                    assert_not_in('weighted.storage.cube.zip?xrdcl.unzip=witrep-', p)

        # archived files without replicas
        files = [{'scope': scope, 'name': 'norep-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE',
                  'bytes': 1234, 'adler32': 'deadbeef'} for i in xrange(2)]
        self.dc.add_files_to_archive(scope=scope, name=archive['name'], files=files)
        res = [r['pfns'] for r in self.rc.list_replicas(dids=[{'scope': scope, 'name': f['name']} for f in files],
                                                        resolve_archives=True)]
        assert_equal(len(res), 2)
        for r in res:
            assert_in('weighted.storage.cube.zip?xrdcl.unzip=norep-', r.keys()[0])

        del_rse(rse)
Beispiel #24
0
def test_reaper():
    """ REAPER2 (DAEMON): Test the reaper2 daemon."""
    if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
        vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
    else:
        vo = {}

    rse_name = rse_name_generator()
    rse_id = rse_core.add_rse(rse_name, **vo)

    mock_protocol = {'scheme': 'MOCK',
                     'hostname': 'localhost',
                     'port': 123,
                     'prefix': '/test/reaper',
                     'impl': 'rucio.rse.protocols.mock.Default',
                     'domains': {
                         'lan': {'read': 1,
                                 'write': 1,
                                 'delete': 1},
                         'wan': {'read': 1,
                                 'write': 1,
                                 'delete': 1}}}
    rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)

    nb_files = 30
    file_size = 2147483648  # 2G

    file_names = []
    for i in range(nb_files):
        file_name = 'lfn' + generate_uuid()
        file_names.append(file_name)
        replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo),
                                 name=file_name, bytes=file_size,
                                 tombstone=datetime.utcnow() - timedelta(days=1),
                                 account=InternalAccount('root', **vo), adler32=None, md5=None)

    rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800)
    rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240)
    rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)

    if vo:
        reaper(once=True, rses=[], include_rses='vo=%s&(%s)' % (vo['vo'], rse_name), exclude_rses=[])
        reaper(once=True, rses=[], include_rses='vo=%s&(%s)' % (vo['vo'], rse_name), exclude_rses=[])
    else:
        reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[])
        reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[])

    assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in file_names], rse_expression=rse_name))) == nb_files - 5
Beispiel #25
0
    def test_add_rule_with_ignore_availability(self):
        """ REPLICATION RULE (CORE): Add a replication rule with ignore_availability setting"""
        rse = rse_name_generator()
        add_rse(rse)
        update_rse(rse, {'availability_write': False})

        scope = 'mock'
        files = create_files(3, scope, self.rse1)
        dataset = 'dataset_' + str(uuid())
        add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
        attach_dids(scope, dataset, files, 'jdoe')

        with assert_raises(InvalidRSEExpression):
            add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=rse, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)[0]

        add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=rse, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=True)[0]
Beispiel #26
0
 def test_delete_rule(self):
     """CLIENT(USER): rule deletion"""
     tmp_file1 = file_generator()
     # add files
     cmd = 'rucio upload --rse {0} --scope {1} --files {2}'.format(self.def_rse, self.user, tmp_file1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rse
     tmp_rse = rse_name_generator()
     cmd = 'rucio-admin rse add {0}'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     # add rse atributes
     cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # add rules
     cmd = "rucio add-rule {0}:{1} 1 'spacetoken=ATLASSCRATCHDISK'".format(self.user, tmp_file1[5:])
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     # get the rules for the file
     cmd = "rucio list-rules --did {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format(self.user, tmp_file1[5:])
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     (rule1, rule2) = out.split()
     # delete the rules for the file
     cmd = "rucio delete-rule {0}".format(rule1)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     cmd = "rucio delete-rule {0}".format(rule2)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     # search for the file
     cmd = "rucio list-dids {0}:{1}".format(self.user, tmp_file1[5:])
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out, err
     nose.tools.assert_equal('', out)
Beispiel #27
0
 def test_attach_files_dataset(self):
     """CLIENT(USER): Rucio attach files to dataset"""
     # Attach files to a dataset using the attach method
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator(
     )  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
         self.def_rse, self.user, tmp_file1, tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     # upload the files
     cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(
         self.def_rse, self.user, tmp_file2, tmp_file3)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     # attach the files to the dataset
     cmd = 'rucio attach {0} {1}:{2} {1}:{3}'.format(
         tmp_dsn, self.user, tmp_file2[5:],
         tmp_file3[5:])  # triming '/tmp/' from filenames
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     # searching for the file in the new dataset
     cmd = 'rucio list-files {0}'.format(tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     # tmp_file2 must be in the dataset
     nose.tools.assert_not_equal(
         re.search("{0}:{1}".format(self.user, tmp_file2[5:]), out), None)
     # tmp_file3 must be in the dataset
     nose.tools.assert_not_equal(
         re.search("{0}:{1}".format(self.user, tmp_file3[5:]), out), None)
Beispiel #28
0
    def setup(self):

        self.rc = client.ReplicaClient()
        self.rse = rse_name_generator()
        add_rse(self.rse)
        add_protocol(
            self.rse, {
                'scheme': 'https',
                'hostname': 'storage.googleapis.com',
                'port': 443,
                'prefix': '/atlas-europe-west1/',
                'impl': 'rucio.rse.protocols.gfal.Default',
                'domains': {
                    'lan': {
                        'read': 1,
                        'write': 1,
                        'delete': 1
                    },
                    'wan': {
                        'read': 1,
                        'write': 1,
                        'delete': 1,
                        'third_party_copy': 1
                    }
                }
            })

        # register some files there
        self.files = [{
            'scope': 'mock',
            'name': 'file-on-gcs_%s' % i,
            'bytes': 1234,
            'adler32': 'deadbeef',
            'meta': {
                'events': 666
            }
        } for i in range(0, 3)]
        add_replicas(rse=self.rse,
                     files=self.files,
                     account='root',
                     ignore_availability=True)

        def tearDown(self):
            delete_replicas(rse=self.rse, files=self.files)
            del_rse(self.rse)
Beispiel #29
0
 def test_detach_non_existing_file(self):
     """CLIENT(USER): Rucio detach a non existing file"""
     tmp_file1 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     # attach the files to the dataset
     cmd = 'rucio detach {0} {1}:{2}'.format(tmp_dsn, self.user, 'file_ghost')  # triming '/tmp/' from filenames
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     nose.tools.assert_not_equal(re.search("Data identifier not found.", err), None)
Beispiel #30
0
 def test_detach_non_existing_file(self):
     """CLIENT(USER): Rucio detach a non existing file"""
     tmp_file1 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} --files {2} --did {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     remove(tmp_file1)
     # attach the files to the dataset
     cmd = 'rucio detach --from {0} {1}:{2}'.format(tmp_dsn, self.user, 'file_ghost')  # triming '/tmp/' from filenames
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     print err
     nose.tools.assert_not_equal(re.search("Failed to detach data identifier", err), None)
Beispiel #31
0
 def test_attach_file_twice(self):
     """CLIENT(USER): Rucio attach a file twice"""
     # Attach files to a dataset using the attach method
     tmp_file1 = file_generator()
     tmp_dsn = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio upload --rse {0} --scope {1} {2} {3}'.format(self.def_rse, self.user, tmp_file1, tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     # attach the files to the dataset
     cmd = 'rucio attach {0} {1}:{2}'.format(tmp_dsn, self.user, tmp_file1[5:])  # triming '/tmp/' from filenames
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     nose.tools.assert_not_equal(re.search("The file already exists", err), None)
Beispiel #32
0
    def setupClass(self):
        if config_get_bool('common',
                           'multi_vo',
                           raise_exception=False,
                           default=False):
            self.vo = {
                'vo':
                config_get('client',
                           'vo',
                           raise_exception=False,
                           default='tst')
            }
        else:
            self.vo = {}

        self.rse_client = RSEClient()
        self.tmp_rse_name = rse_name_generator()
        self.rse_client.add_rse(self.tmp_rse_name, vo=self.vo)
        self.tmp_rse = self.rse_client.get_rse(self.tmp_rse_name)['id']
Beispiel #33
0
    def test_atlas_archival_policy(self):
        """ UNDERTAKER (CORE): Test the atlas archival policy. """
        tmp_scope = 'mock'
        nbdatasets = 5
        nbfiles = 5

        rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
        add_rse(rse)

        set_account_limit('jdoe', get_rse_id(rse), -1)

        dsns2 = [{
            'name':
            'dsn_%s' % generate_uuid(),
            'scope':
            tmp_scope,
            'type':
            'DATASET',
            'lifetime':
            -1,
            'rules': [{
                'account': 'jdoe',
                'copies': 1,
                'rse_expression': rse,
                'grouping': 'DATASET'
            }]
        } for i in xrange(nbdatasets)]

        add_dids(dids=dsns2, account='root')

        replicas = list()
        for dsn in dsns2:
            files = [{
                'scope': tmp_scope,
                'name': 'file_%s' % generate_uuid(),
                'bytes': 1L,
                'adler32': '0cc737eb',
                'tombstone': datetime.utcnow() + timedelta(weeks=2),
                'meta': {
                    'events': 10
                }
            } for i in xrange(nbfiles)]
Beispiel #34
0
 def _make_rse(self, scheme, protocol_impl):
     rse_name = rse_name_generator()
     rse_id = rse_core.add_rse(rse_name, vo=self.vo)
     rse_core.add_protocol(rse_id=rse_id, parameter={
         'scheme': scheme,
         'hostname': 'host%d' % len(self.created_rses),
         'port': 0,
         'prefix': '/test',
         'impl': protocol_impl,
         'domains': {
             'wan': {
                 'read': 1,
                 'write': 1,
                 'delete': 1,
                 'third_party_copy': 1
             }
         }
     })
     self.created_rses.append(rse_id)
     return rse_name, rse_id
Beispiel #35
0
    def test_export_import(self):
        """ IMPORT/EXPORT (REST): Test the export and import of data together to check same syntax."""
        # Setup new RSE, distance, attribute, limits
        new_rse = rse_name_generator()
        add_rse(new_rse)

        # Get token
        mw = []
        headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
        r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
        token = str(r1.header('X-Rucio-Auth-Token'))
        headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)}

        # Export data
        r2 = TestApp(export_app.wsgifunc(*mw)).get('/', headers=headers2, expect_errors=True)
        exported_data = parse_response(r2.body)

        # Import data
        r3 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**exported_data))
        assert_equal(r3.status, 201)
Beispiel #36
0
    def test_list_replica_with_domain(self):
        """ REPLICA (CORE): Add and list file replicas forcing domain"""

        tmp_rse = rse_name_generator()
        add_rse(tmp_rse)

        protocols = [{'scheme': 'MOCK',
                      'hostname': 'localhost',
                      'port': 17,
                      'prefix': '/i/prefer/the/lan',
                      'impl': 'rucio.rse.protocols.mock.Default',
                      'domains': {
                          'lan': {'read': 1,
                                  'write': 1,
                                  'delete': 1},
                          'wan': {'read': 2,
                                  'write': 2,
                                  'delete': 2}}},
                     {'scheme': 'MOCK',
                      'hostname': 'localhost',
                      'port': 18,
                      'prefix': '/i/prefer/the/wan',
                      'impl': 'rucio.rse.protocols.mock.Default',
                      'domains': {
                          'lan': {'read': 2,
                                  'write': 2,
                                  'delete': 2},
                          'wan': {'read': 1,
                                  'write': 1,
                                  'delete': 1}}}, ]
        for p in protocols:
            add_protocol(tmp_rse, p)

        nbfiles = 3
        files = [{'scope': 'mock',
                  'name': 'file_%s' % generate_uuid(),
                  'bytes': 1234L,
                  'adler32': '01234567',
                  'meta': {'events': 1234}} for i in xrange(nbfiles)]
Beispiel #37
0
 def test_upload_repeated_file_dataset(self):
     """CLIENT(USER): Rucio upload repeated files to dataset"""
     # One of the files to upload is already in the dataset
     tmp_file1 = file_generator()
     tmp_file2 = file_generator()
     tmp_file3 = file_generator()
     tmp_file1_name = path.basename(tmp_file1)
     tmp_file3_name = path.basename(tmp_file3)
     tmp_dsn = self.user + ':DSet' + rse_name_generator(
     )  # something like mock:DSetMOCK_S0M37HING
     # Adding files to a new dataset
     cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3}'.format(
         self.def_rse, self.user, tmp_file1, tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     # upload the files to the dataset
     cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5}'.format(
         self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     remove(tmp_file1)
     remove(tmp_file2)
     remove(tmp_file3)
     # searching for the file in the new dataset
     cmd = 'rucio list-files {0}'.format(tmp_dsn)
     print(self.marker + cmd)
     exitcode, out, err = execute(cmd)
     print(out)
     print(err)
     # tmp_file1 must be in the dataset
     nose.tools.assert_not_equal(
         re.search("{0}:{1}".format(self.user, tmp_file1_name), out), None)
     # tmp_file3 must be in the dataset
     nose.tools.assert_not_equal(
         re.search("{0}:{1}".format(self.user, tmp_file3_name), out), None)
 def _make_rse(self,
               scheme,
               protocol_impl,
               parameters=None,
               add_rse_kwargs=None):
     rse_name = rse_name_generator()
     if add_rse_kwargs and 'vo' in add_rse_kwargs:
         rse_id = rse_core.add_rse(rse_name, **add_rse_kwargs)
     else:
         rse_id = rse_core.add_rse(rse_name,
                                   vo=self.vo,
                                   **(add_rse_kwargs or {}))
     if scheme and protocol_impl:
         protocol_parameters = {
             'scheme': scheme,
             'hostname': '%s.cern.ch' % rse_id,
             'port': 0,
             'prefix': '/test_%s/' % rse_id,
             'impl': protocol_impl,
             'domains': {
                 'wan': {
                     'read': 1,
                     'write': 1,
                     'delete': 1,
                     'third_party_copy_read': 1,
                     'third_party_copy_write': 1,
                 },
                 'lan': {
                     'read': 1,
                     'write': 1,
                     'delete': 1,
                 }
             }
         }
         protocol_parameters.update(parameters or {})
         rse_core.add_protocol(rse_id=rse_id, parameter=protocol_parameters)
     self.created_rses.append(rse_id)
     return rse_name, rse_id
Beispiel #39
0
    def test_scope_list(self):
        """ DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """

        # create some dummy data
        self.tmp_accounts = ['jdoe' for i in xrange(3)]
        self.tmp_scopes = [scope_name_generator() for i in xrange(3)]
        self.tmp_rses = [rse_name_generator() for i in xrange(3)]
        self.tmp_files = ['file_%s' % generate_uuid() for i in xrange(3)]
        self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in xrange(3)]
        self.tmp_containers = ['container_%s' % generate_uuid() for i in xrange(3)]

        # add dummy data to the catalogue
        for i in xrange(3):
            self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
            self.rse_client.add_rse(self.tmp_rses[i])
            self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1L, '0cc737eb')

        # put files in datasets
        for i in xrange(3):
            for j in xrange(3):
                files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1L, 'adler32': '0cc737eb'}]
                self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
                self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
Beispiel #40
0
    def test_scope_list(self):
        """ DATA IDENTIFIERS (CLIENT): Add, aggregate, and list data identifiers in a scope """

        # create some dummy data
        self.tmp_accounts = ['jdoe' for i in xrange(3)]
        self.tmp_scopes = [scope_name_generator() for i in xrange(3)]
        self.tmp_rses = [rse_name_generator() for i in xrange(3)]
        self.tmp_files = ['file_%s' % generate_uuid() for i in xrange(3)]
        self.tmp_datasets = ['dataset_%s' % generate_uuid() for i in xrange(3)]
        self.tmp_containers = ['container_%s' % generate_uuid() for i in xrange(3)]

        # add dummy data to the catalogue
        for i in xrange(3):
            self.scope_client.add_scope(self.tmp_accounts[i], self.tmp_scopes[i])
            self.rse_client.add_rse(self.tmp_rses[i])
            self.replica_client.add_replica(self.tmp_rses[i], self.tmp_scopes[i], self.tmp_files[i], 1L, '0cc737eb')

        # put files in datasets
        for i in xrange(3):
            for j in xrange(3):
                files = [{'scope': self.tmp_scopes[j], 'name': self.tmp_files[j], 'bytes': 1L, 'adler32': '0cc737eb'}]
                self.did_client.add_dataset(self.tmp_scopes[i], self.tmp_datasets[j])
                self.did_client.add_files_to_dataset(self.tmp_scopes[i], self.tmp_datasets[j], files)
Beispiel #41
0
    def test_atlas_archival_policy(self):
        """ UNDERTAKER (CORE): Test the atlas archival policy. """
        tmp_scope = 'mock'
        nbdatasets = 5
        nbfiles = 5

        rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
        add_rse(rse)

        set_account_limit('jdoe', get_rse_id(rse), -1)

        dsns2 = [{'name': 'dsn_%s' % generate_uuid(),
                  'scope': tmp_scope,
                  'type': 'DATASET',
                  'lifetime': -1,
                  'rules': [{'account': 'jdoe', 'copies': 1,
                             'rse_expression': rse,
                             'grouping': 'DATASET'}]} for i in range(nbdatasets)]

        add_dids(dids=dsns2, account='root')

        replicas = list()
        for dsn in dsns2:
            files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1,
                      'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)]
            attach_dids(scope=tmp_scope, name=dsn['name'], rse=rse, dids=files, account='root')
            replicas += files

        undertaker(worker_number=1, total_workers=1, once=True)

        for replica in replicas:
            assert(get_replica(scope=replica['scope'], name=replica['name'], rse=rse)['tombstone'] is None)

        for dsn in dsns2:
            assert(get_did(scope='archive', name=dsn['name'])['name'] == dsn['name'])
            assert(len([x for x in list_rules(filters={'scope': 'archive', 'name': dsn['name']})]) == 1)
Beispiel #42
0
    def setup(self):
        if config_get_bool('common',
                           'multi_vo',
                           raise_exception=False,
                           default=False):
            self.vo_header = {'X-Rucio-VO': 'tst'}
            self.vo = {'vo': 'tst'}
        else:
            self.vo_header = {}
            self.vo = {}

        # New RSE
        self.new_rse = rse_name_generator()

        # RSE 1 that already exists
        self.old_rse_1 = rse_name_generator()
        self.old_rse_id_1 = add_rse(self.old_rse_1,
                                    availability=1,
                                    region_code='DE',
                                    country_name='DE',
                                    deterministic=True,
                                    volatile=True,
                                    staging_area=True,
                                    time_zone='Europe',
                                    latitude='1',
                                    longitude='2',
                                    **self.vo)
        add_protocol(
            self.old_rse_id_1, {
                'scheme': 'scheme1',
                'hostname': 'hostname1',
                'port': 1000,
                'impl': 'TODO'
            })
        add_protocol(
            self.old_rse_id_1, {
                'scheme': 'scheme3',
                'hostname': 'hostname3',
                'port': 1000,
                'impl': 'TODO'
            })

        set_rse_limits(rse_id=self.old_rse_id_1,
                       name='MaxBeingDeletedFiles',
                       value='10')
        set_rse_limits(rse_id=self.old_rse_id_1,
                       name='MinFreeSpace',
                       value='10')
        add_rse_attribute(rse_id=self.old_rse_id_1,
                          key='attr1',
                          value='test10')
        add_rse_attribute(rse_id=self.old_rse_id_1,
                          key='lfn2pfn_algorithm',
                          value='test10')
        add_rse_attribute(rse_id=self.old_rse_id_1,
                          key='verify_checksum',
                          value=True)

        # RSE 2 that already exists
        self.old_rse_2 = rse_name_generator()
        self.old_rse_id_2 = add_rse(self.old_rse_2, **self.vo)

        # RSE 3 that already exists
        self.old_rse_3 = rse_name_generator()
        self.old_rse_id_3 = add_rse(self.old_rse_3, **self.vo)

        # RSE 4 that already exists
        self.old_rse_4 = rse_name_generator()
        self.old_rse_id_4 = add_rse(self.old_rse_4, **self.vo)

        # Distance that already exists
        add_distance(self.old_rse_id_1, self.old_rse_id_2)

        self.data1 = {
            'rses': {
                self.new_rse: {
                    'rse_type':
                    RSEType.TAPE,
                    'availability':
                    3,
                    'city':
                    'NewCity',
                    'region_code':
                    'CH',
                    'country_name':
                    'switzerland',
                    'staging_area':
                    False,
                    'time_zone':
                    'Europe',
                    'latitude':
                    1,
                    'longitude':
                    2,
                    'deterministic':
                    True,
                    'volatile':
                    False,
                    'protocols': [{
                        'scheme': 'scheme',
                        'hostname': 'hostname',
                        'port': 1000,
                        'impl': 'impl'
                    }],
                    'attributes': {
                        'attr1': 'test'
                    },
                    'MinFreeSpace':
                    20000,
                    'lfn2pfn_algorithm':
                    'hash2',
                    'verify_checksum':
                    False,
                    'availability_delete':
                    True,
                    'availability_read':
                    False,
                    'availability_write':
                    True
                },
                self.old_rse_1: {
                    'rse_type':
                    RSEType.TAPE,
                    'deterministic':
                    False,
                    'volatile':
                    False,
                    'region_code':
                    'US',
                    'country_name':
                    'US',
                    'staging_area':
                    False,
                    'time_zone':
                    'Asia',
                    'longitude':
                    5,
                    'city':
                    'City',
                    'availability':
                    2,
                    'latitude':
                    10,
                    'protocols': [{
                        'scheme': 'scheme1',
                        'hostname': 'hostname1',
                        'port': 1000,
                        'prefix': 'prefix',
                        'impl': 'impl1'
                    }, {
                        'scheme': 'scheme2',
                        'hostname': 'hostname2',
                        'port': 1001,
                        'impl': 'impl'
                    }],
                    'attributes': {
                        'attr1': 'test1',
                        'attr2': 'test2'
                    },
                    'MinFreeSpace':
                    10000,
                    'MaxBeingDeletedFiles':
                    1000,
                    'verify_checksum':
                    False,
                    'lfn2pfn_algorithm':
                    'hash3',
                    'availability_delete':
                    False,
                    'availability_read':
                    False,
                    'availability_write':
                    True
                },
                self.old_rse_2: {},
                self.old_rse_3: {}
            },
            'distances': {
                self.old_rse_1: {
                    self.old_rse_2: {
                        'src_rse': self.old_rse_1,
                        'dest_rse': self.old_rse_2,
                        'ranking': 10
                    },
                    self.old_rse_3: {
                        'src_rse': self.old_rse_1,
                        'dest_rse': self.old_rse_3,
                        'ranking': 4
                    }
                }
            }
        }

        self.data2 = {'rses': {self.new_rse: {'rse': self.new_rse}}}
        self.data3 = {'distances': {}}
Beispiel #43
0
def protocols_setup(vo):
    rse_info = copy.deepcopy(base_rse_info)

    files = [{
        'scope': InternalScope('mock', vo=vo),
        'name': 'element_0',
        'bytes': 1234,
        'adler32': 'deadbeef'
    }]
    root = InternalAccount('root', vo=vo)

    for idx in range(len(rse_info)):
        rse_info[idx]['name'] = '%s_%s' % (rse_info[idx]['site'],
                                           rse_name_generator())
        rse_info[idx]['id'] = add_rse(rse_info[idx]['name'], vo=vo)
        add_rse_attribute(rse_id=rse_info[idx]['id'],
                          key='site',
                          value=base_rse_info[idx]['site'])
        add_replicas(rse_id=rse_info[idx]['id'], files=files, account=root)

    # invalidate cache for parse_expression('site=…')
    rse_expression_parser.REGION.invalidate()

    # check sites
    for idx in range(len(rse_info)):
        site_rses = rse_expression_parser.parse_expression(
            'site=' + base_rse_info[idx]['site'])
        assert len(site_rses) > 0
        assert rse_info[idx]['id'] in [rse['id'] for rse in site_rses]

    add_protocol(
        rse_info[0]['id'], {
            'scheme': schemes[0],
            'hostname': ('root.%s' % base_rse_info[0]['address']),
            'port': 1409,
            'prefix': '//test/chamber/',
            'impl': 'rucio.rse.protocols.xrootd.Default',
            'domains': {
                'lan': {
                    'read': 1,
                    'write': 1,
                    'delete': 1
                },
                'wan': {
                    'read': 1,
                    'write': 1,
                    'delete': 1
                }
            }
        })
    add_protocol(
        rse_info[0]['id'], {
            'scheme': schemes[2],
            'hostname': ('davs.%s' % base_rse_info[0]['address']),
            'port': 443,
            'prefix': '/test/chamber/',
            'impl': 'rucio.rse.protocols.gfal.Default',
            'domains': {
                'lan': {
                    'read': 2,
                    'write': 2,
                    'delete': 2
                },
                'wan': {
                    'read': 2,
                    'write': 2,
                    'delete': 2
                }
            }
        })
    add_protocol(
        rse_info[0]['id'], {
            'scheme': schemes[1],
            'hostname': ('gsiftp.%s' % base_rse_info[0]['address']),
            'port': 8446,
            'prefix': '/test/chamber/',
            'impl': 'rucio.rse.protocols.gfal.Default',
            'domains': {
                'lan': {
                    'read': 0,
                    'write': 0,
                    'delete': 0
                },
                'wan': {
                    'read': 3,
                    'write': 3,
                    'delete': 3
                }
            }
        })

    add_protocol(
        rse_info[1]['id'], {
            'scheme': schemes[1],
            'hostname': ('gsiftp.%s' % base_rse_info[1]['address']),
            'port': 8446,
            'prefix': '/lambda/complex/',
            'impl': 'rucio.rse.protocols.gfal.Default',
            'domains': {
                'lan': {
                    'read': 2,
                    'write': 2,
                    'delete': 2
                },
                'wan': {
                    'read': 1,
                    'write': 1,
                    'delete': 1
                }
            }
        })
    add_protocol(
        rse_info[1]['id'], {
            'scheme': schemes[2],
            'hostname': ('davs.%s' % base_rse_info[1]['address']),
            'port': 443,
            'prefix': '/lambda/complex/',
            'impl': 'rucio.rse.protocols.gfal.Default',
            'domains': {
                'lan': {
                    'read': 0,
                    'write': 0,
                    'delete': 0
                },
                'wan': {
                    'read': 2,
                    'write': 2,
                    'delete': 2
                }
            }
        })
    add_protocol(
        rse_info[1]['id'], {
            'scheme': schemes[0],
            'hostname': ('root.%s' % base_rse_info[1]['address']),
            'port': 1409,
            'prefix': '//lambda/complex/',
            'impl': 'rucio.rse.protocols.xrootd.Default',
            'domains': {
                'lan': {
                    'read': 1,
                    'write': 1,
                    'delete': 1
                },
                'wan': {
                    'read': 3,
                    'write': 3,
                    'delete': 3
                }
            }
        })

    yield {'files': files, 'rse_info': rse_info}

    for info in rse_info:
        delete_replicas(rse_id=info['id'], files=files)
        del_rse_attribute(rse_id=info['id'], key='site')
        del_rse(info['id'])
Beispiel #44
0
 def test_post_rse(self):
     """RSE (CURL): add RSE"""
     cmd = 'curl -s -i --cacert %s -H "X-Rucio-Account: root" -E %s -X GET %s/auth/x509 | grep X-Rucio-Auth-Token:' % (self.cacert, self.usercert, self.auth_host)
     exitcode, out, err = execute(cmd)
     nose.tools.assert_in('X-Rucio-Auth-Token', out)
     os.environ['RUCIO_TOKEN'] = out[len('X-Rucio-Auth-Token: '):-1]
     cmd = '''curl -s -i --cacert %s -H "X-Rucio-Auth-Token: $RUCIO_TOKEN" -X POST %s/rses/%s''' % (self.cacert, self.host, rse_name_generator())
     print self.marker + cmd
     exitcode, out, err = execute(cmd)
     print out
     nose.tools.assert_in('201 Created', out)