Exemplo n.º 1
0
    def test_list_rules_states(self):
        """ SUBSCRIPTION (API): Test listing of rule states for subscription """
        tmp_scope = 'mock_' + uuid()[:8]
        add_scope(tmp_scope, 'root')
        site_a = 'RSE%s' % uuid().upper()
        site_b = 'RSE%s' % uuid().upper()

        add_rse(site_a)
        add_rse(site_b)

        # Add quota
        set_account_limit('root', get_rse_id(site_a), -1)
        set_account_limit('root', get_rse_id(site_b), -1)

        # add a new dataset
        dsn = 'dataset-%s' % uuid()
        add_did(scope=tmp_scope, name=dsn,
                type=DIDType.DATASET, account='root')

        subscription_name = uuid()
        subid = add_subscription(name=subscription_name, account='root', filter={'account': ['root', ], 'scope': [tmp_scope, ]},
                                 replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root')

        # Add two rules
        add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
        add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)

        for rule in list_subscription_rule_states(account='root', name=subscription_name):
            assert_equal(rule[3], 2)
Exemplo n.º 2
0
    def test_run_transmogrifier(self):
        """ SUBSCRIPTION (DAEMON): Test the transmogrifier and the split_rule mode """
        rse1, _ = self.rse_factory.make_mock_rse()
        rse2, _ = self.rse_factory.make_mock_rse()
        rse3, _ = self.rse_factory.make_mock_rse()
        rse_expression = '%s|%s|%s' % (rse1, rse2, rse3)
        tmp_scope = InternalScope('mock_' + uuid()[:8], **self.vo)
        root = InternalAccount('root', **self.vo)
        add_scope(tmp_scope, root)
        subscription_name = uuid()
        dsn = 'dataset-%s' % uuid()
        add_did(scope=tmp_scope,
                name=dsn,
                did_type=DIDType.DATASET,
                account=root)

        subid = self.sub_client.add_subscription(name=subscription_name,
                                                 account='root',
                                                 filter_={
                                                     'scope': [
                                                         tmp_scope.external,
                                                     ],
                                                     'pattern': 'dataset-.*',
                                                     'split_rule': True
                                                 },
                                                 replication_rules=[{
                                                     'lifetime':
                                                     86400,
                                                     'rse_expression':
                                                     rse_expression,
                                                     'copies':
                                                     2,
                                                     'activity':
                                                     'Data Brokering'
                                                 }],
                                                 lifetime=None,
                                                 retroactive=0,
                                                 dry_run=0,
                                                 comments='Ni ! Ni!',
                                                 priority=1)
        run(threads=1, bulk=1000000, once=True)
        rules = [
            rule for rule in self.did_client.list_did_rules(
                scope=tmp_scope.external, name=dsn)
            if str(rule['subscription_id']) == str(subid)
        ]
        assert len(rules) == 2
        set_new_dids([
            {
                'scope': tmp_scope,
                'name': dsn
            },
        ], 1)
        run(threads=1, bulk=1000000, once=True)
        rules = [
            rule for rule in self.did_client.list_did_rules(
                scope=tmp_scope.external, name=dsn)
            if str(rule['subscription_id']) == str(subid)
        ]
        assert len(rules) == 2
Exemplo n.º 3
0
 def test_list_scopes(self):
     """ SCOPE (CORE): List scopes """
     for scope in self.scopes:
         add_scope(scope=scope, account='jdoe')
     scopes = get_scopes(account='jdoe')
     for s in scopes:
         assert_in(s, scopes)
Exemplo n.º 4
0
    def test_list_rules_states(self):
        """ SUBSCRIPTION (API): Test listing of rule states for subscription """
        tmp_scope = 'mock_' + uuid()[:8]
        add_scope(tmp_scope, 'root')
        site_a = 'RSE%s' % uuid().upper()
        site_b = 'RSE%s' % uuid().upper()

        add_rse(site_a)
        add_rse(site_b)

        # add a new dataset
        dsn = 'dataset-%s' % uuid()
        add_did(scope=tmp_scope, name=dsn,
                type=DIDType.DATASET, account='root')

        subscription_name = uuid()
        id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'},
                              replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment')

        subscriptions = list_subscriptions(name=subscription_name, account='root')
        # workaround until add_subscription returns the id
        id = None
        for s in subscriptions:
            id = s['id']

        # Add two rules
        add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
        add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)

        for r in list_subscription_rule_states(account='root', name=subscription_name):
            assert_equal(r[3], 2)
Exemplo n.º 5
0
 def test_list_scopes(self):
     """ SCOPE (CORE): List scopes """
     for scope in self.scopes:
         add_scope(scope=scope, account=self.jdoe)
     scopes = get_scopes(account=self.jdoe)
     for scope in scopes:
         assert scope in scopes
Exemplo n.º 6
0
def __setup_scopes_for_vos(*vos):
    scope_uuid = str(generate_uuid()).lower()[:16]
    scope_name = 'shr_%s' % scope_uuid
    created_scopes = []
    for vo in vos:
        scope = InternalScope(scope_name, vo=vo)
        scope_core.add_scope(scope, InternalAccount('root', vo=vo))
        created_scopes.append(scope)
    return scope_name, created_scopes
Exemplo n.º 7
0
 def create_scopes(vos, account_name=None):
     scope_uuid = str(generate_uuid()).lower()[:16]
     scope_name = 'shr_%s' % scope_uuid
     created_scopes = []
     for vo in vos:
         scope = InternalScope(scope_name, vo=vo)
         add_scope(scope, InternalAccount(account_name if account_name else 'root', vo=vo))
         created_scopes.append(scope)
     return scope_name, created_scopes
Exemplo n.º 8
0
    def test_run_transmogrifier(self):
        """ SUBSCRIPTION (DAEMON): Test the transmogrifier and the split_rule mode """
        tmp_scope = 'mock_' + uuid()[:8]
        add_scope(tmp_scope, 'root')
        subscription_name = uuid()
        dsn = 'dataset-%s' % uuid()
        add_did(scope=tmp_scope,
                name=dsn,
                type=DIDType.DATASET,
                account='root')

        subid = self.sub_client.add_subscription(name=subscription_name,
                                                 account='root',
                                                 filter={
                                                     'scope': [
                                                         tmp_scope,
                                                     ],
                                                     'pattern': 'dataset-.*',
                                                     'split_rule': True
                                                 },
                                                 replication_rules=[{
                                                     'lifetime':
                                                     86400,
                                                     'rse_expression':
                                                     'MOCK-POSIX|MOCK2|MOCK3',
                                                     'copies':
                                                     2,
                                                     'activity':
                                                     'Data Brokering'
                                                 }],
                                                 lifetime=None,
                                                 retroactive=0,
                                                 dry_run=0,
                                                 comments='Ni ! Ni!',
                                                 priority=1)
        run(threads=1, bulk=1000000, once=True)
        rules = [
            rule for rule in self.did_client.list_did_rules(scope=tmp_scope,
                                                            name=dsn)
            if str(rule['subscription_id']) == str(subid)
        ]
        assert_equal(len(rules), 2)
        set_new_dids([
            {
                'scope': tmp_scope,
                'name': dsn
            },
        ], 1)
        run(threads=1, bulk=1000000, once=True)
        rules = [
            rule for rule in self.did_client.list_did_rules(scope=tmp_scope,
                                                            name=dsn)
            if str(rule['subscription_id']) == str(subid)
        ]
        assert_equal(len(rules), 2)
Exemplo n.º 9
0
 def test_permission_add_did(self):
     """ PERMISSION(CORE): Check permission to add a did"""
     scope = scope_name_generator()
     add_scope(scope=InternalScope(scope), account=InternalAccount('root'))
     assert_true(
         has_permission(issuer='panda',
                        action='add_did',
                        kwargs={'scope': scope}))
     assert_false(
         has_permission(issuer='spock',
                        action='add_did',
                        kwargs={'scope': scope}))
Exemplo n.º 10
0
def add_scope(scope, account, issuer):
    """
    Creates a scope for an account.

    :param account: The account name.
    :param scope: The scope identifier.
    :param issuer: The issuer account.
    """

    validate_schema(name='scope', obj=scope)

    kwargs = {'scope': scope, 'account': account}
    if not rucio.api.permission.has_permission(issuer=issuer, action='add_scope', kwargs=kwargs):
        raise rucio.common.exception.AccessDenied('Account %s can not add scope' % (issuer))

    core_scope.add_scope(scope, account)
Exemplo n.º 11
0
def test_list_rules_states(vo, rest_client, auth_token):
    """ SUBSCRIPTION (REST): Test listing of rule states for subscription """
    tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo)
    root = InternalAccount('root', vo=vo)
    add_scope(tmp_scope, root)
    site_a = 'RSE%s' % uuid().upper()
    site_b = 'RSE%s' % uuid().upper()

    site_a_id = add_rse(site_a, vo=vo)
    site_b_id = add_rse(site_b, vo=vo)

    # Add quota
    set_local_account_limit(root, site_a_id, -1)
    set_local_account_limit(root, site_b_id, -1)

    # add a new dataset
    dsn = 'dataset-%s' % uuid()
    add_did(scope=tmp_scope, name=dsn,
            did_type=DIDType.DATASET, account=root)

    subscription_name = uuid()
    subid = add_subscription(name=subscription_name,
                             account='root',
                             filter_={'account': ['root', ], 'scope': [tmp_scope.external, ]},
                             replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}],
                             lifetime=100000,
                             retroactive=0,
                             dry_run=0,
                             comments='We want a shrubbery',
                             issuer='root',
                             vo=vo)

    # Add two rules
    add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
    add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)

    response = rest_client.get('/subscriptions/%s/%s/Rules/States' % ('root', subscription_name), headers=headers(auth(auth_token)))
    assert response.status_code == 200

    rulestates = None
    for line in response.get_data(as_text=True).split('\n'):
        if line:
            rulestates = loads(line)
            if rulestates[1] == subscription_name:
                break
    assert rulestates is not None
    assert rulestates[3] == 2
Exemplo n.º 12
0
    def test_list_rules_states(self):
        """ SUBSCRIPTION (REST): Test listing of rule states for subscription """
        tmp_scope = 'mock_' + uuid()[:8]
        add_scope(tmp_scope, 'root')
        mw = []
        site_a = 'RSE%s' % uuid().upper()
        site_b = 'RSE%s' % uuid().upper()

        add_rse(site_a)
        add_rse(site_b)

        # add a new dataset
        dsn = 'dataset-%s' % uuid()
        add_did(scope=tmp_scope, name=dsn,
                type=DIDType.DATASET, account='root')

        subscription_name = uuid()
        id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'},
                              replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery')

        subscriptions = list_subscriptions(name=subscription_name, account='root')

        # workaround until add_subscription returns the id
        id = None
        for s in subscriptions:
            id = s['id']

        # Add two rules
        add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
        add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)

        headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
        r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)

        assert_equal(r1.status, 200)
        token = str(r1.header('X-Rucio-Auth-Token'))

        headers2 = {'X-Rucio-Auth-Token': str(token)}
        r2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True)

        for line in r2.body.split('\n'):
            print line
            rs = loads(line)
            if rs[1] == subscription_name:
                break
        assert_equal(rs[3], 2)
Exemplo n.º 13
0
def add_scope(scope, account, issuer):
    """
    Creates a scope for an account.

    :param account: The account name.
    :param scope: The scope identifier.
    :param issuer: The issuer account.
    """

    validate_schema(name='scope', obj=scope)

    kwargs = {'scope': scope, 'account': account}
    if not rucio.api.permission.has_permission(
            issuer=issuer, action='add_scope', kwargs=kwargs):
        raise rucio.common.exception.AccessDenied(
            'Account %s can not add scope' % (issuer))

    core_scope.add_scope(scope, account)
Exemplo n.º 14
0
    def test_list_rules_states(self):
        """ SUBSCRIPTION (REST): Test listing of rule states for subscription """
        tmp_scope = InternalScope('mock_' + uuid()[:8])
        root = InternalAccount('root')
        add_scope(tmp_scope, root)
        mw = []
        site_a = 'RSE%s' % uuid().upper()
        site_b = 'RSE%s' % uuid().upper()

        site_a_id = add_rse(site_a)
        site_b_id = add_rse(site_b)

        # Add quota
        set_account_limit(root, site_a_id, -1)
        set_account_limit(root, site_b_id, -1)

        # add a new dataset
        dsn = 'dataset-%s' % uuid()
        add_did(scope=tmp_scope, name=dsn,
                type=DIDType.DATASET, account=root)

        subscription_name = uuid()
        subid = add_subscription(name=subscription_name, account='root', filter={'account': ['root', ], 'scope': [tmp_scope.external, ]},
                                 replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery', issuer='root')

        # Add two rules
        add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
        add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)

        headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
        res1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)

        assert_equal(res1.status, 200)
        token = str(res1.header('X-Rucio-Auth-Token'))

        headers2 = {'X-Rucio-Auth-Token': str(token)}
        res2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True)

        for line in res2.body.split('\n'):
            print(line)
            rs = loads(line)
            if rs[1] == subscription_name:
                break
        assert_equal(rs[3], 2)
Exemplo n.º 15
0
def add_scope(scope, account, issuer, vo='def', session=None):
    """
    Creates a scope for an account.

    :param account: The account name.
    :param scope: The scope identifier.
    :param issuer: The issuer account.
    :param vo: The VO to act on.
    :param session: The database session in use.
    """

    validate_schema(name='scope', obj=scope, vo=vo)

    kwargs = {'scope': scope, 'account': account}
    if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='add_scope', kwargs=kwargs, session=session):
        raise rucio.common.exception.AccessDenied('Account %s can not add scope' % (issuer))

    scope = InternalScope(scope, vo=vo)
    account = InternalAccount(account, vo=vo)

    core_scope.add_scope(scope, account, session=session)
Exemplo n.º 16
0
def test_overlapping_did_names(rse_factory, did_factory, download_client,
                               root_account, mock_scope, vo):
    """
    Downloading two different did with different scope but same name to the same directory must fail
    """
    rse, _ = rse_factory.make_posix_rse()
    scope1 = mock_scope
    scope2 = InternalScope(scope_name_generator(), vo=vo)
    scope_core.add_scope(scope2, root_account)
    did1 = did_factory.upload_test_file(rse, scope=scope1)
    did2 = did_factory.upload_test_file(rse, scope=scope2, name=did1['name'])
    dataset = did_factory.make_dataset()
    did_core.attach_dids(dids=[did1, did2], account=root_account, **dataset)

    did1_str = '%s:%s' % (did1['scope'], did1['name'])
    did2_str = '%s:%s' % (did2['scope'], did2['name'])
    dataset1_did_str = '%s:%s' % (dataset['scope'], dataset['name'])

    with TemporaryDirectory() as tmp_dir:
        with pytest.raises(RucioException):
            download_client.download_dids([{
                'did': dataset1_did_str,
                'base_dir': tmp_dir,
                'no_subdir': True
            }])

    with TemporaryDirectory() as tmp_dir:
        with pytest.raises(RucioException):
            download_client.download_dids([{
                'did': did1_str,
                'base_dir': tmp_dir,
                'no_subdir': True
            }, {
                'did': did2_str,
                'base_dir': tmp_dir,
                'no_subdir': True
            }])
Exemplo n.º 17
0
def test_reaper():
    """ REAPER2 (DAEMON): Test the reaper2 daemon."""
    if config_get_bool('common',
                       'multi_vo',
                       raise_exception=False,
                       default=False):
        vo = {
            'vo': config_get('client',
                             'vo',
                             raise_exception=False,
                             default='tst')
        }
        new_vo = {'vo': 'new'}
        if not vo_core.vo_exists(**new_vo):
            vo_core.add_vo(description='Test',
                           email='*****@*****.**',
                           **new_vo)
        if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)):
            scope_core.add_scope(InternalScope('data13_hip', **new_vo),
                                 InternalAccount('root', **new_vo))
        nb_rses = 2
    else:
        vo = {}
        new_vo = {}
        nb_rses = 1

    mock_protocol = {
        'scheme': 'MOCK',
        'hostname': 'localhost',
        'port': 123,
        'prefix': '/test/reaper',
        'impl': 'rucio.rse.protocols.mock.Default',
        'domains': {
            'lan': {
                'read': 1,
                'write': 1,
                'delete': 1
            },
            'wan': {
                'read': 1,
                'write': 1,
                'delete': 1
            }
        }
    }

    nb_files = 30
    file_size = 2147483648  # 2G

    rse_names = []
    all_file_names = []
    for j in range(nb_rses):
        rse_name = rse_name_generator()
        rse_names.append(rse_name)
        rse_id = rse_core.add_rse(rse_name, **vo)
        rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
        if new_vo:
            rse_id_new = rse_core.add_rse(rse_name, **new_vo)
            rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol)

        file_names = []
        for i in range(nb_files):
            file_name = 'lfn' + generate_uuid()
            file_names.append(file_name)
            replica_core.add_replica(rse_id=rse_id,
                                     scope=InternalScope('data13_hip', **vo),
                                     name=file_name,
                                     bytes=file_size,
                                     tombstone=datetime.utcnow() -
                                     timedelta(days=1),
                                     account=InternalAccount('root', **vo),
                                     adler32=None,
                                     md5=None)
            if new_vo:
                replica_core.add_replica(
                    rse_id=rse_id_new,
                    scope=InternalScope('data13_hip', **new_vo),
                    name=file_name,
                    bytes=file_size,
                    tombstone=datetime.utcnow() - timedelta(days=1),
                    account=InternalAccount('root', **new_vo),
                    adler32=None,
                    md5=None)

        all_file_names.append(file_names)
        rse_core.set_rse_usage(rse_id=rse_id,
                               source='storage',
                               used=nb_files * file_size,
                               free=800)
        rse_core.set_rse_limits(rse_id=rse_id,
                                name='MinFreeSpace',
                                value=10737418240)
        rse_core.set_rse_limits(rse_id=rse_id,
                                name='MaxBeingDeletedFiles',
                                value=10)
        if new_vo:
            rse_core.set_rse_usage(rse_id=rse_id_new,
                                   source='storage',
                                   used=nb_files * file_size,
                                   free=800)
            rse_core.set_rse_limits(rse_id=rse_id_new,
                                    name='MinFreeSpace',
                                    value=10737418240)
            rse_core.set_rse_limits(rse_id=rse_id_new,
                                    name='MaxBeingDeletedFiles',
                                    value=10)

    if not vo:
        reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
        reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
        assert len(
            list(
                replica_core.list_replicas(
                    dids=[{
                        'scope': InternalScope('data13_hip', **vo),
                        'name': n
                    } for n in all_file_names[0]],
                    rse_expression=rse_name))) == nb_files - 5
    else:
        # Check we reap all VOs by default
        reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
        reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
        assert len(
            list(
                replica_core.list_replicas(
                    dids=[{
                        'scope': InternalScope('data13_hip', **vo),
                        'name': n
                    } for n in all_file_names[0]],
                    rse_expression=rse_names[0]))) == nb_files - 5
        assert len(
            list(
                replica_core.list_replicas(
                    dids=[{
                        'scope': InternalScope('data13_hip', **new_vo),
                        'name': n
                    } for n in all_file_names[0]],
                    rse_expression=rse_names[0]))) == nb_files - 5
        # Check we don't affect a second VO that isn't specified
        reaper(once=True,
               rses=[],
               include_rses=rse_names[1],
               exclude_rses=[],
               vos=['new'])
        reaper(once=True,
               rses=[],
               include_rses=rse_names[1],
               exclude_rses=[],
               vos=['new'])
        assert len(
            list(
                replica_core.list_replicas(
                    dids=[{
                        'scope': InternalScope('data13_hip', **vo),
                        'name': n
                    } for n in all_file_names[1]],
                    rse_expression=rse_names[1]))), nb_files
        assert len(
            list(
                replica_core.list_replicas(
                    dids=[{
                        'scope': InternalScope('data13_hip', **new_vo),
                        'name': n
                    } for n in all_file_names[1]],
                    rse_expression=rse_names[1]))), nb_files - 5
Exemplo n.º 18
0
def test_reaper():
    """ REAPER2 (DAEMON): Test the reaper2 daemon."""
    if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
        vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
        new_vo = {'vo': 'new'}
        if not vo_core.vo_exists(**new_vo):
            vo_core.add_vo(description='Test', email='*****@*****.**', **new_vo)
        if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)):
            scope_core.add_scope(InternalScope('data13_hip', **new_vo), InternalAccount('root', **new_vo))
        nb_rses = 2
    else:
        vo = {}
        new_vo = {}
        nb_rses = 1

    mock_protocol = {'scheme': 'MOCK',
                     'hostname': 'localhost',
                     'port': 123,
                     'prefix': '/test/reaper',
                     'impl': 'rucio.rse.protocols.mock.Default',
                     'domains': {
                         'lan': {'read': 1,
                                 'write': 1,
                                 'delete': 1},
                         'wan': {'read': 1,
                                 'write': 1,
                                 'delete': 1}}}

    nb_files = 250
    file_size = 200  # 2G

    rse_names = []
    all_file_names = []
    for j in range(nb_rses):
        rse_name = rse_name_generator()
        rse_names.append(rse_name)
        rse_id = rse_core.add_rse(rse_name, **vo)
        rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
        if new_vo:
            rse_id_new = rse_core.add_rse(rse_name, **new_vo)
            rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol)

        file_names = []
        for i in range(nb_files):
            file_name = 'lfn' + generate_uuid()
            file_names.append(file_name)
            replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo),
                                     name=file_name, bytes=file_size,
                                     tombstone=datetime.utcnow() - timedelta(days=1),
                                     account=InternalAccount('root', **vo), adler32=None, md5=None)
            if new_vo:
                replica_core.add_replica(rse_id=rse_id_new, scope=InternalScope('data13_hip', **new_vo),
                                         name=file_name, bytes=file_size,
                                         tombstone=datetime.utcnow() - timedelta(days=1),
                                         account=InternalAccount('root', **new_vo), adler32=None, md5=None)

        all_file_names.append(file_names)
        rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1)
        rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=50 * file_size)
        # rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
        if new_vo:
            rse_core.set_rse_usage(rse_id=rse_id_new, source='storage', used=nb_files * file_size, free=1)
            rse_core.set_rse_limits(rse_id=rse_id_new, name='MinFreeSpace', value=50 * file_size)
            # rse_core.set_rse_limits(rse_id=rse_id_new, name='MaxBeingDeletedFiles', value=10)

    from rucio.daemons.reaper.reaper2 import REGION
    REGION.invalidate()
    if not vo:
        assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
                                                   rse_expression=rse_name))) == nb_files
        # Check first if the reaper does not delete anything if no space is needed
        rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=323000000000)
        reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
        assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
                                                   rse_expression=rse_name))) == nb_files
        # Now put it over threshold and delete
        rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1)
        from rucio.daemons.reaper.reaper2 import REGION
        REGION.invalidate()
        reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
        reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
        assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
                                                   rse_expression=rse_name))) == 200
    else:
        # Check we reap all VOs by default
        reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
        reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[])
        assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]],
                                                   rse_expression=rse_names[0]))) == 200
        assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **new_vo), 'name': n} for n in all_file_names[0]],
                                                   rse_expression=rse_names[0]))) == 200
Exemplo n.º 19
0
    def test_list_rules_states(self):
        """ SUBSCRIPTION (API): Test listing of rule states for subscription """
        tmp_scope = InternalScope('mock_' + uuid()[:8], **self.vo)
        root = InternalAccount('root', **self.vo)
        add_scope(tmp_scope, root)
        rse1, _ = self.rse_factory.make_mock_rse()
        rse2, _ = self.rse_factory.make_mock_rse()
        rse_expression = '%s|%s' % (rse1, rse2)

        rse3, _ = self.rse_factory.make_mock_rse()
        rse4, _ = self.rse_factory.make_mock_rse()

        # add a new dataset
        dsn = 'dataset-%s' % uuid()
        add_did(scope=tmp_scope,
                name=dsn,
                did_type=DIDType.DATASET,
                account=root)

        subscription_name = uuid()
        subid = add_subscription(name=subscription_name,
                                 account='root',
                                 filter_={
                                     'account': [
                                         'root',
                                     ],
                                     'scope': [
                                         tmp_scope.external,
                                     ]
                                 },
                                 replication_rules=[{
                                     'lifetime':
                                     86400,
                                     'rse_expression':
                                     rse_expression,
                                     'copies':
                                     2,
                                     'activity':
                                     'Data Brokering'
                                 }],
                                 lifetime=100000,
                                 retroactive=0,
                                 dry_run=0,
                                 comments='This is a comment',
                                 issuer='root',
                                 **self.vo)

        # Add two rules
        add_rule(dids=[{
            'scope': tmp_scope,
            'name': dsn
        }],
                 account=root,
                 copies=1,
                 rse_expression=rse3,
                 grouping='NONE',
                 weight=None,
                 lifetime=None,
                 locked=False,
                 subscription_id=subid)
        add_rule(dids=[{
            'scope': tmp_scope,
            'name': dsn
        }],
                 account=root,
                 copies=1,
                 rse_expression=rse4,
                 grouping='NONE',
                 weight=None,
                 lifetime=None,
                 locked=False,
                 subscription_id=subid)

        for rule in list_subscription_rule_states(account='root',
                                                  name=subscription_name,
                                                  **self.vo):
            assert rule[3] == 2
Exemplo n.º 20
0
 def test_permission_add_did(self):
     """ PERMISSION(CORE): Check permission to add a did"""
     scope = scope_name_generator()
     add_scope(scope=scope, account='root')
     assert_true(has_permission(issuer='panda', action='add_did', kwargs={'scope': scope}))
     assert_false(has_permission(issuer='spock', action='add_did', kwargs={'scope': scope}))
Exemplo n.º 21
0
 def test_is_scope_owner(self):
     """ SCOPE (CORE): Is scope owner """
     scope = scope_name_generator()
     add_scope(scope=scope, account='jdoe')
     anwser = is_scope_owner(scope=scope, account='jdoe')
     assert_equal(anwser, True)
Exemplo n.º 22
0
 def test_is_scope_owner(self):
     """ SCOPE (CORE): Is scope owner """
     scope = InternalScope(scope_name_generator(), **self.vo)
     add_scope(scope=scope, account=self.jdoe)
     anwser = is_scope_owner(scope=scope, account=self.jdoe)
     assert anwser is True
Exemplo n.º 23
0
 def test_is_scope_owner(self):
     """ SCOPE (CORE): Is scope owner """
     scope = scope_name_generator()
     add_scope(scope=scope, account='jdoe')
     anwser = is_scope_owner(scope=scope, account='jdoe')
     assert_equal(anwser, True)