Пример #1
0
def add_sync_rule(rse, scope, block, account, session=None):
    rse_id = get_rse_id(rse=rse, session=session)
    scope = InternalScope(scope)
    account = InternalAccount(account)

    files = []
    for file in list_files(scope, block, long=False, session=session):
        try:
            update_replicas_states(
                replicas=[
                    {
                        "scope": scope,
                        "name": file["name"],
                        "rse_id": rse_id,
                        "state": "A",
                    }
                ],
                add_tombstone=False,
                session=session,
            )
        except ReplicaNotFound:
            add_replicas(
                rse_id=rse_id,
                files=[
                    {
                        "scope": scope,
                        "name": file["name"],
                        "bytes": file["bytes"],
                        "adler32": file["adler32"],
                    }
                ],
                account=account,
                ignore_availability=True,
                session=session,
            )

    rules = add_rule(
        dids=[{"scope": scope, "name": block}],
        copies=1,
        rse_expression=rse,
        weight=None,
        lifetime=None,
        grouping="DATASET",
        account=account,
        locked=False,
        subscription_id=None,
        source_replica_expression=None,
        activity="Data Consolidation",
        notify=None,
        purge_replicas=False,
        ignore_availability=True,
        comment=None,
        ask_approval=False,
        asynchronous=False,
        priority=3,
        split_container=False,
        meta=json.dumps({"phedex_group": "DataOps", "phedex_custodial": True}),
        session=session,
    )
    return rules[0]
Пример #2
0
def list_files(scope, name, long):
    """
    List data identifier file contents.

    :param scope: The scope name.
    :param name: The data identifier name.
    :param long:       A boolean to choose if GUID is returned or not.
    """

    return did.list_files(scope=scope, name=name, long=long)
Пример #3
0
Файл: did.py Проект: kbg/rucio
def list_files(scope, name, long):
    """
    List data identifier file contents.

    :param scope: The scope name.
    :param name: The data identifier name.
    :param long:       A boolean to choose if GUID is returned or not.
    """

    return did.list_files(scope=scope, name=name, long=long)
Пример #4
0
def list_files(scope, name, long):
    """
    List data identifier file contents.

    :param scope: The scope name.
    :param name: The data identifier name.
    :param long:       A boolean to choose if GUID is returned or not.
    """

    scope = InternalScope(scope)

    dids = did.list_files(scope=scope, name=name, long=long)

    for d in dids:
        yield api_update_return_dict(d)
Пример #5
0
    def test_delete_replicas_from_datasets(self):
        """ REPLICA (CORE): Delete replicas from dataset """
        tmp_scope = 'mock'
        tmp_dsn1 = 'dsn_%s' % generate_uuid()
        tmp_dsn2 = 'dsn_%s' % generate_uuid()
        nbfiles = 5
        files1 = [{
            'scope': tmp_scope,
            'name': 'file_%s' % generate_uuid(),
            'bytes': 1,
            'adler32': '0cc737eb',
            'meta': {
                'events': 10
            }
        } for _ in range(nbfiles)]

        add_did(scope=tmp_scope,
                name=tmp_dsn1,
                type=DIDType.DATASET,
                account='root')
        add_did(scope=tmp_scope,
                name=tmp_dsn2,
                type=DIDType.DATASET,
                account='root')

        attach_dids(scope=tmp_scope,
                    name=tmp_dsn1,
                    rse='MOCK',
                    dids=files1,
                    account='root')
        attach_dids(scope=tmp_scope,
                    name=tmp_dsn2,
                    dids=files1,
                    account='root')

        set_status(scope=tmp_scope, name=tmp_dsn1, open=False)

        delete_replicas(rse='MOCK', files=files1)

        with assert_raises(DataIdentifierNotFound):
            get_did(scope=tmp_scope, name=tmp_dsn1)

        get_did(scope=tmp_scope, name=tmp_dsn2)

        assert_equal([f for f in list_files(scope=tmp_scope, name=tmp_dsn2)],
                     [])
Пример #6
0
def list_files(scope, name, long, vo='def', session=None):
    """
    List data identifier file contents.

    :param scope: The scope name.
    :param name: The data identifier name.
    :param long:       A boolean to choose if GUID is returned or not.
    :param vo: The VO to act on.
    :param session: The database session in use.
    """

    scope = InternalScope(scope, vo=vo)

    dids = did.list_files(scope=scope, name=name, long=long, session=session)

    for d in dids:
        yield api_update_return_dict(d, session=session)
Пример #7
0
def delete_sync_rule(rule_id, session=None):
    rule = get_rule(rule_id, session=session)
    if rule["did_type"] != DIDType.DATASET:
        raise RuntimeError("Rule applies to did with wrong type")
    block = rule["name"]
    try:
        rse_id = get_rse_id(rse=rule["rse_expression"], session=session)
    except RSENotFound:
        raise RuntimeError("Rule does not apply to a specific RSE")
    scope = rule["scope"]
    account = rule["account"]

    files = []
    for file in list_files(scope, block, long=False, session=session):
        files.append(
            {"scope": scope, "name": file["name"], "rse_id": rse_id, "state": "U"}
        )

    update_replicas_states(
        replicas=files, add_tombstone=False, session=session
    )
    delete_rule(rule_id=rule_id, purge_replicas=True, soft=False, session=session)
Пример #8
0
                    account='root')
        attach_dids(scope=tmp_scope,
                    name=tmp_dsn2,
                    dids=files1,
                    account='root')

        set_status(scope=tmp_scope, name=tmp_dsn1, open=False)

        delete_replicas(rse='MOCK', files=files1)

        with assert_raises(DataIdentifierNotFound):
            get_did(scope=tmp_scope, name=tmp_dsn1)

        get_did(scope=tmp_scope, name=tmp_dsn2)

        assert_equal([f for f in list_files(scope=tmp_scope, name=tmp_dsn2)],
                     [])

    def test_update_lock_counter(self):
        """ RSE (CORE): Test the update of a replica lock counter """
        rse = 'MOCK'
        tmp_scope = 'mock'
        tmp_file = 'file_%s' % generate_uuid()
        add_replica(rse=rse,
                    scope=tmp_scope,
                    name=tmp_file,
                    bytes=1L,
                    adler32='0cc737eb',
                    account='jdoe')

        values = (1, 1, 1, -1, -1, -1, 1, 1, -1)
Пример #9
0
        add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root')
        add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account='root')

        attach_dids(scope=tmp_scope, name=tmp_dsn1, rse='MOCK', dids=files1, account='root')
        attach_dids(scope=tmp_scope, name=tmp_dsn2, dids=files1, account='root')

        set_status(scope=tmp_scope, name=tmp_dsn1, open=False)

        delete_replicas(rse='MOCK', files=files1)

        with assert_raises(DataIdentifierNotFound):
            get_did(scope=tmp_scope, name=tmp_dsn1)

        get_did(scope=tmp_scope, name=tmp_dsn2)

        assert_equal([f for f in list_files(scope=tmp_scope, name=tmp_dsn2)], [])

    def test_update_lock_counter(self):
        """ RSE (CORE): Test the update of a replica lock counter """
        rse = 'MOCK'
        tmp_scope = 'mock'
        tmp_file = 'file_%s' % generate_uuid()
        add_replica(rse=rse, scope=tmp_scope, name=tmp_file, bytes=1L, adler32='0cc737eb', account='jdoe')

        values = (1, 1, 1, -1, -1, -1, 1, 1, -1)
        tombstones = (True, True, True, True, True, False, True, True, True)
        lock_counters = (1, 2, 3, 2, 1, 0, 1, 2, 1)
        for value, tombstone, lock_counter in zip(values, tombstones, lock_counters):
            status = update_replica_lock_counter(rse=rse, scope=tmp_scope, name=tmp_file, value=value)
            assert_equal(status, True)
            replica = get_replica(rse=rse, scope=tmp_scope, name=tmp_file)