Example #1
0
    def test_touch_dids(self):
        """ DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp"""
        tmp_scope = 'mock'
        tmp_dsn1 = 'dsn_%s' % generate_uuid()
        tmp_dsn2 = 'dsn_%s' % generate_uuid()

        add_did(scope=tmp_scope,
                name=tmp_dsn1,
                type=DIDType.DATASET,
                account='root')
        add_did(scope=tmp_scope,
                name=tmp_dsn2,
                type=DIDType.DATASET,
                account='root')
        now = datetime.utcnow()

        now -= timedelta(microseconds=now.microsecond)
        assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
        assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))

        touch_dids(dids=[{
            'scope': tmp_scope,
            'name': tmp_dsn1,
            'type': DIDType.DATASET,
            'accessed_at': now
        }])
        assert_equal(now, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
        assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
Example #2
0
    def test_touch_dids_access_cnt(self):
        """ DATA IDENTIFIERS (CORE): Increase dids access_cnt"""
        tmp_scope = 'mock'
        tmp_dsn1 = 'dsn_%s' % generate_uuid()
        tmp_dsn2 = 'dsn_%s' % generate_uuid()

        add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root')
        add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account='root')

        assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1))
        assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2))

        for i in range(100):
            touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET}])
        assert_equal(100, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1))
        assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2))
Example #3
0
def __update_datasets(dataset_queue):
    len_ds = dataset_queue.qsize()
    datasets = {}
    dslocks = {}
    now = time()
    for _ in range(0, len_ds):
        dataset = dataset_queue.get()
        did = dataset['scope'] + ":" + dataset['name']
        rse = dataset['rse']
        if did not in datasets:
            datasets[did] = dataset['accessed_at']
        else:
            datasets[did] = max(datasets[did], dataset['accessed_at'])

        if rse is None:
            continue
        if did not in dslocks:
            dslocks[did] = {}
        if rse not in dslocks[did]:
            dslocks[did][rse] = dataset['accessed_at']
        else:
            dslocks[did][rse] = max(dataset['accessed_at'], dslocks[did][rse])
    logging.debug('(kronos_dataset) fetched %d datasets from queue (%ds)' % (len_ds, time() - now))

    total, failed, start = 0, 0, time()
    for did, accessed_at in datasets.items():
        scope, name = did.split(':')
        update_did = {'scope': scope, 'name': name, 'type': DIDType.DATASET, 'accessed_at': accessed_at}
        # if update fails, put back in queue and retry next time
        if not touch_dids((update_did,)):
            update_did['rse'] = None
            dataset_queue.put(update_did)
            failed += 1
        total += 1
    logging.debug('(kronos_dataset) did update for %d datasets, %d failed (%ds)' % (total, failed, time() - start))

    total, failed, start = 0, 0, time()
    for did, rses in dslocks.items():
        scope, name = did.split(':')
        for rse, accessed_at in rses.items():
            update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at}
            # if update fails, put back in queue and retry next time
            if not touch_dataset_locks((update_dslock,)):
                dataset_queue.put(update_dslock)
                failed += 1
            total += 1
    logging.debug('(kronos_dataset) did update for %d locks, %d failed (%ds)' % (total, failed, time() - start))

    total, failed, start = 0, 0, time()
    for did, rses in dslocks.items():
        scope, name = did.split(':')
        for rse, accessed_at in rses.items():
            update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at}
            # if update fails, put back in queue and retry next time
            if not touch_collection_replicas((update_dslock,)):
                dataset_queue.put(update_dslock)
                failed += 1
            total += 1
    logging.debug('(kronos_dataset) did update for %d collection replicas, %d failed (%ds)' % (total, failed, time() - start))
Example #4
0
    def test_touch_dids(self):
        """ DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp"""
        tmp_scope = 'mock'
        tmp_dsn1 = 'dsn_%s' % generate_uuid()
        tmp_dsn2 = 'dsn_%s' % generate_uuid()

        add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root')
        add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account='root')
        now = datetime.utcnow()

        now -= timedelta(microseconds=now.microsecond)
        assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
        assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))

        touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET, 'accessed_at': now}])
        assert_equal(now, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
        assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
Example #5
0
    def test_touch_dids_access_cnt(self):
        """ DATA IDENTIFIERS (CORE): Increase dids access_cnt"""
        tmp_scope = InternalScope('mock', **self.vo)
        root = InternalAccount('root', **self.vo)
        tmp_dsn1 = 'dsn_%s' % generate_uuid()
        tmp_dsn2 = 'dsn_%s' % generate_uuid()

        add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account=root)
        add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account=root)

        assert get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1) is None
        assert get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2) is None

        for i in range(100):
            touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET}])
        assert 100 == get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1)
        assert get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2) is None
Example #6
0
    def test_touch_dids_atime(self):
        """ DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp"""
        tmp_scope = InternalScope('mock', **self.vo)
        root = InternalAccount('root', **self.vo)
        tmp_dsn1 = 'dsn_%s' % generate_uuid()
        tmp_dsn2 = 'dsn_%s' % generate_uuid()

        add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account=root)
        add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account=root)
        now = datetime.utcnow()

        now -= timedelta(microseconds=now.microsecond)
        assert get_did_atime(scope=tmp_scope, name=tmp_dsn1) is None
        assert get_did_atime(scope=tmp_scope, name=tmp_dsn2) is None

        touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET, 'accessed_at': now}])
        assert now == get_did_atime(scope=tmp_scope, name=tmp_dsn1)
        assert get_did_atime(scope=tmp_scope, name=tmp_dsn2) is None
Example #7
0
def __update_datasets(dataset_queue):
    len_ds = dataset_queue.qsize()
    datasets = {}
    dslocks = {}
    now = time()
    for i in xrange(0, len_ds):
        dataset = dataset_queue.get()
        did = dataset['scope'] + ":" + dataset['name']
        rse = dataset['rse']
        if did not in datasets:
            datasets[did] = dataset['accessed_at']
        else:
            datasets[did] = max(datasets[did], dataset['accessed_at'])

        if rse is None:
            continue
        if did not in dslocks:
            dslocks[did] = {}
        if rse not in dslocks[did]:
            dslocks[did][rse] = dataset['accessed_at']
        else:
            dslocks[did][rse] = max(dataset['accessed_at'], dslocks[did][rse])
    logging.info('(kronos_dataset) fetched %d datasets from queue (%ds)' % (len_ds, time() - now))

    total, failed, start = 0, 0, time()
    for did, accessed_at in datasets.items():
        scope, name = did.split(':')
        update_did = {'scope': scope, 'name': name, 'type': DIDType.DATASET, 'accessed_at': accessed_at}
        # if update fails, put back in queue and retry next time
        if not touch_dids((update_did,)):
            update_did['rse'] = None
            dataset_queue.put(update_did)
            failed += 1
        total += 1
    logging.info('(kronos_dataset) did update for %d datasets, %d failed (%ds)' % (total, failed, time() - start))

    total, failed, start = 0, 0, time()
    for did, rses in dslocks.items():
        scope, name = did.split(':')
        for rse, accessed_at in rses.items():
            update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at}
            # if update fails, put back in queue and retry next time
            if not touch_dataset_locks((update_dslock,)):
                dataset_queue.put(update_dslock)
                failed += 1
            total += 1
    logging.info('(kronos_dataset) did update for %d locks, %d failed (%ds)' % (total, failed, time() - start))