def __update_datasets(dataset_queue): len_ds = dataset_queue.qsize() datasets = {} dslocks = {} now = time() for _ in range(0, len_ds): dataset = dataset_queue.get() did = dataset['scope'] + ":" + dataset['name'] rse = dataset['rse'] if did not in datasets: datasets[did] = dataset['accessed_at'] else: datasets[did] = max(datasets[did], dataset['accessed_at']) if rse is None: continue if did not in dslocks: dslocks[did] = {} if rse not in dslocks[did]: dslocks[did][rse] = dataset['accessed_at'] else: dslocks[did][rse] = max(dataset['accessed_at'], dslocks[did][rse]) logging.debug('(kronos_dataset) fetched %d datasets from queue (%ds)' % (len_ds, time() - now)) total, failed, start = 0, 0, time() for did, accessed_at in datasets.items(): scope, name = did.split(':') update_did = {'scope': scope, 'name': name, 'type': DIDType.DATASET, 'accessed_at': accessed_at} # if update fails, put back in queue and retry next time if not touch_dids((update_did,)): update_did['rse'] = None dataset_queue.put(update_did) failed += 1 total += 1 logging.debug('(kronos_dataset) did update for %d datasets, %d failed (%ds)' % (total, failed, time() - start)) total, failed, start = 0, 0, time() for did, rses in dslocks.items(): scope, name = did.split(':') for rse, accessed_at in rses.items(): update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at} # if update fails, put back in queue and retry next time if not touch_dataset_locks((update_dslock,)): dataset_queue.put(update_dslock) failed += 1 total += 1 logging.debug('(kronos_dataset) did update for %d locks, %d failed (%ds)' % (total, failed, time() - start)) total, failed, start = 0, 0, time() for did, rses in dslocks.items(): scope, name = did.split(':') for rse, accessed_at in rses.items(): update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at} # if update fails, put back in queue and retry next time if not touch_collection_replicas((update_dslock,)): dataset_queue.put(update_dslock) failed += 1 total += 1 logging.debug('(kronos_dataset) did update for %d collection replicas, %d failed (%ds)' % (total, failed, time() - start))
def __update_datasets(dataset_queue): len_ds = dataset_queue.qsize() datasets = {} dslocks = {} now = time() for i in xrange(0, len_ds): dataset = dataset_queue.get() did = dataset['scope'] + ":" + dataset['name'] rse = dataset['rse'] if did not in datasets: datasets[did] = dataset['accessed_at'] else: datasets[did] = max(datasets[did], dataset['accessed_at']) if rse is None: continue if did not in dslocks: dslocks[did] = {} if rse not in dslocks[did]: dslocks[did][rse] = dataset['accessed_at'] else: dslocks[did][rse] = max(dataset['accessed_at'], dslocks[did][rse]) logging.info('(kronos_dataset) fetched %d datasets from queue (%ds)' % (len_ds, time() - now)) total, failed, start = 0, 0, time() for did, accessed_at in datasets.items(): scope, name = did.split(':') update_did = {'scope': scope, 'name': name, 'type': DIDType.DATASET, 'accessed_at': accessed_at} # if update fails, put back in queue and retry next time if not touch_dids((update_did,)): update_did['rse'] = None dataset_queue.put(update_did) failed += 1 total += 1 logging.info('(kronos_dataset) did update for %d datasets, %d failed (%ds)' % (total, failed, time() - start)) total, failed, start = 0, 0, time() for did, rses in dslocks.items(): scope, name = did.split(':') for rse, accessed_at in rses.items(): update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at} # if update fails, put back in queue and retry next time if not touch_dataset_locks((update_dslock,)): dataset_queue.put(update_dslock) failed += 1 total += 1 logging.info('(kronos_dataset) did update for %d locks, %d failed (%ds)' % (total, failed, time() - start))