def __update_datasets(dataset_queue):
len_ds = dataset_queue.qsize()
datasets = {}
dslocks = {}
now = time()
for _ in range(0, len_ds):
dataset = dataset_queue.get()
did = dataset['scope'] + ":" + dataset['name']
rse = dataset['rse']
if did not in datasets:
datasets[did] = dataset['accessed_at']
else:
datasets[did] = max(datasets[did], dataset['accessed_at'])
if rse is None:
continue
if did not in dslocks:
dslocks[did] = {}
if rse not in dslocks[did]:
dslocks[did][rse] = dataset['accessed_at']
else:
dslocks[did][rse] = max(dataset['accessed_at'], dslocks[did][rse])
logging.debug('(kronos_dataset) fetched %d datasets from queue (%ds)' % (len_ds, time() - now))
total, failed, start = 0, 0, time()
for did, accessed_at in datasets.items():
scope, name = did.split(':')
update_did = {'scope': scope, 'name': name, 'type': DIDType.DATASET, 'accessed_at': accessed_at}
# if update fails, put back in queue and retry next time
if not touch_dids((update_did,)):
update_did['rse'] = None
dataset_queue.put(update_did)
failed += 1
total += 1
logging.debug('(kronos_dataset) did update for %d datasets, %d failed (%ds)' % (total, failed, time() - start))
total, failed, start = 0, 0, time()
for did, rses in dslocks.items():
scope, name = did.split(':')
for rse, accessed_at in rses.items():
update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at}
# if update fails, put back in queue and retry next time
if not touch_dataset_locks((update_dslock,)):
dataset_queue.put(update_dslock)
failed += 1
total += 1
logging.debug('(kronos_dataset) did update for %d locks, %d failed (%ds)' % (total, failed, time() - start))
total, failed, start = 0, 0, time()
for did, rses in dslocks.items():
scope, name = did.split(':')
for rse, accessed_at in rses.items():
update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at}
# if update fails, put back in queue and retry next time
if not touch_collection_replicas((update_dslock,)):
dataset_queue.put(update_dslock)
failed += 1
total += 1
logging.debug('(kronos_dataset) did update for %d collection replicas, %d failed (%ds)' % (total, failed, time() - start))
def __update_datasets(dataset_queue):
len_ds = dataset_queue.qsize()
datasets = {}
dslocks = {}
now = time()
for i in xrange(0, len_ds):
dataset = dataset_queue.get()
did = dataset['scope'] + ":" + dataset['name']
rse = dataset['rse']
if did not in datasets:
datasets[did] = dataset['accessed_at']
else:
datasets[did] = max(datasets[did], dataset['accessed_at'])
if rse is None:
continue
if did not in dslocks:
dslocks[did] = {}
if rse not in dslocks[did]:
dslocks[did][rse] = dataset['accessed_at']
else:
dslocks[did][rse] = max(dataset['accessed_at'], dslocks[did][rse])
logging.info('(kronos_dataset) fetched %d datasets from queue (%ds)' % (len_ds, time() - now))
total, failed, start = 0, 0, time()
for did, accessed_at in datasets.items():
scope, name = did.split(':')
update_did = {'scope': scope, 'name': name, 'type': DIDType.DATASET, 'accessed_at': accessed_at}
# if update fails, put back in queue and retry next time
if not touch_dids((update_did,)):
update_did['rse'] = None
dataset_queue.put(update_did)
failed += 1
total += 1
logging.info('(kronos_dataset) did update for %d datasets, %d failed (%ds)' % (total, failed, time() - start))
total, failed, start = 0, 0, time()
for did, rses in dslocks.items():
scope, name = did.split(':')
for rse, accessed_at in rses.items():
update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at}
# if update fails, put back in queue and retry next time
if not touch_dataset_locks((update_dslock,)):
dataset_queue.put(update_dslock)
failed += 1
total += 1
logging.info('(kronos_dataset) did update for %d locks, %d failed (%ds)' % (total, failed, time() - start))