def test_touch_dids(self):
""" DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp"""
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
add_did(scope=tmp_scope,
name=tmp_dsn1,
type=DIDType.DATASET,
account='root')
add_did(scope=tmp_scope,
name=tmp_dsn2,
type=DIDType.DATASET,
account='root')
now = datetime.utcnow()
now -= timedelta(microseconds=now.microsecond)
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
touch_dids(dids=[{
'scope': tmp_scope,
'name': tmp_dsn1,
'type': DIDType.DATASET,
'accessed_at': now
}])
assert_equal(now, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
def test_touch_dids_access_cnt(self):
""" DATA IDENTIFIERS (CORE): Increase dids access_cnt"""
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root')
add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account='root')
assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2))
for i in range(100):
touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET}])
assert_equal(100, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2))
def __update_datasets(dataset_queue):
len_ds = dataset_queue.qsize()
datasets = {}
dslocks = {}
now = time()
for _ in range(0, len_ds):
dataset = dataset_queue.get()
did = dataset['scope'] + ":" + dataset['name']
rse = dataset['rse']
if did not in datasets:
datasets[did] = dataset['accessed_at']
else:
datasets[did] = max(datasets[did], dataset['accessed_at'])
if rse is None:
continue
if did not in dslocks:
dslocks[did] = {}
if rse not in dslocks[did]:
dslocks[did][rse] = dataset['accessed_at']
else:
dslocks[did][rse] = max(dataset['accessed_at'], dslocks[did][rse])
logging.debug('(kronos_dataset) fetched %d datasets from queue (%ds)' % (len_ds, time() - now))
total, failed, start = 0, 0, time()
for did, accessed_at in datasets.items():
scope, name = did.split(':')
update_did = {'scope': scope, 'name': name, 'type': DIDType.DATASET, 'accessed_at': accessed_at}
# if update fails, put back in queue and retry next time
if not touch_dids((update_did,)):
update_did['rse'] = None
dataset_queue.put(update_did)
failed += 1
total += 1
logging.debug('(kronos_dataset) did update for %d datasets, %d failed (%ds)' % (total, failed, time() - start))
total, failed, start = 0, 0, time()
for did, rses in dslocks.items():
scope, name = did.split(':')
for rse, accessed_at in rses.items():
update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at}
# if update fails, put back in queue and retry next time
if not touch_dataset_locks((update_dslock,)):
dataset_queue.put(update_dslock)
failed += 1
total += 1
logging.debug('(kronos_dataset) did update for %d locks, %d failed (%ds)' % (total, failed, time() - start))
total, failed, start = 0, 0, time()
for did, rses in dslocks.items():
scope, name = did.split(':')
for rse, accessed_at in rses.items():
update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at}
# if update fails, put back in queue and retry next time
if not touch_collection_replicas((update_dslock,)):
dataset_queue.put(update_dslock)
failed += 1
total += 1
logging.debug('(kronos_dataset) did update for %d collection replicas, %d failed (%ds)' % (total, failed, time() - start))
def test_touch_dids(self):
""" DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp"""
tmp_scope = 'mock'
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root')
add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account='root')
now = datetime.utcnow()
now -= timedelta(microseconds=now.microsecond)
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET, 'accessed_at': now}])
assert_equal(now, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
def test_touch_dids_access_cnt(self):
""" DATA IDENTIFIERS (CORE): Increase dids access_cnt"""
tmp_scope = InternalScope('mock', **self.vo)
root = InternalAccount('root', **self.vo)
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account=root)
add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account=root)
assert get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1) is None
assert get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2) is None
for i in range(100):
touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET}])
assert 100 == get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1)
assert get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2) is None
def test_touch_dids_atime(self):
""" DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp"""
tmp_scope = InternalScope('mock', **self.vo)
root = InternalAccount('root', **self.vo)
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account=root)
add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account=root)
now = datetime.utcnow()
now -= timedelta(microseconds=now.microsecond)
assert get_did_atime(scope=tmp_scope, name=tmp_dsn1) is None
assert get_did_atime(scope=tmp_scope, name=tmp_dsn2) is None
touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET, 'accessed_at': now}])
assert now == get_did_atime(scope=tmp_scope, name=tmp_dsn1)
assert get_did_atime(scope=tmp_scope, name=tmp_dsn2) is None
def __update_datasets(dataset_queue):
len_ds = dataset_queue.qsize()
datasets = {}
dslocks = {}
now = time()
for i in xrange(0, len_ds):
dataset = dataset_queue.get()
did = dataset['scope'] + ":" + dataset['name']
rse = dataset['rse']
if did not in datasets:
datasets[did] = dataset['accessed_at']
else:
datasets[did] = max(datasets[did], dataset['accessed_at'])
if rse is None:
continue
if did not in dslocks:
dslocks[did] = {}
if rse not in dslocks[did]:
dslocks[did][rse] = dataset['accessed_at']
else:
dslocks[did][rse] = max(dataset['accessed_at'], dslocks[did][rse])
logging.info('(kronos_dataset) fetched %d datasets from queue (%ds)' % (len_ds, time() - now))
total, failed, start = 0, 0, time()
for did, accessed_at in datasets.items():
scope, name = did.split(':')
update_did = {'scope': scope, 'name': name, 'type': DIDType.DATASET, 'accessed_at': accessed_at}
# if update fails, put back in queue and retry next time
if not touch_dids((update_did,)):
update_did['rse'] = None
dataset_queue.put(update_did)
failed += 1
total += 1
logging.info('(kronos_dataset) did update for %d datasets, %d failed (%ds)' % (total, failed, time() - start))
total, failed, start = 0, 0, time()
for did, rses in dslocks.items():
scope, name = did.split(':')
for rse, accessed_at in rses.items():
update_dslock = {'scope': scope, 'name': name, 'rse': rse, 'accessed_at': accessed_at}
# if update fails, put back in queue and retry next time
if not touch_dataset_locks((update_dslock,)):
dataset_queue.put(update_dslock)
failed += 1
total += 1
logging.info('(kronos_dataset) did update for %d locks, %d failed (%ds)' % (total, failed, time() - start))