def __update_atime(self):
"""
Bulk update atime.
"""
replicas = []
for report in self.__reports:
try:
# check if scope in report. if not skip this one.
if 'scope' not in report:
record_counter('daemons.tracer.kronos.missing_scope')
continue
else:
record_counter('daemons.tracer.kronos.with_scope')
# for the moment only report with eventType get* are handled.
if not report['eventType'].startswith('get'):
continue
record_counter('daemons.tracer.kronos.total_get')
if report['eventType'] == 'get':
record_counter('daemons.tracer.kronos.dq2clients')
elif report['eventType'] == 'get_sm':
record_counter('daemons.tracer.kronos.panda_production')
elif report['eventType'] == 'get_sm_a':
record_counter('daemons.tracer.kronos.panda_analysis')
else:
record_counter('daemons.tracer.kronos.other_get')
# check if the report has the right state.
if report['clientState'] in self.__excluded_states:
continue
if report['usrdn'] in self.__excluded_usrdns:
continue
if not report['remoteSite']:
continue
replicas.append({'name': report['filename'], 'scope': report['scope'], 'rse': report['remoteSite'], 'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix'])})
except (KeyError, AttributeError):
logging.error(format_exc())
record_counter('daemons.tracer.kronos.report_error')
continue
for did in list_parent_dids(report['scope'], report['filename']):
if did['type'] != DIDType.DATASET:
continue
# do not update _dis datasets
if did['scope'] == 'panda' and '_dis' in did['name']:
continue
self.__dataset_queue.put({'scope': did['scope'], 'name': did['name'], 'did_type': did['type'], 'rse': report['remoteSite'], 'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix'])})
logging.info(replicas)
try:
ts = time()
touch_replicas(replicas)
record_timer('daemons.tracer.kronos.update_atime', (time() - ts) * 1000)
except:
logging.error(format_exc())
record_counter('daemons.tracer.kronos.update_error')
logging.info('(kronos_file) updated %d replicas' % len(replicas))
tmp_scope = 'mock'
nbfiles = 5
files1 = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in xrange(nbfiles)]
files2 = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in xrange(nbfiles)]
files2.append(files1[0])
add_replicas(rse='MOCK', files=files1, account='root', ignore_availability=True)
add_replicas(rse='MOCK', files=files2, account='root', ignore_availability=True)
now = datetime.utcnow()
now -= timedelta(microseconds=now.microsecond)
assert_equal(None, get_replica_atime({'scope': files1[0]['scope'], 'name': files1[0]['name'], 'rse': 'MOCK'}))
assert_equal(None, get_did_atime(scope=tmp_scope, name=files1[0]['name']))
touch_replicas(replicas=[{'scope': files1[0]['scope'], 'name': files1[0]['name'], 'rse': 'MOCK', 'accessed_at': now}])
assert_equal(now, get_replica_atime({'scope': files1[0]['scope'], 'name': files1[0]['name'], 'rse': 'MOCK'}))
assert_equal(now, get_did_atime(scope=tmp_scope, name=files1[0]['name']))
for i in range(1, nbfiles):
assert_equal(None, get_replica_atime({'scope': files1[i]['scope'], 'name': files1[i]['name'], 'rse': 'MOCK'}))
for i in range(0, nbfiles - 1):
assert_equal(None, get_replica_atime({'scope': files2[i]['scope'], 'name': files2[i]['name'], 'rse': 'MOCK'}))
def test_list_replicas_all_states(self):
""" REPLICA (CORE): list file replicas with all_states"""
tmp_scope = 'mock'
nbfiles = 13
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in xrange(nbfiles)]
