class DatasetCache(object):
"""
Utility to count the accesses of the datasets during the last day.
"""
def __init__(self,
redis_host,
redis_port,
timeout=1,
prefix='did_cache',
delete_keys=False):
self._prefix = prefix + '_' + str(uuid4()).split('-')[0]
self._tms = RedisTimeSeries(redis_host, redis_port, timeout,
self._prefix)
if delete_keys:
self._tms.delete_keys()
def add_did(self, did):
self._tms.add_point('_'.join(did), 1)
def get_did(self, did):
self._tms.trim()
series = self._tms.get_series('_'.join(did))
return len(series)
class __WorkloadCollector(object):
"""
Private class needed implement singleton.
"""
def __init__(self, delete_keys=False):
self._avg_jobs = {}
self._cur_jobs = {}
self._max_jobs = {}
self._tms = RedisTimeSeries(
config_get('c3po', 'redis_host'),
config_get_int('c3po', 'redis_port'),
config_get_int('c3po-workload', 'window'), 'jobs_')
self._request_headers = {
"Accept": "application/json",
"Content-Type": "application/json"
}
self._request_url = config_get('c3po-workload', 'panda_url')
if delete_keys:
self._tms.delete_keys()
self.reload_cache()
def reload_cache(self):
self._tms.trim()
for key in self._tms.get_keys():
site = "_".join(key.split('_')[1:])
job_series = self._tms.get_series(site)
num_jobs = len(job_series)
if num_jobs > 0:
self._avg_jobs[site] = sum(job_series) / num_jobs
self._max_jobs[site] = max(job_series)
self._cur_jobs[site] = job_series[-1]
def collect_workload(self):
start = time()
resp = get(self._request_url, headers=self._request_headers)
logging.debug("PanDA response took %fs" % (time() - start))
start = time()
jobs = loads(resp.text)['jobs']
logging.debug("decoding JSON response took %fs" % (time() - start))
sites = {}
start = time()
for job in jobs:
if job['computingsite'] not in sites:
sites[job['computingsite']] = 0
sites[job['computingsite']] += 1
for site, jobs in sites.items():
self._tms.add_point(site, jobs)
logging.debug("processing took %fs" % (time() - start))
self.reload_cache()
