def test_pop_db_memory(self):
"Test pop_db data memory usage"
print ""
pop_db = PopDBService(self.config)
api = 'getDSdata'
sitename = 'summary'
aggr = 'day'
n = 200000
orderby = 'totcpu'
tstart = datetime_to_string(datetime_day(datetime.datetime.utcnow() - datetime.timedelta(days=10)))
tstop = datetime_to_string(datetime_day(datetime.datetime.utcnow()))
params = {'sitename':sitename, 'tstart':tstart, 'tstop':tstop, 'aggr':aggr, 'n':n, 'orderby':orderby}
pop_db_data = pop_db.fetch(api=api, params=params, cache=False)
total_size = total_size_of(pop_db_data)
logger.info('Total size of Pop DB data in memory is %d bytes (%dMB)', total_size, total_size/10**6)
def test_pop_db_memory(self):
"Test pop_db data memory usage"
print ""
pop_db = PopDBService(self.config)
api = 'getDSdata'
sitename = 'summary'
aggr = 'day'
n = 200000
orderby = 'totcpu'
tstart = datetime_to_string(datetime_day(datetime.datetime.utcnow() - datetime.timedelta(days=10)))
tstop = datetime_to_string(datetime_day(datetime.datetime.utcnow()))
params = {'sitename':sitename, 'tstart':tstart, 'tstop':tstop, 'aggr':aggr, 'n':n, 'orderby':orderby}
pop_db_data = pop_db.fetch(api=api, params=params, cache=False)
total_size = total_size_of(pop_db_data)
logger.info('Total size of Pop DB data in memory is %d bytes (%dMB)', total_size, total_size/10**6)
def insert_popularity_data(self, i, q):
"""
Insert popularity data for one dataset into db
"""
coll = 'dataset_popularity'
while True:
date = q.get()
self.logger.info('Inserting date %s', datetime_to_string(date))
api = 'DSStatInTimeWindow/'
tstart = datetime_to_string(date)
tstop = tstart
params = {'sitename':'summary', 'tstart':tstart, 'tstop':tstop}
json_data = self.pop_db.fetch(api=api, params=params)
# sort it in dictionary for easy fetching
for dataset in json_data['DATA']:
dataset_name = dataset['COLLNAME']
popularity_data = {'name':dataset_name, 'date':date}
popularity_data['n_accesses'] = dataset['NACC']
popularity_data['n_cpus'] = dataset['TOTCPU']
popularity_data['n_users'] = dataset['NUSERS']
query = {'name':dataset_name, 'date':date}
data = {'$set':popularity_data}
self.storage.update_data(coll=coll, query=query, data=data, upsert=True)
q.task_done()
def test_datetime_to_string(self):
"Test datetime_to_string function"
datetime_ = datetime.datetime(1987, 10, 27)
expected = '1987-10-27'
result = datetime_to_string(datetime_)
self.assertEqual(result, expected)
def test_datetime_to_string(self):
"Test datetime_to_string function"
datetime_ = datetime.datetime(1987, 10, 27)
expected = '1987-10-27'
result = datetime_to_string(datetime_)
self.assertEqual(result, expected)
