def test_pop_db_memory(self): "Test pop_db data memory usage" print "" pop_db = PopDBService(self.config) api = 'getDSdata' sitename = 'summary' aggr = 'day' n = 200000 orderby = 'totcpu' tstart = datetime_to_string(datetime_day(datetime.datetime.utcnow() - datetime.timedelta(days=10))) tstop = datetime_to_string(datetime_day(datetime.datetime.utcnow())) params = {'sitename':sitename, 'tstart':tstart, 'tstop':tstop, 'aggr':aggr, 'n':n, 'orderby':orderby} pop_db_data = pop_db.fetch(api=api, params=params, cache=False) total_size = total_size_of(pop_db_data) logger.info('Total size of Pop DB data in memory is %d bytes (%dMB)', total_size, total_size/10**6)
def insert_popularity_data(self, i, q): """ Insert popularity data for one dataset into db """ coll = 'dataset_popularity' while True: date = q.get() self.logger.info('Inserting date %s', datetime_to_string(date)) api = 'DSStatInTimeWindow/' tstart = datetime_to_string(date) tstop = tstart params = {'sitename':'summary', 'tstart':tstart, 'tstop':tstop} json_data = self.pop_db.fetch(api=api, params=params) # sort it in dictionary for easy fetching for dataset in json_data['DATA']: dataset_name = dataset['COLLNAME'] popularity_data = {'name':dataset_name, 'date':date} popularity_data['n_accesses'] = dataset['NACC'] popularity_data['n_cpus'] = dataset['TOTCPU'] popularity_data['n_users'] = dataset['NUSERS'] query = {'name':dataset_name, 'date':date} data = {'$set':popularity_data} self.storage.update_data(coll=coll, query=query, data=data, upsert=True) q.task_done()
def test_datetime_to_string(self): "Test datetime_to_string function" datetime_ = datetime.datetime(1987, 10, 27) expected = '1987-10-27' result = datetime_to_string(datetime_) self.assertEqual(result, expected)