def insert_dataset(self, dataset_name):
"""
Fetch all popularity data for dataset
"""
api = 'getSingleDSstat'
sitename = 'summary'
name = dataset_name
aggr = 'day'
orderbys = ['totcpu', 'naccess']
coll = 'dataset_popularity'
for orderby in orderbys:
params = {'sitename':sitename, 'name':name, 'aggr':aggr, 'orderby':orderby}
json_data = self.pop_db.fetch(api=api, params=params)
data = get_json(json_data, 'data')
for pop_data in get_json(data, 'data'):
date = pop_db_timestamp_to_datetime(pop_data[0])
query = {'name':dataset_name, 'data':date}
popularity_data = {'name':dataset_name, 'date':date}
popularity_data[orderby] = pop_data[1]
data = {'$set':popularity_data}
self.storage.update_data(coll=coll, query=query, data=data, upsert=True)
def test_pop_db_timestamp_to_datetime(self):
"Test pop_db_timestamp_to_datetime function"
timestamp = 1406246400000
expected = datetime.datetime(2014, 7, 25, 0, 0)
result = pop_db_timestamp_to_datetime(timestamp)
self.assertEqual(result, expected)
def test_pop_db_timestamp_to_datetime(self):
"Test pop_db_timestamp_to_datetime function"
timestamp = 1406246400000
expected = datetime.datetime(2014, 7, 25, 0, 0)
result = pop_db_timestamp_to_datetime(timestamp)
self.assertEqual(result, expected)
