def insert_dataset(self, dataset_name): """ Fetch all popularity data for dataset """ api = 'getSingleDSstat' sitename = 'summary' name = dataset_name aggr = 'day' orderbys = ['totcpu', 'naccess'] coll = 'dataset_popularity' for orderby in orderbys: params = {'sitename':sitename, 'name':name, 'aggr':aggr, 'orderby':orderby} json_data = self.pop_db.fetch(api=api, params=params) data = get_json(json_data, 'data') for pop_data in get_json(data, 'data'): date = pop_db_timestamp_to_datetime(pop_data[0]) query = {'name':dataset_name, 'data':date} popularity_data = {'name':dataset_name, 'date':date} popularity_data[orderby] = pop_data[1] data = {'$set':popularity_data} self.storage.update_data(coll=coll, query=query, data=data, upsert=True)
def test_pop_db_timestamp_to_datetime(self): "Test pop_db_timestamp_to_datetime function" timestamp = 1406246400000 expected = datetime.datetime(2014, 7, 25, 0, 0) result = pop_db_timestamp_to_datetime(timestamp) self.assertEqual(result, expected)