class TestLoader(unittest.TestCase):
def setUp(self):
self.loader = Loader(db_uri='mongodb://localhost:27017/',
db_name="test_fever_unittest")
self.coll_purchase = "purchase_detail"
self.coll_events = "events_info"
def tearDown(self):
self.loader.delete_many(self.coll_purchase, {})
self.loader.delete_many(self.coll_events, {})
def test_insert_many_events(self):
with open('/opt/repos/plan-test/test/out/events.json') as f:
data = json.load(f)
self.loader.insert_many(self.coll_events, data)
def test_insert_many_purchase(self):
with open('/opt/repos/plan-test/test/out/purchase.json') as f:
data = json.load(f)
self.loader.insert_many(self.coll_events, data)
def test_insert_one(self):
num_before = self.loader.count_elems(self.coll_events)
self.loader.insert_one(
self.coll_events, {
"city": "MAD",
"event": "end_app_session",
"os": "ios",
"plan_id": "6846",
"time": 1530057600000,
"user_id": 611248618,
"_id": "611248618|nan|2018-06-27 00:00:00|end_app_session|ios"
})
num_after = self.loader.count_elems(self.coll_events)
self.assertEqual(num_after, num_before + 1)
def run(self):
# TODO: Unncomment for uncompress and move events json file compressed
# self._uncompress_events()
# TODO: Unncomment for deleting events json file compressed
# Delete events json file
# os.remove(src_gz_file)
extractor = Extractor(self.src_db_config)
df_plans = extractor.export_table_to_df(self.table_plans)
transformer = Transformer()
df_events = transformer.create_events_info_df_from_file(
self.src_events)
df_purchase = transformer.create_purchase_detail_df_from_df(
df_events, df_plans)
loader = Loader(db_name='test_fever')
events_json = df_events.to_dict(orient="records")
# loader.delete_many(collection_name=self.mongo_db_events, json_query={}) # Test
try:
loader.insert_many(collection_name=self.mongo_db_events,
json_list=events_json)
except pymongo.errors.BulkWriteError:
loader.upsert_many_one_by_one(collection_name=self.mongo_db_events,
json_list=events_json)
purchase_json = df_purchase.to_dict(orient="records")
# loader.delete_many(collection_name=self.mongo_db_purchase, json_query={}) # Test
try:
loader.insert_many(collection_name=self.mongo_db_purchase,
json_list=purchase_json)
except pymongo.errors.BulkWriteError:
loader.upsert_many_one_by_one(
collection_name=self.mongo_db_purchase,
json_list=purchase_json)