def test_3(self):
'''inserting 6 documents and updating it using uosert in 2 diff dbs
and checking values in both dbs'''
mongo_agg = MongoQueryAggregator(MONGO_DB_SETTINGS, 0.1, 5)
docs = [{
'id': 1
}, {
'id': 2
}, {
'id': 3
}, {
'id': 4
}, {
'id': 5
}, {
'id': 6
}]
self.conn['testdb1'].profiles.insert(docs[:3])
self.conn['testdb2'].profiles.insert(docs[3:])
mongo_agg.testdb1.profiles.find({}).upsert().update(
{'$set': {
'status': 'updated'
}})
mongo_agg.testdb2.profiles.find({}).upsert().update(
{'$set': {
'status': 'updated'
}})
time.sleep(0.1)
# query just to flush older queries
mongo_agg.testdb2.profiles.find({
'css': 'sacas'
}).update({'$set': docs[0]})
docs_in_db = []
aggregators_expected_results = {
('testdb1', 'profiles'): Counter({
'nModified': 3,
'nMatched': 3
}),
('testdb2', 'profiles'): Counter({
'nModified': 3,
'nMatched': 3
})
}
data = self.conn.testdb1.profiles.find()
output_data = []
for doc in data:
output_data.append(doc)
data = self.conn.testdb2.profiles.find()
for doc in data:
output_data.append(doc)
self.assertEqual(aggregators_expected_results, mongo_agg.get_results())
for doc in docs:
doc['status'] = 'updated'
self.assertListEqual(output_data, docs)
def test_4(self):
'''inserting to multiple data to multiple dbs
and checking mongo data and aggregators resuts'''
mongo_agg = MongoQueryAggregator(MONGO_DB_SETTINGS, 0.1, 5)
dbs_to_data = {
'testdb1': [{
'name': 'User2',
'id': 1
}, {
'name': 'User2',
'id': 2
}],
'testdb2': [{
'name': 'User3',
'id': 3
}],
'testdb3': [{
'name': 'User5',
'id': 5
}, {
'name': 'User6',
'id': 6
}, {
'name': 'User7',
'id': 8
}]
}
for db_name in dbs_to_data:
for doc in dbs_to_data[db_name]:
mongo_agg[db_name].profiles.insert(doc)
time.sleep(0.1)
mongo_agg.testdb1.profiles.insert({'key': 1
}) # dummy data to flush older data
projection = {'name': 1, 'id': 1, '_id': 0}
for db_name in dbs_to_data:
data = self.conn[db_name].profiles.find().sort([('id', 1)])
self.assertEqual(data.count(), len(dbs_to_data[db_name]))
docs_in_db = []
for doc in data:
docs_in_db.append(doc)
self.assertListEqual(docs_in_db, dbs_to_data[db_name])
aggregators_expected_results = {
('testdb1', 'profiles'): Counter({'nInserted': 2}),
('testdb3', 'profiles'): Counter({'nInserted': 3}),
('testdb2', 'profiles'): Counter({'nInserted': 1})
}
aggregators_results = mongo_agg.get_results()
self.assertEqual(aggregators_expected_results, aggregators_results)
def test_1(self):
'''inserting 2 document in interval of 0.1 sec
and updating same 2 documents'''
mongo_agg = MongoQueryAggregator(MONGO_DB_SETTINGS, 0.1, 10)
docs = [{'name': 'User1', 'id': 1}, {'name': 'User2', 'id': 2}]
#------ insert data to update --------
mongo_agg.testdb1.profiles.insert(docs[0])
mongo_agg.testdb1.profiles.insert(docs[1])
#-------------------------------------
mongo_agg.testdb1.profiles.find({
'id': 1
}).upsert().update({'$set': {
'updated': True
}})
mongo_agg.testdb1.profiles.find({
'id': 2
}).upsert().update({'$set': {
'updated': True
}})
self.assertEqual(self.conn['testdb1'].profiles.count(), 0)
time.sleep(0.1)
mongo_agg.testdb1.profiles.insert({'dummy':
1}) # just to flush old data
data = self.conn['testdb1'].profiles.find()
self.assertEqual(self.conn['testdb1'].profiles.count(), 2)
data_in_dbs = []
for doc in data:
data_in_dbs.append(doc)
for doc in docs:
doc['updated'] = True
self.assertListEqual(docs, data_in_dbs)
aggregators_expected_results = {
('testdb1', 'profiles'):
Counter({
'nModified': 2,
'nMatched': 2,
'nInserted': 2
})
}
self.assertEqual(aggregators_expected_results, mongo_agg.get_results())
def test_3(self):
'''inserting 6 documents with max_ops_limit=5
so first five docs should be present in mongodb
here inserting to multiple dbs'''
mongo_agg = MongoQueryAggregator(MONGO_DB_SETTINGS, 1, 5)
docs = [{
'name': 'User1',
'id': 1
}, {
'name': 'User2',
'id': 2
}, {
'name': 'User3',
'id': 3
}, {
'name': 'User4',
'id': 4
}, {
'name': 'User5',
'id': 5
}, {
'name': 'User6',
'id': 6
}]
for doc in docs[:6]:
# inserting first 6 records to db testdb1
mongo_agg.testdb1.profiles.insert(doc)
mongo_agg.testdb2.profiles.insert({'name': 'User1', 'id': 1})
# while inserting 6 records, 6th record will flush first 5 to db
data = self.conn['testdb1'].profiles.find().sort([('id', 1)])
docs_in_db = []
for doc in data:
docs_in_db.append(doc)
self.assertListEqual(docs_in_db, docs[:5])
aggregators_expected_results = {
('testdb1', 'profiles'): Counter({'nInserted': 5})
}
aggregators_results = mongo_agg.get_results()
self.assertEqual(aggregators_expected_results, aggregators_results)
def test_2(self):
'''Inserting/updating multiple documents to multiple dbs db doing operation greater than
max_ops_limit and checking the dbs
'''
mongo_agg = MongoQueryAggregator(MONGO_DB_SETTINGS, 0.1, 5)
dbs_to_data_to_insert = {
'testdb1': [{
'id': 1
}, {
'id': 2
}],
'testdb2': [{
'id': 3
}],
'testdb3': [{
'id': 5
}, {
'id': 6
}, {
'id': 8
}]
}
db_to_data_to_update = {
'testdb1': [({
'id': 1
}, {
'name': 'new1'
})],
'testdb2': [({
'id': 100
}, {
'name': 'new100'
})], # will insert
'testdb3': [({
'id': 5
}, {
'name': 'new5'
}), ({
'id': 6
}, {
'name': 'new6'
})]
}
for db_name, docs in dbs_to_data_to_insert.iteritems():
for doc in docs:
mongo_agg[db_name].profiles.insert(doc)
for db_name, data in db_to_data_to_update.iteritems():
for search_query, update_query in data:
mongo_agg[db_name].profiles.find(search_query).upsert().update(
{'$set': update_query})
time.sleep(0.1)
mongo_agg.testdb1.profiles.insert({'dummy':
1}) # just to flush old data
# expected data on dbs
expected_data = {
'testdb1': [{
'id': 1,
'name': 'new1'
}, {
'id': 2
}],
'testdb2': [{
'id': 3
}, {
'id': 100,
'name': 'new100'
}],
'testdb3': [{
'id': 5,
'name': 'new5'
}, {
'id': 6,
'name': 'new6'
}, {
'id': 8
}]
}
for db in expected_data:
data = self.conn[db].profiles.find({}, {
'name': 1,
'id': 1,
'_id': 0
})
# db data count check
self.assertEqual(data.count(), len(expected_data[db]))
data_in_db = []
for doc in data:
data_in_db.append(doc)
self.assertListEqual(data_in_db, expected_data[db])
aggregators_expected_results = {
('testdb1', 'profiles'):
Counter({
'nInserted': 2,
'nModified': 1,
'nMatched': 1
}),
('testdb3', 'profiles'):
Counter({
'nInserted': 3,
'nModified': 2,
'nMatched': 2
}),
('testdb2', 'profiles'):
Counter({
'nUpserted': 1,
'nInserted': 1
})
}
self.assertEqual(mongo_agg.get_results(), aggregators_expected_results)