def testInsertSingle(self):
"""Test case - create collection and insert data -
"""
try:
with Connection(cfgOb=self.__cfgOb,
resourceName=self.__resourceName) as client:
mg = MongoDbUtil(client)
ok = mg.createCollection(self.__dbName, self.__collectionName)
self.assertTrue(ok)
ok = mg.databaseExists(self.__dbName)
self.assertTrue(ok)
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertTrue(ok)
#
dObj = self.__makeDataObj(2, 5, 5)
rId = mg.insert(self.__dbName, self.__collectionName, dObj)
self.assertTrue(rId is not None)
# Note that dObj is mutated by additional key '_id' that is added on insert -
#
rObj = mg.fetchOne(self.__dbName, self.__collectionName, "_id",
rId)
logger.debug("Return Object %s", pprint.pformat(rObj))
self.assertEqual(len(dObj), len(rObj))
self.assertEqual(dObj, rObj)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testInsertList(self):
"""Test case - create collection and insert data -"""
try:
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
mg = MongoDbUtil(client)
ok = mg.createCollection(self.__dbName, self.__collectionName)
self.assertTrue(ok)
ok = mg.databaseExists(self.__dbName)
self.assertTrue(ok)
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertTrue(ok)
#
dList = []
for ii in range(100):
dList.append(self.__makeDataObj(2, 5, 5, ii))
#
keyName = "DOC_ID"
rIdL = mg.insertList(self.__dbName, self.__collectionName, dList, keyNames=[keyName], salvage=True)
self.assertEqual(len(rIdL), len(dList))
#
# Note that dObj is mutated by additional key '_id' that is added on insert -
#
for ii, rId in enumerate(rIdL):
rObj = mg.fetchOne(self.__dbName, self.__collectionName, "_id", rId)
logger.debug("Return Object %s", pprint.pformat(rObj))
jj = int(rObj["DOC_ID"][4:])
self.assertEqual(len(dList[jj]), len(rObj))
self.assertEqual(dList[jj], rObj)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testCreateDropCollection(self):
"""Test case - create/drop collection -
"""
try:
with Connection(cfgOb=self.__cfgOb,
resourceName=self.__resourceName) as client:
mg = MongoDbUtil(client)
ok = mg.createCollection(self.__dbName, self.__collectionName)
self.assertTrue(ok)
ok = mg.databaseExists(self.__dbName)
self.assertTrue(ok)
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertTrue(ok)
#
logger.debug("Databases = %r", mg.getDatabaseNames())
logger.debug("Collections = %r",
mg.getCollectionNames(self.__dbName))
ok = mg.dropCollection(self.__dbName, self.__collectionName)
self.assertTrue(ok)
logger.debug("Databases = %r", mg.getDatabaseNames())
logger.debug("Collections = %r",
mg.getCollectionNames(self.__dbName))
# Removing the last collection will remove the database (results appear differ between mac and linux - )
ok = mg.databaseExists(self.__dbName)
# self.assertFalse(ok)
#
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertFalse(ok)
logger.debug("Collections = %r",
mg.getCollectionNames(self.__dbName))
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testSchemaValidation1(self):
"""Test case - create collection and insert data with schema validation (ext. schema assignment)
"""
# Example of a Mongo flavor of JsonSchema
vexpr = {"$jsonSchema": self.__mongoSchema}
query = [("collMod", self.__collectionName), ("validator", vexpr),
("validationLevel", "moderate")]
query = OrderedDict(query)
try:
with Connection(cfgOb=self.__cfgOb,
resourceName=self.__resourceName) as client:
mg = MongoDbUtil(client)
if mg.databaseExists(self.__dbName):
ok = mg.dropDatabase(self.__dbName)
self.assertTrue(ok)
#
ok = mg.createDatabase(self.__dbName)
self.assertTrue(ok)
#
ok = mg.createCollection(self.__dbName, self.__collectionName)
self.assertTrue(ok)
ok = mg.databaseExists(self.__dbName)
self.assertTrue(ok)
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertTrue(ok)
#
mg.databaseCommand(self.__dbName, query)
dObj = {"x": 1}
rId = mg.insert(self.__dbName, self.__collectionName, dObj)
logger.info("rId is %r", rId)
self.assertEqual(rId, None)
#
s2 = unescapeXmlCharRef(
" " Φ Ψ α £ ℅ ☆ 𝕫"
)
dObj = {
"strField1": "test value",
"strField2": s2,
"intField1": 50,
"enumField1": "v3",
"dblField1": 100.1
}
rId = mg.insert(self.__dbName, self.__collectionName, dObj)
logger.info("rId is %r", rId)
rObj = mg.fetchOne(self.__dbName, self.__collectionName, "_id",
rId)
logger.debug("Return Object %s", pprint.pformat(rObj))
self.assertEqual(len(dObj), len(rObj))
self.assertEqual(dObj, rObj)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testSchemaValidation3(self):
"""Test case - create collection and insert data with schema validation (warn mode) (integrated schema assignment)
"""
try:
with Connection(cfgOb=self.__cfgOb,
resourceName=self.__resourceName) as client:
mg = MongoDbUtil(client)
if mg.databaseExists(self.__dbName):
ok = mg.dropDatabase(self.__dbName)
self.assertTrue(ok)
#
ok = mg.createDatabase(self.__dbName)
self.assertTrue(ok)
#
ok = mg.createCollection(self.__dbName,
self.__collectionName,
overWrite=True,
bsonSchema=self.__mongoSchema,
validationAction="warn")
self.assertTrue(ok)
ok = mg.databaseExists(self.__dbName)
self.assertTrue(ok)
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertTrue(ok)
#
dObj = {"x": 1}
rId = mg.insert(self.__dbName, self.__collectionName, dObj)
logger.info("rId is %r", rId)
self.assertNotEqual(rId, None)
#
s2 = unescapeXmlCharRef(
" " Φ Ψ α £ ℅ ☆ 𝕫"
)
dObj = {
"strField1": "test value",
"strField2": s2,
"intField1": 50,
"enumField1": "v3a",
"dblField1": 100.1
}
rId = mg.insert(self.__dbName, self.__collectionName, dObj)
self.assertNotEqual(rId, None)
logger.info("rId is %r", rId)
rObj = mg.fetchOne(self.__dbName, self.__collectionName, "_id",
rId)
logger.debug("Return Object %s", pprint.pformat(rObj))
self.assertEqual(len(dObj), len(rObj))
self.assertEqual(dObj, rObj)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testCreateCollection(self):
"""Test case - create collection -"""
try:
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
mg = MongoDbUtil(client)
ok = mg.createCollection(self.__dbName, self.__collectionName)
self.assertTrue(ok)
ok = mg.databaseExists(self.__dbName)
self.assertTrue(ok)
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertTrue(ok)
#
ok = mg.createCollection(self.__dbName, self.__collectionName)
self.assertTrue(ok)
ok = mg.databaseExists(self.__dbName)
self.assertTrue(ok)
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReplaceList(self):
"""Test case - create collection and insert document list - replace and upsert document list"""
try:
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
nDocs = 10
mg = MongoDbUtil(client)
ok = mg.createCollection(self.__dbName, self.__collectionName)
self.assertTrue(ok)
ok = mg.databaseExists(self.__dbName)
self.assertTrue(ok)
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertTrue(ok)
#
dList = []
for ii in range(nDocs):
dObj = self.__makeDataObj(2, 5, 5, ii)
dList.append(dObj)
#
keyName = "DOC_ID"
rIdL = mg.insertList(self.__dbName, self.__collectionName, dList, keyNames=[keyName], salvage=True)
self.assertEqual(len(rIdL), len(dList))
#
for ii, rId in enumerate(rIdL):
rObj = mg.fetchOne(self.__dbName, self.__collectionName, "_id", rId)
# logger.debug("Return Object %s", pprint.pformat(rObj))
self.assertEqual(len(dList[ii]), len(rObj))
self.assertEqual(dList[ii], rObj)
#
# Replace with 2x the list length - half are duplicates id's
dList = []
for ii in range(nDocs + nDocs):
dObj = self.__makeDataObj(4, 10, 10, ii)
dList.append(dObj)
#
updL = mg.replaceList(self.__dbName, self.__collectionName, dList, ["DOC_ID"], upsertFlag=True)
#
logger.info("Upserted id list length %d", len(updL))
for ii in range(nDocs + nDocs):
kVal = "DOC_%d" % ii
rObj = mg.fetchOne(self.__dbName, self.__collectionName, "DOC_ID", kVal)
if not rObj:
logger.info("Failing to recover doc %s", kVal)
# logger.debug("Return Object %s", pprint.pformat(rObj))
rObj.pop("_id", None)
dList[ii].pop("_id", None)
self.assertEqual(len(dList[ii]), len(rObj))
self.assertEqual(dList[ii], rObj)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReplaceSingle(self):
"""Test case - create collection and insert document and then replace document -"""
try:
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
mg = MongoDbUtil(client)
ok = mg.createCollection(self.__dbName, self.__collectionName)
self.assertTrue(ok)
ok = mg.databaseExists(self.__dbName)
self.assertTrue(ok)
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertTrue(ok)
#
dObj = self.__makeDataObj(2, 5, 5, 1)
rId = mg.insert(self.__dbName, self.__collectionName, dObj)
self.assertTrue(rId is not None)
# Note that dObj is mutated by additional key '_id' that is added on insert -
#
rObj = mg.fetchOne(self.__dbName, self.__collectionName, "_id", rId)
logger.debug("Return Object %s", pprint.pformat(rObj))
self.assertEqual(len(dObj), len(rObj))
self.assertEqual(dObj, rObj)
#
# Now replace with a new document with the same document id
dObj = self.__makeDataObj(3, 2, 2, 1)
logger.debug("Replace Object %s", pprint.pformat(dObj))
rId = mg.replace(self.__dbName, self.__collectionName, dObj, {"DOC_ID": "DOC_1"}, upsertFlag=True)
# self.assertTrue(rId is not None)
rObj = mg.fetchOne(self.__dbName, self.__collectionName, "DOC_ID", "DOC_1")
rObj.pop("_id", None)
dObj.pop("_id", None)
logger.debug("Return Object %s", pprint.pformat(rObj))
self.assertEqual(len(dObj), len(rObj))
self.assertEqual(dObj, rObj)
#
# Now replace with a new document with a different key
dObj2 = self.__makeDataObj(5, 5, 5, 2)
logger.debug("Replace Object %s", pprint.pformat(dObj))
#
rId = mg.replace(self.__dbName, self.__collectionName, dObj2, {"DOC_ID": "DOC_2"}, upsertFlag=True)
rObj = mg.fetchOne(self.__dbName, self.__collectionName, "DOC_ID", "DOC_2")
rObj.pop("_id", None)
dObj2.pop("_id", None)
logger.debug("Return Object %s", pprint.pformat(rObj))
self.assertEqual(len(dObj2), len(rObj))
self.assertEqual(dObj2, rObj)
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def createCollection(self,
databaseName,
collectionName,
indexAttributeNames=None,
indexName="primary",
checkExists=False,
bsonSchema=None):
"""Create collection and optionally set index attributes for the named index and validation schema for a new collection.
Args:
databaseName (str): target database name
collectionName (str): target collection name
indexAttributeNames (list, optional): list of attribute names for the 'primary' index. Defaults to None.
checkExists (bool, optional): reuse an existing collection if True. Defaults to False.
bsonSchema (object, optional): BSON compatable validation schema. Defaults to None.
Returns:
(bool): True for success or False otherwise
"""
try:
logger.debug("Create database %s collection %s", databaseName,
collectionName)
with Connection(cfgOb=self.__cfgOb,
resourceName=self.__resourceName) as client:
mg = MongoDbUtil(client)
if checkExists and mg.databaseExists(
databaseName) and mg.collectionExists(
databaseName, collectionName):
ok1 = True
else:
ok1 = mg.createCollection(databaseName,
collectionName,
bsonSchema=bsonSchema)
ok2 = mg.databaseExists(databaseName)
ok3 = mg.collectionExists(databaseName, collectionName)
okI = True
if indexAttributeNames:
okI = mg.createIndex(databaseName,
collectionName,
indexAttributeNames,
indexName=indexName,
indexType="DESCENDING",
uniqueFlag=False)
return ok1 and ok2 and ok3 and okI
#
except Exception as e:
logger.exception("Failing with %s", str(e))
return False
def testSingleIndex(self):
"""Test case - create collection, create simple single index, insert document list, read check documents"""
try:
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
nDocs = 100
mg = MongoDbUtil(client)
ok = mg.createCollection(self.__dbName, self.__collectionName)
self.assertTrue(ok)
ok = mg.databaseExists(self.__dbName)
self.assertTrue(ok)
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertTrue(ok)
#
# Create before insert
ok = mg.createIndex(self.__dbName, self.__collectionName, keyList=["DOC_ID"], indexName="primary", indexType="DESCENDING", uniqueFlag=True)
self.assertTrue(ok)
dList = []
for ii in range(nDocs):
dObj = self.__makeDataObj(2, 5, 5, ii)
dList.append(dObj)
#
keyName = "DOC_ID"
rIdL = mg.insertList(self.__dbName, self.__collectionName, dList, keyNames=[keyName], salvage=True)
self.assertEqual(len(dList), len(rIdL))
#
for ii in range(nDocs):
kVal = "DOC_%d" % ii
rObj = mg.fetchOne(self.__dbName, self.__collectionName, "DOC_ID", kVal)
# logger.debug("Return Object %s" % pprint.pformat(rObj))
rObj.pop("_id", None)
dList[ii].pop("_id", None)
self.assertEqual(len(dList[ii]), len(rObj))
self.assertEqual(dList[ii], rObj)
#
ok = mg.dropIndex(self.__dbName, self.__collectionName, indexName="primary")
self.assertTrue(ok)
ok = mg.createIndex(self.__dbName, self.__collectionName, keyList=["DOC_ID"], indexName="primary", indexType="DESCENDING", uniqueFlag=True)
self.assertTrue(ok)
ok = mg.reIndex(self.__dbName, self.__collectionName)
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def __createCollection(self,
dbName,
collectionName,
indexAttributeNames=None,
checkExists=False,
bsonSchema=None):
"""Create database and collection and optionally a primary index -
"""
try:
logger.debug("Create database %s collection %s", dbName,
collectionName)
with Connection(cfgOb=self.__cfgOb,
resourceName=self.__resourceName) as client:
mg = MongoDbUtil(client)
if checkExists and mg.databaseExists(
dbName) and mg.collectionExists(
dbName, collectionName):
ok1 = True
else:
ok1 = mg.createCollection(dbName,
collectionName,
bsonSchema=bsonSchema)
ok2 = mg.databaseExists(dbName)
ok3 = mg.collectionExists(dbName, collectionName)
okI = True
if indexAttributeNames:
okI = mg.createIndex(dbName,
collectionName,
indexAttributeNames,
indexName="primary",
indexType="DESCENDING",
uniqueFlag=False)
return ok1 and ok2 and ok3 and okI
#
except Exception as e:
logger.exception("Failing with %s", str(e))
return False
def testSingleIndexSelect(self):
"""Test case - create collection, create simple single index, insert document list, read check documents."""
try:
logger.debug("Starting testSingleIndexSelect")
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
nDocs = 100
mg = MongoDbUtil(client)
ok = mg.createCollection(self.__dbName, self.__collectionName)
self.assertTrue(ok)
ok = mg.databaseExists(self.__dbName)
self.assertTrue(ok)
ok = mg.collectionExists(self.__dbName, self.__collectionName)
self.assertTrue(ok)
#
# Create before insert
ok = mg.createIndex(self.__dbName, self.__collectionName, keyList=["DOC_ID"], indexName="primary", indexType="DESCENDING", uniqueFlag=True)
self.assertTrue(ok)
dList = []
nRows = 5
for ii in range(nDocs):
dObj = self.__makeDataObj(2, 5, nRows, ii)
dList.append(dObj)
#
keyName = "DOC_ID"
rIdL = mg.insertList(self.__dbName, self.__collectionName, dList, keyNames=[keyName], salvage=True)
self.assertEqual(len(dList), len(rIdL))
#
for ii in range(nDocs):
kVal = "DOC_%d" % ii
rObj = mg.fetchOne(self.__dbName, self.__collectionName, "DOC_ID", kVal)
# logger.debug("Return Object %s" % pprint.pformat(rObj))
rObj.pop("_id", None)
dList[ii].pop("_id", None)
self.assertEqual(len(dList[ii]), len(rObj))
self.assertEqual(dList[ii], rObj)
#
ok = mg.dropIndex(self.__dbName, self.__collectionName, indexName="primary")
self.assertTrue(ok)
ok = mg.createIndex(self.__dbName, self.__collectionName, keyList=["DOC_ID"], indexName="primary", indexType="DESCENDING", uniqueFlag=True)
self.assertTrue(ok)
ok = mg.reIndex(self.__dbName, self.__collectionName)
self.assertTrue(ok)
#
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
mg = MongoDbUtil(client)
ii = mg.count(self.__dbName, self.__collectionName)
logger.debug("collection length %d", ii)
#
dList = mg.fetch(self.__dbName, self.__collectionName, ["DOC_ID"])
self.assertEqual(len(dList), nDocs)
logger.debug("Fetch length %d", len(dList))
for ii, dD in enumerate(dList):
logger.debug("Fetch num %d: %r", ii, dD)
#
dList = mg.fetch(self.__dbName, self.__collectionName, ["category_0.attribute_0"], queryD={"category_0.attribute_0": "val_0_0"})
self.assertEqual(len(dList), nDocs)
logger.debug("Fetch length %d", len(dList))
for ii, dD in enumerate(dList):
logger.debug("Fetch num %d: %r", ii, dD)
atName = "category_0.attribute_0"
vL0 = mg.distinct(self.__dbName, self.__collectionName, atName)
self.assertEqual(len(vL0), nRows + 2)
logger.debug("vL0 %r", vL0)
vL1 = mg.distinct(self.__dbName, self.__collectionName, "category_1.attribute_0")
self.assertEqual(len(vL1), nRows + 2)
for v in vL0:
num = mg.count(self.__dbName, self.__collectionName, countFilter={atName: v})
logger.debug("%s value %s (%d)", atName, v, num)
self.assertGreaterEqual(num, 100)
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()