class SchemaDataPrepValidateTests(unittest.TestCase): def setUp(self): self.__numProc = 2 # self.__fileLimit = 200 self.__fileLimit = None self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example-ihm.yml") configName = "site_info_configuration" self.__configName = configName self.__cfgOb = ConfigUtil(configPath=self.__configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath) self.__mU = MarshalUtil(workPath=self.__cachePath) #self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=False, rebuildFlag=True) self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=True) self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb, numProc=self.__numProc, fileLimit=self.__fileLimit, cachePath=self.__cachePath) # self.__birdRepoPath = self.__cfgOb.getPath("BIRD_REPO_PATH", sectionName=configName) # self.__fTypeRow = "drop-empty-attributes|drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs" self.__fTypeCol = "drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs" self.__verbose = True # self.__modulePathMap = self.__cfgOb.get("DICT_METHOD_HELPER_MODULE_PATH_MAP", sectionName=configName) self.__testDirPath = os.path.join(HERE, "test-output", "pdbx-files") self.__testIhmDirPath = os.path.join(HERE, "test-output", "ihm-files") self.__export = True # #self.__extraOpts = None # The following for extended parent/child info - self.__extraOpts = 'addParentRefs|addPrimaryKey' # self.__alldatabaseNameD = { "ihm_dev": ["ihm_dev"], "pdbx": ["pdbx", "pdbx_ext"], "pdbx_core": ["pdbx_core_entity", "pdbx_core_entry", "pdbx_core_assembly", "pdbx_core_entity_instance", "pdbx_core_entity_instance_validation"], "bird": ["bird"], "bird_family": ["family"], "chem_comp": ["chem_comp"], "bird_chem_comp": ["bird_chem_comp"], "bird_chem_comp_core": ["bird_chem_comp_core"], } self.__databaseNameD = { "pdbx_core": ["pdbx_core_entity", "pdbx_core_entry", "pdbx_core_assembly", "pdbx_core_entity_instance", "pdbx_core_entity_instance_validation"], "bird_chem_comp_core": ["bird_chem_comp_core"], } self.__mergeContentTypeD = {"pdbx_core": ["vrpt"]} # self.__databaseNameD = {"chem_comp_core": ["chem_comp_core"], "bird_chem_comp_core": ["bird_chem_comp_core"]} # self.__databaseNameD = {"ihm_dev_full": ["ihm_dev_full"]} # self.__databaseNameD = {"pdbx_core": ["pdbx_core_entity_instance_validation"]} # self.__databaseNameD = {"pdbx_core": ["pdbx_core_entity_monomer"]} self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime())) def tearDown(self): endTime = time.time() logger.debug("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime) def testValidateOptsRepo(self): # schemaLevel = "min" schemaLevel = "full" inputPathList = None eCount = self.__testValidateOpts(databaseNameD=self.__databaseNameD, inputPathList=inputPathList, schemaLevel=schemaLevel, mergeContentTypeD=self.__mergeContentTypeD) logger.info("Total validation errors schema level %s : %d", schemaLevel, eCount) self.assertLessEqual(eCount, 1) @unittest.skip("Disable troubleshooting test") def testValidateOptsList(self): schemaLevel = "min" inputPathList = self.__mU.doImport(os.path.join(HERE, "test-output", "failed-path.list"), "list") # inputPathList = glob.glob(self.__testDirPath + "/*.cif") if not inputPathList: return True databaseNameD = {"pdbx_core": ["pdbx_core_entity", "pdbx_core_entry", "pdbx_core_entity_instance", "pdbx_core_entity_instance_validation"]} for ii, subList in enumerate(chunkList(inputPathList[::-1], 40)): if ii < 5: continue eCount = self.__testValidateOpts(databaseNameD=databaseNameD, inputPathList=subList, schemaLevel=schemaLevel, mergeContentTypeD=self.__mergeContentTypeD) logger.info("Chunk %d total validation errors schema level %s : %d", ii, schemaLevel, eCount) # self.assertGreaterEqual(eCount, 20) #@unittest.skip("Disable IHM troubleshooting test") def testValidateOptsIhmRepo(self): schemaLevel = "min" inputPathList = None self.__export = True databaseNameD = {"ihm_dev_full": ["ihm_dev_full"]} databaseNameD = {"ihm_dev": ["ihm_dev"]} eCount = self.__testValidateOpts(databaseNameD=databaseNameD, inputPathList=inputPathList, schemaLevel=schemaLevel, mergeContentTypeD=self.__mergeContentTypeD) logger.info("Total validation errors schema level %s : %d", schemaLevel, eCount) # self.assertGreaterEqual(eCount, 20) # #@unittest.skip("Disable IHM troubleshooting test") def testValidateOptsIhmList(self): #schemaLevel = "full" schemaLevel = "min" inputPathList = glob.glob(self.__testIhmDirPath + "/*.cif") if not inputPathList: return True #databaseNameD = {"ihm_dev_full": ["ihm_dev_full"]} databaseNameD = {"ihm_dev": ["ihm_dev"]} eCount = self.__testValidateOpts(databaseNameD=databaseNameD, inputPathList=inputPathList, schemaLevel=schemaLevel, mergeContentTypeD=self.__mergeContentTypeD) logger.info("Total validation errors schema level %s : %d", schemaLevel, eCount) # self.assertGreaterEqual(eCount, 20) # def __testValidateOpts(self, databaseNameD, inputPathList=None, schemaLevel="full", mergeContentTypeD=None): # eCount = 0 for databaseName in databaseNameD: mergeContentTypes = mergeContentTypeD[databaseName] if databaseName in mergeContentTypeD else None _ = self.__schP.makeSchemaDef(databaseName, dataTyping="ANY", saveSchema=True) pthList = inputPathList if inputPathList else self.__rpP.getLocatorObjList(databaseName, mergeContentTypes=mergeContentTypes) for collectionName in databaseNameD[databaseName]: cD = self.__schP.makeSchema(databaseName, collectionName, encodingType="JSON", level=schemaLevel, saveSchema=True, extraOpts=self.__extraOpts) # dL, cnL = self.__testPrepDocumentsFromContainers( pthList, databaseName, collectionName, styleType="rowwise_by_name_with_cardinality", mergeContentTypes=mergeContentTypes ) # Raises exceptions for schema compliance. try: Draft4Validator.check_schema(cD) except Exception as e: logger.error("%s %s schema validation fails with %s", databaseName, collectionName, str(e)) # valInfo = Draft4Validator(cD, format_checker=FormatChecker()) logger.info("Validating %d documents from %s %s", len(dL), databaseName, collectionName) for ii, dD in enumerate(dL): logger.debug("Schema %s collection %s document %d", databaseName, collectionName, ii) try: cCount = 0 #for error in sorted(valInfo.iter_errors(dD), key=str): # logger.info("schema %s collection %s (%s) path %s error: %s", databaseName, collectionName, cnL[ii], error.path, error.message) # logger.debug("Failing document %d : %r", ii, list(dD.items())) # eCount += 1 # cCount += 1 #if cCount > 0: # logger.info("schema %s collection %s container %s error count %d", databaseName, collectionName, cnL[ii], cCount) except Exception as e: logger.exception("Validation processing error %s", str(e)) return eCount def __testPrepDocumentsFromContainers(self, inputPathList, databaseName, collectionName, styleType="rowwise_by_name_with_cardinality", mergeContentTypes=None): """Test case - create loadable PDBx data from repository files """ try: sd, _, _, _ = self.__schP.getSchemaInfo(databaseName) # dP = DictionaryApiProviderWrapper(self.__cfgOb, self.__cachePath, useCache=False) dictApi = dP.getApiByName(databaseName) rP = DictMethodResourceProvider(self.__cfgOb, configName=self.__configName, cachePath=self.__cachePath, siftsAbbreviated="TEST") dmh = DictMethodRunner(dictApi, modulePathMap=self.__modulePathMap, resourceProvider=rP) # dtf = DataTransformFactory(schemaDefAccessObj=sd, filterType=self.__fTypeRow) sdp = SchemaDefDataPrep(schemaDefAccessObj=sd, dtObj=dtf, workPath=self.__cachePath, verbose=self.__verbose) containerList = self.__rpP.getContainerList(inputPathList) for container in containerList: cName = container.getName() logger.debug("Processing container %s", cName) dmh.apply(container) if self.__export: savePath = os.path.join(HERE, "test-output", cName + "-with-method.cif") #self.__mU.doExport(savePath, [container], fmt="mmcif") # tableIdExcludeList = sd.getCollectionExcluded(collectionName) tableIdIncludeList = sd.getCollectionSelected(collectionName) sliceFilter = sd.getCollectionSliceFilter(collectionName) sdp.setSchemaIdExcludeList(tableIdExcludeList) sdp.setSchemaIdIncludeList(tableIdIncludeList) # docList, containerNameList, _ = sdp.processDocuments( containerList, styleType=styleType, filterType=self.__fTypeRow, dataSelectors=["PUBLIC_RELEASE"], sliceFilter=sliceFilter, collectionName=collectionName ) docList = sdp.addDocumentPrivateAttributes(docList, collectionName) docList = sdp.addDocumentSubCategoryAggregates(docList, collectionName) # mergeS = "-".join(mergeContentTypes) if mergeContentTypes else "" if self.__export and docList: # for ii, doc in enumerate(docList[:1]): for ii, doc in enumerate(docList): cn = containerNameList[ii] fp = os.path.join(HERE, "test-output", "prep-%s-%s-%s-%s.json" % (cn, databaseName, collectionName, mergeS)) self.__mU.doExport(fp, [doc], fmt="json", indent=3) logger.debug("Exported %r", fp) # return docList, containerNameList except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
class SchemaDefLoadercrateDbMultiTests(unittest.TestCase): def __init__(self, methodName="runTest"): super(SchemaDefLoadercrateDbMultiTests, self).__init__(methodName) self.__verbose = True self.__createFlag = True def setUp(self): self.__verbose = True self.__numProc = 2 self.__fileLimit = 100 self.__chunkSize = 0 self.__workPath = os.path.join(HERE, "test-output") self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName) self.__resourceName = "CRATE_DB" self.__schP = SchemaProvider(self.__cfgOb, self.__workPath, useCache=True) self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb, numProc=self.__numProc, fileLimit=self.__fileLimit, cachePath=self.__workPath) # # self.__tableIdSkipD = { "ATOM_SITE": True, "ATOM_SITE_ANISOTROP": True, "__LOAD_STATUS__": True } self.__ioObj = IoAdapter(verbose=self.__verbose) # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime())) def tearDown(self): endTime = time.time() logger.debug("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime) def testConnection(self): try: with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: self.assertNotEqual(client, None) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testSchemaCreate(self): """Create table schema (live) for BIRD, chemical component, and PDBx data.""" try: sd, _, _, _ = self.__schP.getSchemaInfo("bird") ret = self.__schemaCreate(schemaDefObj=sd) self.assertEqual(ret, True) # sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp") ret = self.__schemaCreate(schemaDefObj=sd) self.assertEqual(ret, True) # sd, _, _, _ = self.__schP.getSchemaInfo("pdbx") ret = self.__schemaCreate(schemaDefObj=sd) self.assertEqual(ret, True) # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testSchemaRemove(self): """Remove table schema (live) for BIRD, chemical component, and PDBx data.""" try: sd, _, _, _ = self.__schP.getSchemaInfo("bird") ret = self.__schemaRemove(schemaDefObj=sd) self.assertEqual(ret, True) # sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp") ret = self.__schemaRemove(schemaDefObj=sd) self.assertEqual(ret, True) # sd, _, _, _ = self.__schP.getSchemaInfo("pdbx") ret = self.__schemaRemove(schemaDefObj=sd) self.assertEqual(ret, True) # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testLoadChemCompMulti(self): self.__testLoadFilesMulti("chem_comp") def testLoadBirdMulti(self): self.__testLoadFilesMulti("bird") def testLoadPdbxMulti(self): self.__testLoadFilesMulti("pdbx") def __getPathList(self, fType): pathList = [] if fType == "chem_comp": pathList = self.__rpP.getLocatorObjList("chem_comp") elif fType == "bird": pathList = self.__rpP.getLocatorObjList("bird") pathList.extend(self.__rpP.getLocatorObjList("bird_family")) elif fType == "pdbx": pathList = self.__rpP.getLocatorObjList("pdbx") return pathList def loadInsertMany(self, dataList, procName, optionsD, workingDir): try: _ = workingDir ret = None sd = optionsD["sd"] skipD = optionsD["skip"] ioObj = IoAdapter(verbose=self.__verbose) logger.debug("%s pathlist %r", procName, dataList) with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: sdl = CrateDbLoader(schemaDefObj=sd, ioObj=ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose) ret = sdl.load(inputPathList=dataList, loadType="crate-insert-many", deleteOpt="selected", tableIdSkipD=skipD) # all or nothing here if ret: return dataList, dataList, [] else: return [], [], [] except Exception as e: logger.info("Failing with dataList %r", dataList) logger.exception("Failing with %s", str(e)) return [], [], [] def __testLoadFilesMulti(self, contentType): """Test case - create load w/insert-many all chemical component definition data files - (multiproc test)""" numProc = self.__numProc chunkSize = self.__chunkSize try: # sd, _, _, _ = self.__schP.getSchemaInfo(contentType) if self.__createFlag: self.__schemaCreate(schemaDefObj=sd) optD = {} optD["sd"] = sd if contentType == "pdbx": optD["skip"] = self.__tableIdSkipD else: optD["skip"] = {} # pathList = self.__getPathList(fType=contentType) logger.debug("Input path list %r", pathList) mpu = MultiProcUtil(verbose=True) mpu.setOptions(optionsD=optD) mpu.set(workerObj=self, workerMethod="loadInsertMany") ok, _, _, _ = mpu.runMulti(dataList=pathList, numProc=numProc, numResults=1, chunkSize=chunkSize) self.assertEqual(ok, True) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def __schemaCreate(self, schemaDefObj): """Test case - create table schema using schema definition""" ret = 0 try: tableIdList = schemaDefObj.getTableIdList() sqlGen = SqlGenAdmin(self.__verbose, serverType="cratedb") sqlL = [] for tableId in tableIdList: if tableId in self.__tableIdSkipD: continue tableDefObj = schemaDefObj.getTable(tableId) sqlL.extend( sqlGen.createTableSQL( databaseName=schemaDefObj.getVersionedDatabaseName(), tableDefObj=tableDefObj)) logger.debug("Schema creation SQL string\n %s\n\n", "\n".join(sqlL)) logger.info("Creating schema using database %s", schemaDefObj.getVersionedDatabaseName()) # with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: crQ = CrateDbQuery(dbcon=client, verbose=self.__verbose) ret = crQ.sqlCommandList(sqlCommandList=sqlL) logger.debug("Schema create command returns %r\n", ret) return ret # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def __schemaRemove(self, schemaDefObj): """Test case - remove table schema using schema definition""" ret = 0 try: tableIdList = schemaDefObj.getTableIdList() sqlGen = SqlGenAdmin(self.__verbose, serverType="cratedb") sqlL = [] for tableId in tableIdList: if tableId in self.__tableIdSkipD: continue tableDefObj = schemaDefObj.getTable(tableId) sqlL.extend( sqlGen.dropTableSQL( databaseName=schemaDefObj.getVersionedDatabaseName(), tableDefObj=tableDefObj)) sqlL.extend( sqlGen.dropTableSQL( databaseName=schemaDefObj.getDatabaseName(), tableDefObj=tableDefObj)) logger.debug("Schema Remove SQL string\n %s", "\n".join(sqlL)) with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: crQ = CrateDbQuery(dbcon=client, verbose=self.__verbose) ret = crQ.sqlCommandList(sqlCommandList=sqlL) logger.debug("Schema remove command returns %r\n", ret) return ret # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
class ChemRefEtlWorker(object): """Prepare and load chemical reference data collections.""" def __init__(self, cfgOb, cachePath, useCache=True, numProc=2, chunkSize=10, readBackCheck=False, documentLimit=None, verbose=False): self.__cfgOb = cfgOb self.__cachePath = cachePath self.__useCache = useCache self.__readBackCheck = readBackCheck self.__numProc = numProc self.__chunkSize = chunkSize self.__documentLimit = documentLimit # self.__resourceName = "MONGO_DB" self.__verbose = verbose self.__statusList = [] self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=self.__useCache) # def __updateStatus(self, updateId, databaseName, collectionName, status, startTimestamp): try: sFlag = "Y" if status else "N" desp = DataExchangeStatus() desp.setStartTime(tS=startTimestamp) desp.setObject(databaseName, collectionName) desp.setStatus(updateId=updateId, successFlag=sFlag) desp.setEndTime() self.__statusList.append(desp.getStatus()) return True except Exception as e: logger.exception("Failing with %s", str(e)) return False def load(self, updateId, extResource, loadType="full"): """Load chemical reference integrated data for the input external resource-""" try: self.__statusList = [] desp = DataExchangeStatus() statusStartTimestamp = desp.setStartTime() # if extResource == "DrugBank": databaseName = "drugbank_core" configName = self.__cfgOb.getDefaultSectionName() user = self.__cfgOb.get("_DRUGBANK_AUTH_USERNAME", sectionName=configName) pw = self.__cfgOb.get("_DRUGBANK_AUTH_PASSWORD", sectionName=configName) # dbP = DrugBankProvider(cachePath=self.__cachePath, useCache=self.__useCache, username=user, password=pw) # crExt = ChemRefExtractor(self.__cfgOb) idD = crExt.getChemCompAccessionMapping(extResource) dList = dbP.getDocuments(mapD=idD) # logger.info("Resource %r extracted mapped document length %d", extResource, len(dList)) logger.debug("Objects %r", dList[:2]) sD, _, collectionList, _ = self.__schP.getSchemaInfo( databaseName) collectionName = collectionList[ 0] if collectionList else "unassigned" indexL = sD.getDocumentIndex(collectionName, "primary") logger.info("Database %r collection %r index attributes %r", databaseName, collectionName, indexL) # collectionVersion = sD.getCollectionVersion(collectionName) addValues = {"_schema_version": collectionVersion} # addValues = {} # dl = DocumentLoader( self.__cfgOb, self.__cachePath, self.__resourceName, numProc=self.__numProc, chunkSize=self.__chunkSize, documentLimit=self.__documentLimit, verbose=self.__verbose, readBackCheck=self.__readBackCheck, ) # ok = dl.load(databaseName, collectionName, loadType=loadType, documentList=dList, indexAttributeList=indexL, keyNames=None, addValues=addValues) self.__updateStatus(updateId, databaseName, collectionName, ok, statusStartTimestamp) return True except Exception as e: logger.exception("Failing with %s", str(e)) return False def getLoadStatus(self): return self.__statusList
class CockroachDbLoaderCockroachDbTests(unittest.TestCase): def __init__(self, methodName="runTest"): super(CockroachDbLoaderCockroachDbTests, self).__init__(methodName) self.__verbose = True self.__createFlag = False def setUp(self): self.__verbose = True self.__numProc = 2 self.__fileLimit = 100 self.__workPath = os.path.join(HERE, "test-output") self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName) self.__resourceName = "COCKROACH_DB" self.__schP = SchemaProvider(self.__cfgOb, self.__workPath, useCache=True) self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb, numProc=self.__numProc, fileLimit=self.__fileLimit, cachePath=self.__workPath) # self.__tableIdSkipD = {"ATOM_SITE": True, "ATOM_SITE_ANISOTROP": True} self.__ioObj = IoAdapter(verbose=self.__verbose) # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime())) def tearDown(self): endTime = time.time() logger.debug("Completed %s at %s (%.4f seconds)\n", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime) def testConnection(self): try: with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: self.assertNotEqual(client, None) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testSchemaCreate(self): """Create table schema (live) for BIRD, chemical component, and PDBx data.""" try: sd, _, _, _ = self.__schP.getSchemaInfo("bird") ret = self.__schemaCreate(schemaDefObj=sd) self.assertEqual(ret, True) # sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp") ret = self.__schemaCreate(schemaDefObj=sd) self.assertEqual(ret, True) # sd, _, _, _ = self.__schP.getSchemaInfo("pdbx") ret = self.__schemaCreate(schemaDefObj=sd) self.assertEqual(ret, True) # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testSchemaRemove(self): """Remove table schema (live) for BIRD, chemical component, and PDBx data.""" try: sd, _, _, _ = self.__schP.getSchemaInfo("bird") ret = self.__schemaRemove(schemaDefObj=sd) self.assertEqual(ret, True) # sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp") ret = self.__schemaRemove(schemaDefObj=sd) self.assertEqual(ret, True) # sd, _, _, _ = self.__schP.getSchemaInfo("pdbx") ret = self.__schemaRemove(schemaDefObj=sd) self.assertEqual(ret, True) # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testLoadInsertBirdReference(self): try: sd, _, _, _ = self.__schP.getSchemaInfo("bird") if self.__createFlag: self.__schemaCreate(schemaDefObj=sd) inputPathList = self.__rpP.getLocatorObjList("bird") inputPathList.extend(self.__rpP.getLocatorObjList("bird_family")) with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose) ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert", deleteOpt="selected") self.assertEqual(ret, True) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testLoadInsertManyBirdReference(self): try: sd, _, _, _ = self.__schP.getSchemaInfo("bird") if self.__createFlag: self.__schemaCreate(schemaDefObj=sd) inputPathList = self.__rpP.getLocatorObjList("bird") inputPathList.extend(self.__rpP.getLocatorObjList("bird_family")) with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose) ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert-many", deleteOpt="selected") self.assertEqual(ret, True) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testLoadInsertChemCompReference(self): try: sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp") if self.__createFlag: self.__schemaCreate(schemaDefObj=sd) inputPathList = self.__rpP.getLocatorObjList("chem_comp") with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose) ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert", deleteOpt="selected") self.assertEqual(ret, True) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testLoadInsertManyChemCompReference(self): try: sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp") if self.__createFlag: self.__schemaCreate(schemaDefObj=sd) inputPathList = self.__rpP.getLocatorObjList("chem_comp") with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose) ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert-many", deleteOpt="selected") self.assertEqual(ret, True) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testLoadInsertPdbxExampleFiles(self): try: sd, _, _, _ = self.__schP.getSchemaInfo("pdbx") if self.__createFlag: self.__schemaCreate(schemaDefObj=sd) inputPathList = self.__rpP.getLocatorObjList("pdbx") with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose) ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert", deleteOpt="selected", tableIdSkipD=self.__tableIdSkipD) self.assertEqual(ret, True) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testLoadInsertManyPdbxExampleFiles(self): try: sd, _, _, _ = self.__schP.getSchemaInfo("pdbx") if self.__createFlag: self.__schemaCreate(schemaDefObj=sd) inputPathList = self.__rpP.getLocatorObjList("pdbx") with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose) ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert-many", deleteOpt="selected", tableIdSkipD=self.__tableIdSkipD) self.assertEqual(ret, True) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def __schemaCreateSQL(self, schemaDefObj): """Test case - create table schema using schema definition""" sqlL = [] try: tableIdList = schemaDefObj.getTableIdList() sqlGen = SqlGenAdmin(self.__verbose, serverType="CockroachDb") dbName = schemaDefObj.getVersionedDatabaseName() sqlL = sqlGen.createDatabaseSQL(dbName) for tableId in tableIdList: tableDefObj = schemaDefObj.getTable(tableId) sqlL.extend(sqlGen.createTableSQL(databaseName=schemaDefObj.getVersionedDatabaseName(), tableDefObj=tableDefObj)) logger.debug("\nSchema creation SQL string\n %s\n\n", "\n".join(sqlL)) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() return sqlL def __schemaCreate(self, schemaDefObj): """Test case - create table schema using schema definition""" ret = 0 try: tableIdList = schemaDefObj.getTableIdList() sqlGen = SqlGenAdmin(self.__verbose, serverType="CockroachDb") dbName = schemaDefObj.getVersionedDatabaseName() sqlL = sqlGen.createDatabaseSQL(dbName) for tableId in tableIdList: tableDefObj = schemaDefObj.getTable(tableId) sqlL.extend(sqlGen.createTableSQL(databaseName=schemaDefObj.getVersionedDatabaseName(), tableDefObj=tableDefObj)) logger.debug("\nSchema creation SQL string\n %s\n\n", "\n".join(sqlL)) logger.info("Creating schema using database %s", schemaDefObj.getVersionedDatabaseName()) # with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: crQ = CockroachDbQuery(dbcon=client, verbose=self.__verbose) ret = crQ.sqlCommandList(sqlCommandList=sqlL) # ret = crQ.sqlCommand(' '.join(sqlL)) logger.info("Schema create command returns %r\n", ret) return ret # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def __schemaRemove(self, schemaDefObj): """Test case - remove table schema using schema definition""" ret = 0 try: dbName = schemaDefObj.getVersionedDatabaseName() sqlGen = SqlGenAdmin(self.__verbose, serverType="CockroachDb") sqlL = sqlGen.removeDatabaseSQL(dbName) logger.debug("Schema Remove SQL string\n %s", "\n".join(sqlL)) with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client: crQ = CockroachDbQuery(dbcon=client, verbose=self.__verbose) ret = crQ.sqlCommandList(sqlCommandList=sqlL) # ret = crQ.sqlCommand(' '.join(sqlL)) logger.debug("Schema remove command returns %r\n", ret) return ret # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
class SchemaDefDataPrepTests(unittest.TestCase): def __init__(self, methodName="runTest"): super(SchemaDefDataPrepTests, self).__init__(methodName) self.__loadPathList = [] self.__verbose = True def setUp(self): self.__isMac = platform.system() == "Darwin" self.__excludeType = None if self.__isMac else "optional" self.__numProc = 2 mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__outputPath = os.path.join(HERE, "test-output") self.__savedOutputPath = os.path.join(HERE, "test-saved-output") configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__configName = configName self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=mockTopPath) self.__mU = MarshalUtil(workPath=self.__cachePath) self.__discoveryMode = self.__cfgOb.get("DISCOVERY_MODE", sectionName=configName, default="local") self.__fileLimit = 100 if self.__discoveryMode == "local" else 10 self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=True) self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb, numProc=self.__numProc, fileLimit=self.__fileLimit, cachePath=self.__cachePath) # # self.__fTypeRow = "drop-empty-attributes|drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs" self.__fTypeCol = "drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs" self.__chemCompMockLen = 24 self.__pdbxMockLen = 30 # removes timestamped data items to allow diffs.) excludeExtras = ["rcsb_load_status"] # excludeExtras = [] # self.__verbose = True self.__modulePathMap = self.__cfgOb.get( "DICT_METHOD_HELPER_MODULE_PATH_MAP", sectionName=configName) # self.__exportFlag = True self.__diffFlag = False # self.__simpleTestCaseList = [ { "contentType": "chem_comp", "mockLength": self.__chemCompMockLen, "filterType": self.__fTypeRow, "styleType": "rowwise_by_name", "mergeContentTypes": None, "rejectLength": 2, }, { "contentType": "chem_comp", "mockLength": self.__chemCompMockLen, "filterType": self.__fTypeRow, "styleType": "rowwise_no_name", "mergeContentTypes": None, "rejectLength": 2, }, { "contentType": "chem_comp", "mockLength": self.__chemCompMockLen, "filterType": self.__fTypeCol, "styleType": "columnwise_by_name", "mergeContentTypes": None, "rejectLength": 2, }, { "contentType": "chem_comp", "mockLength": self.__chemCompMockLen, "filterType": self.__fTypeRow, "styleType": "rowwise_by_name", "mergeContentTypes": None, "rejectLength": 2, }, { "contentType": "pdbx_core", "mockLength": self.__pdbxMockLen, "filterType": self.__fTypeRow, "styleType": "rowwise_by_name", "mergeContentTypes": None, "rejectLength": 0, }, ] # self.__fullTestCaseList = [ { "contentType": "pdbx_core", "mockLength": self.__pdbxMockLen, "filterType": self.__fTypeRow, "styleType": "rowwise_by_name_with_cardinality", "mergeContentTypes": ["vrpt"], "rejectLength": 0, "excludeExtras": excludeExtras, }, { "contentType": "bird_chem_comp_core", "mockLength": self.__chemCompMockLen, "filterType": self.__fTypeRow, "styleType": "rowwise_by_name_with_cardinality", "mergeContentTypes": None, "rejectLength": 2, "excludeExtras": excludeExtras, }, ] # self.__fullTestCaseListA = [ { "contentType": "pdbx_core", "mockLength": self.__pdbxMockLen, "filterType": self.__fTypeRow, "styleType": "rowwise_by_name_with_cardinality", "mergeContentTypes": ["vrpt"], "rejectLength": 0, "excludeExtras": excludeExtras, }, ] # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime())) def tearDown(self): unitS = "MB" if platform.system() == "Darwin" else "GB" rusageMax = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss logger.info("Maximum resident memory size %.4f %s", rusageMax / 1.0e6, unitS) endTime = time.time() logger.info("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime) def __timeStep(self, msg): endTime = time.time() logger.info("Completed %s at %s (%.4f seconds)", msg, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime) def testSimpleSchemaDefDataPrep(self): for tcD in self.__simpleTestCaseList: rejectLength = 0 if self.__discoveryMode == "remote" else tcD[ "rejectLength"] mockLength = self.__fileLimit if self.__discoveryMode == "remote" else tcD[ "mockLength"] if tcD["contentType"] == "bird_chem_comp_core" and self.__discoveryMode == "remote": logger.info("Skipping %r in discovery mode %r", tcD["contentType"], self.__discoveryMode) continue self.__simpleSchemaDataPrep( tcD["contentType"], tcD["filterType"], tcD["styleType"], mockLength, rejectLength=rejectLength, mergeContentTypes=tcD["mergeContentTypes"]) def testFullSchemaDefDataPrep(self): for tcD in self.__fullTestCaseList: rejectLength = 0 if self.__discoveryMode == "remote" else tcD[ "rejectLength"] mockLength = self.__fileLimit if self.__discoveryMode == "remote" else tcD[ "mockLength"] if tcD["contentType"] == "bird_chem_comp_core" and self.__discoveryMode == "remote": logger.info("Skipping %r in discovery mode %r", tcD["contentType"], self.__discoveryMode) continue self.__fullSchemaDataPrep( tcD["contentType"], tcD["filterType"], tcD["styleType"], mockLength, rejectLength=rejectLength, mergeContentTypes=tcD["mergeContentTypes"], excludeExtras=tcD["excludeExtras"], ) def __simpleSchemaDataPrep(self, contentType, filterType, styleType, mockLength, rejectLength=0, dataSelectors=None, mergeContentTypes=None): """Internal method for preparing file-based data NOT requiring dynamic methods, slicing, or key injection. Args: contentType (str): Content type name filterType (str): List of data processing options (separated by '|') (e.g. "drop-empty-attributes|drop-empty-tables|skip-max-width|...) styleType (str): organization of output document (e.g. rowise-by-name) mockLength (int): Expected length of the test data for the input content type rejectLength (int, optional): number of input data sets rejected by the dataselection criteria. Defaults to 0. dataSelectors (list of str, optional): data selection criteria. Defaults to None. mergeContentTypes (list of str, optional): list content types to merge with the input data set. Defaults to None. (e.g. ['vrpt']) """ try: dataSelectors = dataSelectors if dataSelectors else [ "PUBLIC_RELEASE" ] dD = self.__schP.makeSchemaDef(contentType, dataTyping="ANY", saveSchema=True) _ = SchemaDefAccess(dD) inputPathList = self.__rpP.getLocatorObjList( contentType=contentType, mergeContentTypes=mergeContentTypes) sd, _, _, _ = self.__schP.getSchemaInfo(databaseName=contentType, dataTyping="ANY") dtf = DataTransformFactory(schemaDefAccessObj=sd, filterType=filterType) sdp = SchemaDefDataPrep(schemaDefAccessObj=sd, dtObj=dtf, workPath=self.__cachePath, verbose=self.__verbose) # logger.debug("For %s mock length %d length of path list %d\n", contentType, mockLength, len(inputPathList)) self.assertGreaterEqual(len(inputPathList), mockLength) tableDataDictList, containerNameList, rejectList = sdp.fetchDocuments( inputPathList, styleType=styleType, filterType=filterType, dataSelectors=dataSelectors) logger.debug( "For %s mock length %d reject length %d length of tddl list %d\n", contentType, mockLength, rejectLength, len(tableDataDictList)) self.assertGreaterEqual(len(tableDataDictList), mockLength - rejectLength) self.assertGreaterEqual(len(containerNameList), mockLength - rejectLength) if rejectList: logger.debug("For %s rejecting components %r", contentType, rejectList) # self.assertEqual(len(rejectList), rejectLength) fName = "simple-prep-%s-%s.json" % (contentType, styleType) if self.__exportFlag: fPath = os.path.join(self.__outputPath, fName) self.__mU.doExport(fPath, tableDataDictList, fmt="json", indent=3) if self.__diffFlag: fPath = os.path.join(self.__savedOutputPath, fName) refDocList = self.__mU.doImport(fPath, fmt="json") self.assertEqual(len(refDocList), len(tableDataDictList)) # jD = diff(refDocList, tableDataDictList, syntax="explicit", marshal=True) if jD: _, fn = os.path.split(fPath) bn, _ = os.path.splitext(fn) fPath = os.path.join(self.__outputPath, bn + "-diff.json") logger.debug("jsondiff for %s %s = \n%s", contentType, styleType, pprint.pformat(jD, indent=3, width=100)) self.__mU.doExport(fPath, jD, fmt="json", indent=3) self.assertEqual(len(jD), 0) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def __logDocumentOrder(self, docList): for doc in docList: logger.debug("keys %r", list(doc.keys())) def __filterDocuments(self, docList, excludeList=None): excludeList = excludeList if excludeList else [] for doc in docList: for excl in excludeList: if excl in doc: del doc[excl] def __fullSchemaDataPrep(self, contentType, filterType, styleType, mockLength, rejectLength=0, dataSelectors=None, mergeContentTypes=None, excludeExtras=None): """Internal method for preparing file-based data requiring dynamic methods, slicing, or key injection. Args: contentType (str): Content type name filterType (str): List of data processing options (separated by '|') (e.g. "drop-empty-attributes|drop-empty-tables|skip-max-width|...) styleType (str): organization of output document (e.g. rowise-by-name) mockLength (int): Expected length of the test data for the input content type rejectLength (int, optional): number of input data sets rejected by the dataselection criteria. Defaults to 0. dataSelectors (list of str, optional): data selection criteria. Defaults to None. mergeContentTypes (list of str, optional): list content types to merge with the input data set. Defaults to None. (e.g. ['vrpt']) """ try: excludeExtras = excludeExtras if excludeExtras else [] _ = mockLength _ = rejectLength dD = self.__schP.makeSchemaDef(contentType, dataTyping="ANY", saveSchema=True) _ = SchemaDefAccess(dD) inputPathList = self.__rpP.getLocatorObjList( contentType=contentType, mergeContentTypes=mergeContentTypes) sd, _, collectionNameList, _ = self.__schP.getSchemaInfo( databaseName=contentType, dataTyping="ANY") # dP = DictionaryApiProviderWrapper(self.__cachePath, cfgOb=self.__cfgOb, configName=self.__configName, useCache=True) dictApi = dP.getApiByName(contentType) # rP = DictMethodResourceProvider( self.__cfgOb, configName=self.__configName, cachePath=self.__cachePath, restoreUseStash=False, restoreUseGit=True, providerTypeExclude=self.__excludeType, ) dmh = DictMethodRunner(dictApi, modulePathMap=self.__modulePathMap, resourceProvider=rP) # dtf = DataTransformFactory(schemaDefAccessObj=sd, filterType=filterType) sdp = SchemaDefDataPrep(schemaDefAccessObj=sd, dtObj=dtf, workPath=self.__cachePath, verbose=self.__verbose) containerList = self.__rpP.getContainerList(inputPathList) for container in containerList: cName = container.getName() logger.debug("Processing container %s", cName) dmh.apply(container) # for collectionName in collectionNameList: tableIdExcludeList = sd.getCollectionExcluded(collectionName) tableIdIncludeList = sd.getCollectionSelected(collectionName) sliceFilter = sd.getCollectionSliceFilter(collectionName) sdp.setSchemaIdExcludeList(tableIdExcludeList) sdp.setSchemaIdIncludeList(tableIdIncludeList) # docList, _, _ = sdp.processDocuments( containerList, styleType=styleType, sliceFilter=sliceFilter, filterType=filterType, dataSelectors=dataSelectors, collectionName=collectionName) docList = sdp.addDocumentPrivateAttributes( docList, collectionName) docList = sdp.addDocumentSubCategoryAggregates( docList, collectionName) # Special exclusions for the test harness. (removes timestamped data items to allow diffs.) self.__filterDocuments(docList, excludeExtras) mergeS = "-".join( mergeContentTypes) if mergeContentTypes else "" fName = "full-prep-%s-%s-%s-%s.json" % ( contentType, collectionName, mergeS, styleType) if self.__exportFlag: self.__logDocumentOrder(docList) fPath = os.path.join(self.__outputPath, fName) self.__mU.doExport(fPath, docList, fmt="json", indent=3) logger.debug("Exported %r", fPath) # if self.__diffFlag: fPath = os.path.join(self.__savedOutputPath, fName) refDocList = self.__mU.doImport(fPath, fmt="json") self.assertEqual(len(refDocList), len(docList)) logger.debug("For %s %s len refDocList %d", contentType, collectionName, len(refDocList)) logger.debug("For %s %s len docList %d", contentType, collectionName, len(docList)) jD = diff(refDocList, docList, syntax="explicit", marshal=True) if jD: _, fn = os.path.split(fPath) bn, _ = os.path.splitext(fn) fPath = os.path.join(self.__outputPath, bn + "-diff.json") logger.debug("jsondiff for %s %s = \n%s", contentType, collectionName, pprint.pformat(jD, indent=3, width=100)) self.__mU.doExport(fPath, jD, fmt="json", indent=3) self.assertEqual(len(jD), 0) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()