def setUp(self): # # mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=mockTopPath) # self.__cfgOb.dump() self.__resourceName = "MONGO_DB" self.__readBackCheck = True self.__numProc = 2 self.__chunkSize = 10 self.__documentLimit = None self.__filterType = "assign-dates" # self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__sandboxPath = self.__cfgOb.getPath("RCSB_EXCHANGE_SANDBOX_PATH", sectionName=configName) # sample data set self.__updateId = "2019_23" # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): self.__verbose = True # self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") self.__pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__updateId = "2018_25" self.__export = False # configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=self.__pathConfig, defaultSectionName=configName, mockTopPath=self.__mockTopPath) self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=True) self.__sandboxPath = self.__cfgOb.getPath("RCSB_EXCHANGE_SANDBOX_PATH", sectionName=configName) # self.__mU = MarshalUtil(workPath=self.__cachePath) self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): self.__verbose = True # mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") # configName = "site_info_configuration" cfgOb = ConfigUtil(configPath=pathConfig, defaultSectionName=configName, mockTopPath=mockTopPath) self.__pathClusterData = cfgOb.getPath( "RCSB_SEQUENCE_CLUSTER_DATA_PATH", sectionName=configName) # sample data set self.__dataSetId = "2018_23" # self.__levels = ["100", "95", "90", "70", "50", "30"] self.__levels = ["95"] # self.__workPath = os.path.join(HERE, "test-output") self.__pathSaveStyleCif = os.path.join(HERE, "test-output", "cluster-data-cif.json") self.__pathSaveStyleDocSequence = os.path.join( HERE, "test-output", "cluster-data-doc-sequence.json") self.__pathSaveStyleDocCluster = os.path.join( HERE, "test-output", "cluster-data-doc-cluster.json") # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): # self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml") # configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath) # self.__cachePath = os.path.join(TOPDIR, "CACHE") # self.__cacheKwargs = {"fmt": "json", "indent": 3} self.__exdbCacheDirPath = os.path.join( self.__cachePath, self.__cfgOb.get("EXDB_CACHE_DIR", sectionName=configName)) # # Reference sequence test data cache - # self.__refDbCachePath = os.path.join(HERE, "test-output", "unp-data-test-cache.json") self.__cacheKwargs = {"fmt": "json", "indent": 3} self.__useCache = False self.__fetchLimit = None # # Entity polymer extracted data ... # self.__entryLimit = 500 # self.__mU = MarshalUtil() # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): self.__verbose = True mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=pathConfig, defaultSectionName=configName, mockTopPath=mockTopPath) self.__docHelper = DocumentDefinitionHelper(cfgOb=self.__cfgOb) # self.__pathPdbxDictionaryFile = self.__cfgOb.getPath( "PDBX_DICT_LOCATOR", sectionName=configName) self.__pathRcsbDictionaryFile = self.__cfgOb.getPath( "RCSB_DICT_LOCATOR", sectionName=configName) self.__pathVrptDictionaryFile = self.__cfgOb.getPath( "VRPT_DICT_LOCATOR", sectionName=configName) # self.__mU = MarshalUtil() # self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__dP = DictionaryApiProviderWrapper(self.__cfgOb, self.__cachePath, useCache=True) self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): # # self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml") # # Caution: this is very site specific setting ! configName = "site_info_remote" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath) if configName != "site_info_configuration": self.__cfgOb.replaceSectionName("site_info_configuration", configName) # self.__workPath = os.path.join(HERE, "test-cache-preserve") # self.__entityPolymerCachePath = os.path.join( self.__workPath, "entity-polymer-data-cache.pic") self.__entityPolymerCacheKwargs = {"fmt": "pickle"} self.__useEntityPolymerCache = True # self.__refDbCachePath = os.path.join(self.__workPath, "unp-data-test-cache.json") self.__refDbCacheKwargs = {"fmt": "json", "indent": 3} # self.__refDbUseCache = True self.__fetchLimit = 500 # self.__mU = MarshalUtil() # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): # # self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml") # configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath) # self.__cachePath = os.path.join(TOPDIR, "CACHE") # self.__cacheKwargs = {"fmt": "json", "indent": 3} self.__exdbDirPath = os.path.join( self.__cachePath, self.__cfgOb.get("EXDB_CACHE_DIR", sectionName=configName)) # self.__mU = MarshalUtil() self.__entryLimitTest = 20 # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def __init__(self, **kwargs): # Configuration Details configPath = kwargs.get("configPath", "exdb-config-example.yml") self.__configName = kwargs.get("configName", "site_info_configuration") mockTopPath = kwargs.get("mockTopPath", None) self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=self.__configName, mockTopPath=mockTopPath) # self.__cachePath = kwargs.get("cachePath", ".") self.__cachePath = os.path.abspath(self.__cachePath) self.__debugFlag = kwargs.get("debugFlag", False) if self.__debugFlag: logger.setLevel(logging.DEBUG) # # Rebuild or check resource cache rebuildCache = kwargs.get("rebuildCache", False) self.__useCache = not rebuildCache restoreUseGit = kwargs.get("restoreUseGit", True) restoreUseStash = kwargs.get("restoreUseStash", True) providerTypeExclude = kwargs.get("providerTypeExclude", None) # self.__cacheStatus = True rebuildCache = False if rebuildCache: self.__cacheStatus = self.buildResourceCache( rebuildCache=rebuildCache, providerTypeExclude=providerTypeExclude, restoreUseGit=restoreUseGit, restoreUseStash=restoreUseStash, ) logger.debug("Cache status if %r", self.__cacheStatus)
def setUp(self): # # mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=mockTopPath) # self.__resourceName = "MONGO_DB" self.__failedFilePath = os.path.join(HERE, "test-output", "failed-list.txt") self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__readBackCheck = True self.__numProc = 2 self.__chunkSize = 10 self.__fileLimit = None self.__documentStyle = "rowwise_by_name_with_cardinality" self.__ldList = [ # {"databaseName": "chem_comp_core", "collectionNameList": None, "loadType": "full", "mergeContentTypes": None, "validationLevel": "min"}, { "databaseName": "bird_chem_comp_core", "collectionNameList": None, "loadType": "full", "mergeContentTypes": None, "validationLevel": "full", "updateSchemaOnReplace": False, "status": True, }, { "databaseName": "bird_chem_comp_core", "collectionNameList": None, "loadType": "replace", "mergeContentTypes": None, "validationLevel": "full", "updateSchemaOnReplace": True, "status": True, }, { "databaseName": "pdbx_core", "collectionNameList": None, "loadType": "full", "mergeContentTypes": ["vrpt"], "validationLevel": "min", "updateSchemaOnReplace": False, "status": True, }, { "databaseName": "pdbx_core", "collectionNameList": None, "loadType": "replace", "mergeContentTypes": ["vrpt"], "validationLevel": "full", "updateSchemaOnReplace": True, "status": True, }, ] # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
class DictionaryProviderTests(unittest.TestCase): def setUp(self): mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__dirPath = os.path.join(self.__cachePath, "dictionaries") configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__configName = configName self.__contentInfoConfigName = "content_info_helper_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=mockTopPath) dictLocatorMap = self.__cfgOb.get( "DICT_LOCATOR_CONFIG_MAP", sectionName=self.__contentInfoConfigName) schemaName = "pdbx_core" self.__dictLocators = [ self.__cfgOb.getPath(configLocator, sectionName=self.__configName) for configLocator in dictLocatorMap[schemaName] ] # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime())) def tearDown(self): endTime = time.time() logger.debug("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime) def testResourceCache(self): """Test case - generate and check dictonary artifact and api caches """ try: logger.debug("Dictionary locators %r", self.__dictLocators) dp = DictionaryApiProvider(dirPath=self.__dirPath, useCache=False) dApi = dp.getApi(self.__dictLocators) ok = dApi.testCache() self.assertTrue(ok) title = dApi.getDictionaryTitle() logger.debug("Title %r", title) self.assertEqual( title, "mmcif_pdbx.dic,rcsb_mmcif_ext.dic,vrpt_mmcif_ext.dic") # revL = dApi.getDictionaryHistory() numRev = dApi.getDictionaryRevisionCount() logger.debug("Number of dictionary revisions (numRev) %r", numRev) self.assertGreater(numRev, 220) # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def setUp(self): self.__numProc = 2 # self.__fileLimit = 200 self.__fileLimit = None self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example-ihm.yml") configName = "site_info_configuration" self.__configName = configName self.__cfgOb = ConfigUtil(configPath=self.__configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath) self.__mU = MarshalUtil(workPath=self.__cachePath) #self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=False, rebuildFlag=True) self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=True) self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb, numProc=self.__numProc, fileLimit=self.__fileLimit, cachePath=self.__cachePath) # self.__birdRepoPath = self.__cfgOb.getPath("BIRD_REPO_PATH", sectionName=configName) # self.__fTypeRow = "drop-empty-attributes|drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs" self.__fTypeCol = "drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs" self.__verbose = True # self.__modulePathMap = self.__cfgOb.get("DICT_METHOD_HELPER_MODULE_PATH_MAP", sectionName=configName) self.__testDirPath = os.path.join(HERE, "test-output", "pdbx-files") self.__testIhmDirPath = os.path.join(HERE, "test-output", "ihm-files") self.__export = True # #self.__extraOpts = None # The following for extended parent/child info - self.__extraOpts = 'addParentRefs|addPrimaryKey' # self.__alldatabaseNameD = { "ihm_dev": ["ihm_dev"], "pdbx": ["pdbx", "pdbx_ext"], "pdbx_core": ["pdbx_core_entity", "pdbx_core_entry", "pdbx_core_assembly", "pdbx_core_entity_instance", "pdbx_core_entity_instance_validation"], "bird": ["bird"], "bird_family": ["family"], "chem_comp": ["chem_comp"], "bird_chem_comp": ["bird_chem_comp"], "bird_chem_comp_core": ["bird_chem_comp_core"], } self.__databaseNameD = { "pdbx_core": ["pdbx_core_entity", "pdbx_core_entry", "pdbx_core_assembly", "pdbx_core_entity_instance", "pdbx_core_entity_instance_validation"], "bird_chem_comp_core": ["bird_chem_comp_core"], } self.__mergeContentTypeD = {"pdbx_core": ["vrpt"]} # self.__databaseNameD = {"chem_comp_core": ["chem_comp_core"], "bird_chem_comp_core": ["bird_chem_comp_core"]} # self.__databaseNameD = {"ihm_dev_full": ["ihm_dev_full"]} # self.__databaseNameD = {"pdbx_core": ["pdbx_core_entity_instance_validation"]} # self.__databaseNameD = {"pdbx_core": ["pdbx_core_entity_monomer"]} self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def testWriteFromIniConfig(self): try: cfgOb = ConfigUtil(configPath=self.__inpPathConfigIni, mockTopPath=self.__dataPath) ok = cfgOb.writeConfig(self.__outPathConfigIni, configFormat="ini") ok = cfgOb.writeConfig(self.__outPathConfigYaml, configFormat="yaml") self.assertTrue(ok) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() return {}
def setUp(self): configPath = os.path.join(HERE, "test-data", "drugbank-config-example.yml") configName = "site_info_configuration" cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName) self.__user = cfgOb.get("_DRUGBANK_AUTH_USERNAME", sectionName=configName) self.__pw = cfgOb.get("_DRUGBANK_AUTH_PASSWORD", sectionName=configName) self.__cachePath = os.path.join(HERE, "test-output", "CACHE") self.__fastaPath = os.path.join(HERE, "test-output", "drugbank-targets.fa") self.__taxonPath = os.path.join(HERE, "test-output", "drugbank-targets-taxon.tdd") # self.__seqMatchResultsPath = os.path.join(HERE, "test-data", "drugbank-vs-pdbprent-filtered-results.json.gz") self.__startTime = time.time() logger.info("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def __init__(self, **kwargs): # Configuration Details configPath = kwargs.get("configPath", "exdb-config-example.yml") self.__configName = kwargs.get("configName", "site_info_configuration") mockTopPath = kwargs.get("mockTopPath", None) self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=self.__configName, mockTopPath=mockTopPath) # self.__cachePath = kwargs.get("cachePath", ".") self.__cachePath = os.path.abspath(self.__cachePath) self.__debugFlag = kwargs.get("debugFlag", False) if self.__debugFlag: logger.setLevel(logging.DEBUG)
def setUp(self): configPath = os.path.join(HERE, "test-data", "pharos-config-example.yml") self.__configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=self.__configName) self.__user = self.__cfgOb.get("_MYSQL_DB_USER_NAME", sectionName=self.__configName) self.__pw = self.__cfgOb.get("_MYSQL_DB_PASSWORD", sectionName=self.__configName) self.__cachePath = os.path.join(HERE, "test-output", "CACHE") self.__dirPath = os.path.join(self.__cachePath, "Pharos-targets") self.__dataPath = os.path.join(HERE, "test-data") # self.__pharosFixture()
def testReadYamlConfigWithAutoAppend(self): try: cfgOb = ConfigUtil( configPath=self.__inpPathConfigAutoYaml, configFormat="yaml", mockTopPath=self.__mockTopPath, defaultSectionName="site_info_1", cachePath=None, useCache=False, ) ok = cfgOb.appendConfig(self.__inpPathConfigAppendYaml, configFormat="yaml") self.assertTrue(ok) # for sName in [ "section_appended_1", "section_appended_2", "Section1", "Section2" ]: pathBird = cfgOb.getPath("BIRD_REPO_PATH", sectionName=sName) pathPdbx = cfgOb.getPath("PDBX_REPO_PATH", sectionName=sName) # self.assertEqual( pathBird, os.path.join(self.__mockTopPath, "MOCK_BIRD_REPO")) self.assertEqual( pathPdbx, os.path.join(self.__mockTopPath, "MOCK_PDBX_SANDBOX")) # # cfgOb = ConfigUtil( configPath=self.__inpPathConfigAutoYaml, configFormat="yaml", mockTopPath=self.__mockTopPath, defaultSectionName="site_info_1", cachePath=None, useCache=True, ) ok = cfgOb.appendConfig(self.__inpPathConfigAppendYaml, configFormat="yaml") self.assertTrue(ok) # for sName in [ "section_appended_1", "section_appended_2", "Section1", "Section2" ]: pathBird = cfgOb.getPath("BIRD_REPO_PATH", sectionName=sName) pathPdbx = cfgOb.getPath("PDBX_REPO_PATH", sectionName=sName) # self.assertEqual( pathBird, os.path.join(self.__mockTopPath, "MOCK_BIRD_REPO")) self.assertEqual( pathPdbx, os.path.join(self.__mockTopPath, "MOCK_PDBX_SANDBOX")) # except Exception as e: logger.error("Failing with %s", str(e)) self.fail()
def setUp(self): self.__verbose = True self.__numProc = 2 self.__fileLimit = 100 self.__chunkSize = 0 self.__workPath = os.path.join(HERE, "test-output") self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName) self.__resourceName = "CRATE_DB" self.__schP = SchemaProvider(self.__cfgOb, self.__workPath, useCache=True) self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb, numProc=self.__numProc, fileLimit=self.__fileLimit, cachePath=self.__workPath) # # self.__tableIdSkipD = { "ATOM_SITE": True, "ATOM_SITE_ANISOTROP": True, "__LOAD_STATUS__": True } self.__ioObj = IoAdapter(verbose=self.__verbose) # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): self.__isMac = platform.system() == "Darwin" self.__excludeType = None if self.__isMac else "optional" self.__verbose = True # fileLimit = 100 numProc = 2 self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__workPath = os.path.join(HERE, "test-output") mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") # configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=mockTopPath) self.__resourceName = "MYSQL_DB" # self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=True) self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb, numProc=numProc, fileLimit=fileLimit, cachePath=self.__cachePath) # # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): self.__cachePath = os.path.join(HERE, "test-output", "CACHE") # self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") self.__configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml") configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=self.__configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath)
def setUp(self): self.__dbName = "test_database" self.__collectionName = "test_collection" # configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName) self.__resourceName = "MONGO_DB" self.__connectD = self.__assignResource(self.__cfgOb, resourceName=self.__resourceName, sectionName=configName) # self.__cObj = self.__open(self.__connectD) # self.__mongoSchema = { "bsonType": "object", "required": ["strField1", "intField1", "enumField1", "dblField1"], "properties": { "strField1": {"bsonType": "string", "description": "must be a string and is required"}, "strField2": {"bsonType": "string", "description": "must be a string and is not required"}, "strField3": {"bsonType": "string", "description": "must be a string and is not required"}, "intField1": {"bsonType": "int", "minimum": 1, "maximum": 100, "exclusiveMaximum": False, "description": "must be an integer in [ 1, 100 ] and is required"}, "enumField1": {"enum": ["v1", "v2", "v3", "v4", None], "description": "can only be one of the enum values and is required"}, "dblField1": {"bsonType": ["double"], "minimum": 0, "description": "must be a double and is required"}, "dateField1": {"bsonType": "date", "description": "must be a date and is not required"}, }, } self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): self.__isMac = platform.system() == "Darwin" self.__doLoad = True if self.__isMac else False # # self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml") # configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath) # self.__cachePath = os.path.join(TOPDIR, "CACHE") # self.__mU = MarshalUtil() # self.__readBackCheck = True self.__numProc = 2 self.__chunkSize = 10 self.__documentLimit = None self.__debugFlag = False self.__loadType = "full" self.__useCache = True # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def testRoundTripYaml(self): try: cfgOb = ConfigUtil(configPath=self.__inpPathConfigYaml, configFormat="yaml", mockTopPath=self.__mockTopPath, roundTrip=True) ok = cfgOb.writeConfig(self.__outPathConfigIni, configFormat="ini") self.assertTrue(ok) ok = cfgOb.writeConfig(self.__outPathConfigYaml, configFormat="yaml") self.assertTrue(ok) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() return {}
def setUp(self): self.__verbose = True mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__cfgOb = ConfigUtil(configPath=pathConfig, defaultSectionName=configName, mockTopPath=mockTopPath) self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=False) # self.__validationLevels = self.__cfgOb.getList( "VALIDATION_LEVELS_TEST", sectionName="database_catalog_configuration") self.__encodingTypes = self.__cfgOb.getList( "ENCODING_TYPES_TEST", sectionName="database_catalog_configuration") # buildAll = True if buildAll: self.__databaseNameList = self.__cfgOb.getList( "DATABASE_NAMES_DEPLOYED", sectionName="database_catalog_configuration") self.__dataTypingList = self.__cfgOb.getList( "DATATYPING_DEPLOYED", sectionName="database_catalog_configuration") # else: self.__databaseNameList = self.__cfgOb.getList( "DATABASE_NAMES_TEST", sectionName="database_catalog_configuration") # self.__databaseNameList = ["repository_holdings"] self.__dataTypingList = self.__cfgOb.getList( "DATATYPING_TEST", sectionName="database_catalog_configuration") # self.__databaseNameList = ["sequence_clusters"] self.__saveSchema = True self.__compareDefSchema = False self.__compareSchema = False # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
class CitationUtilsTests(unittest.TestCase): def __init__(self, methodName="runTest"): super(CitationUtilsTests, self).__init__(methodName) self.__verbose = True def setUp(self): # # self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml") # configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath) # self.__cachePath = os.path.join(TOPDIR, "CACHE") # self.__cacheKwargs = {"fmt": "json", "indent": 3} self.__exdbDirPath = os.path.join( self.__cachePath, self.__cfgOb.get("EXDB_CACHE_DIR", sectionName=configName)) # self.__mU = MarshalUtil() self.__entryLimitTest = 20 # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime())) def tearDown(self): unitS = "MB" if platform.system() == "Darwin" else "GB" rusageMax = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss logger.info("Maximum resident memory size %.4f %s", rusageMax / 10**6, unitS) endTime = time.time() logger.info("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime) def testEntryCitationAccess(self): """Test case - extract entry citations""" try: ce = CitationUtils(self.__cfgOb, exdbDirPath=self.__exdbDirPath, useCache=True, cacheKwargs=self.__cacheKwargs, entryLimit=self.__entryLimitTest) eCount = ce.getCitationEntryCount() self.assertGreaterEqual(eCount, self.__entryLimitTest) # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
class RepoHoldingsDataPrepTests(unittest.TestCase): def setUp(self): self.__verbose = True # self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") self.__pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__updateId = "2019_25" # configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=self.__pathConfig, defaultSectionName=configName, mockTopPath=self.__mockTopPath) self.__sandboxPath = self.__cfgOb.getPath("RCSB_EXCHANGE_SANDBOX_PATH", sectionName=configName) # self.__startTime = time.time() logger.info("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime())) def tearDown(self): endTime = time.time() logger.info("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime) def testProcessLegacyFiles(self): """Test loading and processing operations for repository holdings and status echange data.""" try: rhdp = RepoHoldingsDataPrep(cfgOb=self.__cfgOb, sandboxPath=self.__sandboxPath, cachePath=self.__cachePath) rL = rhdp.getHoldingsUpdateEntry(updateId=self.__updateId) logger.info("update data length %r", len(rL)) self.assertGreaterEqual(len(rL), 10) # rL = rhdp.getHoldingsCurrentEntry(updateId=self.__updateId) self.assertGreaterEqual(len(rL), 10) logger.info("holdings data length %r", len(rL)) # rL = rhdp.getHoldingsUnreleasedEntry(updateId=self.__updateId) self.assertGreaterEqual(len(rL), 10) logger.info("unreleased data length %r", len(rL)) # rL = rhdp.getHoldingsRemovedEntry(updateId=self.__updateId) self.assertGreaterEqual(len(rL), 10) logger.info("removed data length %r", len(rL)) rL = rhdp.getHoldingsCombinedEntry(updateId=self.__updateId) self.assertGreaterEqual(len(rL), 10) logger.info("combined data length %r", len(rL)) # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def testEntryInfoRemote(self): minCount = 182000 configName = "site_info_remote_configuration" cfgOb = ConfigUtil(configPath=self.__configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath) eiP = EntryInfoProvider(cachePath=self.__cachePath, useCache=True) ok = eiP.testCache(minCount=minCount) ok = eiP.update(cfgOb, fmt="json", indent=3) self.assertTrue(ok) riD = eiP.getEntryInfo("1kip") logger.info("riD (%d)", len(riD)) self.assertGreaterEqual(len(riD), 1) configName = "site_info_configuration" cfgOb = ConfigUtil(configPath=self.__configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath) ok = eiP.backup(cfgOb, configName, useStash=False, useGit=True) self.assertTrue(ok) ok = eiP.restore(cfgOb, configName, useStash=False, useGit=True) self.assertTrue(ok) eiP = EntryInfoProvider(cachePath=self.__cachePath, useCache=True) ok = eiP.testCache(minCount=minCount)
def setUp(self): # self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml") configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath) # # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): self.__verbose = True # self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") self.__pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") self.__cachePath = os.path.join(TOPDIR, "CACHE") # self.__configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=self.__pathConfig, defaultSectionName=self.__configName, mockTopPath=self.__mockTopPath) self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=True) # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): # # mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=mockTopPath) # self.__cfgOb.dump() self.__resourceName = "MONGO_DB" self.__failedFilePath = os.path.join(HERE, "test-output", "failed-list.txt") self.__readBackCheck = True self.__numProc = 2 self.__chunkSize = 10 self.__documentLimit = 1000 # # sample data set self.__dataSetId = "2018_23" self.__pathClusterData = self.__cfgOb.getPath( "RCSB_SEQUENCE_CLUSTER_DATA_PATH", sectionName=configName) self.__levels = ["100", "95", "90", "70", "50", "30"] # self.__workPath = os.path.join(HERE, "test-output") self.__cachePath = os.path.join(TOPDIR, "CACHE") self.__pathSaveStyleCif = os.path.join(HERE, "test-output", "cluster-data-cif.json") self.__pathSaveStyleDocSequence = os.path.join( HERE, "test-output", "cluster-data-doc-sequence.json") self.__pathSaveStyleDocCluster = os.path.join( HERE, "test-output", "cluster-data-doc-cluster.json") # self.__entitySchemaName = "rcsb_entity_sequence_cluster_list" self.__clusterSchemaName = "rcsb_entity_sequence_cluster_identifer_list" self.__provKeyName = "rcsb_entity_sequence_cluster_prov" # # self.__startTime = time.time() logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self): configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml") configName = "site_info_configuration" self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName) self.__resourceName = "MONGO_DB" self.__startTime = time.time() logger.debug("Starting at %s", time.strftime("%Y %m %d %H:%M:%S", time.localtime()))