def setUp(self):
#
#
mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName,
mockTopPath=mockTopPath)
# self.__cfgOb.dump()
self.__resourceName = "MONGO_DB"
self.__readBackCheck = True
self.__numProc = 2
self.__chunkSize = 10
self.__documentLimit = None
self.__filterType = "assign-dates"
#
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__sandboxPath = self.__cfgOb.getPath("RCSB_EXCHANGE_SANDBOX_PATH",
sectionName=configName)
# sample data set
self.__updateId = "2019_23"
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
self.__verbose = True
#
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
self.__pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__updateId = "2018_25"
self.__export = False
#
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=self.__pathConfig,
defaultSectionName=configName,
mockTopPath=self.__mockTopPath)
self.__schP = SchemaProvider(self.__cfgOb,
self.__cachePath,
useCache=True)
self.__sandboxPath = self.__cfgOb.getPath("RCSB_EXCHANGE_SANDBOX_PATH",
sectionName=configName)
#
self.__mU = MarshalUtil(workPath=self.__cachePath)
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
self.__verbose = True
#
mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
#
configName = "site_info_configuration"
cfgOb = ConfigUtil(configPath=pathConfig,
defaultSectionName=configName,
mockTopPath=mockTopPath)
self.__pathClusterData = cfgOb.getPath(
"RCSB_SEQUENCE_CLUSTER_DATA_PATH", sectionName=configName)
# sample data set
self.__dataSetId = "2018_23"
# self.__levels = ["100", "95", "90", "70", "50", "30"]
self.__levels = ["95"]
#
self.__workPath = os.path.join(HERE, "test-output")
self.__pathSaveStyleCif = os.path.join(HERE, "test-output",
"cluster-data-cif.json")
self.__pathSaveStyleDocSequence = os.path.join(
HERE, "test-output", "cluster-data-doc-sequence.json")
self.__pathSaveStyleDocCluster = os.path.join(
HERE, "test-output", "cluster-data-doc-cluster.json")
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
#
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config",
"dbload-setup-example.yml")
#
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName,
mockTopPath=self.__mockTopPath)
#
self.__cachePath = os.path.join(TOPDIR, "CACHE")
#
self.__cacheKwargs = {"fmt": "json", "indent": 3}
self.__exdbCacheDirPath = os.path.join(
self.__cachePath,
self.__cfgOb.get("EXDB_CACHE_DIR", sectionName=configName))
#
# Reference sequence test data cache -
#
self.__refDbCachePath = os.path.join(HERE, "test-output",
"unp-data-test-cache.json")
self.__cacheKwargs = {"fmt": "json", "indent": 3}
self.__useCache = False
self.__fetchLimit = None
#
# Entity polymer extracted data ...
#
self.__entryLimit = 500
#
self.__mU = MarshalUtil()
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
self.__verbose = True
mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=pathConfig,
defaultSectionName=configName,
mockTopPath=mockTopPath)
self.__docHelper = DocumentDefinitionHelper(cfgOb=self.__cfgOb)
#
self.__pathPdbxDictionaryFile = self.__cfgOb.getPath(
"PDBX_DICT_LOCATOR", sectionName=configName)
self.__pathRcsbDictionaryFile = self.__cfgOb.getPath(
"RCSB_DICT_LOCATOR", sectionName=configName)
self.__pathVrptDictionaryFile = self.__cfgOb.getPath(
"VRPT_DICT_LOCATOR", sectionName=configName)
# self.__mU = MarshalUtil()
#
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__dP = DictionaryApiProviderWrapper(self.__cfgOb,
self.__cachePath,
useCache=True)
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
#
#
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config",
"dbload-setup-example.yml")
#
# Caution: this is very site specific setting !
configName = "site_info_remote"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName,
mockTopPath=self.__mockTopPath)
if configName != "site_info_configuration":
self.__cfgOb.replaceSectionName("site_info_configuration",
configName)
#
self.__workPath = os.path.join(HERE, "test-cache-preserve")
#
self.__entityPolymerCachePath = os.path.join(
self.__workPath, "entity-polymer-data-cache.pic")
self.__entityPolymerCacheKwargs = {"fmt": "pickle"}
self.__useEntityPolymerCache = True
#
self.__refDbCachePath = os.path.join(self.__workPath,
"unp-data-test-cache.json")
self.__refDbCacheKwargs = {"fmt": "json", "indent": 3}
#
self.__refDbUseCache = True
self.__fetchLimit = 500
#
self.__mU = MarshalUtil()
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
#
#
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config",
"dbload-setup-example.yml")
#
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName,
mockTopPath=self.__mockTopPath)
#
self.__cachePath = os.path.join(TOPDIR, "CACHE")
#
self.__cacheKwargs = {"fmt": "json", "indent": 3}
self.__exdbDirPath = os.path.join(
self.__cachePath,
self.__cfgOb.get("EXDB_CACHE_DIR", sectionName=configName))
#
self.__mU = MarshalUtil()
self.__entryLimitTest = 20
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def __init__(self, **kwargs):
# Configuration Details
configPath = kwargs.get("configPath", "exdb-config-example.yml")
self.__configName = kwargs.get("configName", "site_info_configuration")
mockTopPath = kwargs.get("mockTopPath", None)
self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=self.__configName, mockTopPath=mockTopPath)
#
self.__cachePath = kwargs.get("cachePath", ".")
self.__cachePath = os.path.abspath(self.__cachePath)
self.__debugFlag = kwargs.get("debugFlag", False)
if self.__debugFlag:
logger.setLevel(logging.DEBUG)
#
# Rebuild or check resource cache
rebuildCache = kwargs.get("rebuildCache", False)
self.__useCache = not rebuildCache
restoreUseGit = kwargs.get("restoreUseGit", True)
restoreUseStash = kwargs.get("restoreUseStash", True)
providerTypeExclude = kwargs.get("providerTypeExclude", None)
#
self.__cacheStatus = True
rebuildCache = False
if rebuildCache:
self.__cacheStatus = self.buildResourceCache(
rebuildCache=rebuildCache,
providerTypeExclude=providerTypeExclude,
restoreUseGit=restoreUseGit,
restoreUseStash=restoreUseStash,
)
logger.debug("Cache status if %r", self.__cacheStatus)
def setUp(self):
#
#
mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml")
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=mockTopPath)
#
self.__resourceName = "MONGO_DB"
self.__failedFilePath = os.path.join(HERE, "test-output", "failed-list.txt")
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__readBackCheck = True
self.__numProc = 2
self.__chunkSize = 10
self.__fileLimit = None
self.__documentStyle = "rowwise_by_name_with_cardinality"
self.__ldList = [
# {"databaseName": "chem_comp_core", "collectionNameList": None, "loadType": "full", "mergeContentTypes": None, "validationLevel": "min"},
{
"databaseName": "bird_chem_comp_core",
"collectionNameList": None,
"loadType": "full",
"mergeContentTypes": None,
"validationLevel": "full",
"updateSchemaOnReplace": False,
"status": True,
},
{
"databaseName": "bird_chem_comp_core",
"collectionNameList": None,
"loadType": "replace",
"mergeContentTypes": None,
"validationLevel": "full",
"updateSchemaOnReplace": True,
"status": True,
},
{
"databaseName": "pdbx_core",
"collectionNameList": None,
"loadType": "full",
"mergeContentTypes": ["vrpt"],
"validationLevel": "min",
"updateSchemaOnReplace": False,
"status": True,
},
{
"databaseName": "pdbx_core",
"collectionNameList": None,
"loadType": "replace",
"mergeContentTypes": ["vrpt"],
"validationLevel": "full",
"updateSchemaOnReplace": True,
"status": True,
},
]
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
class DictionaryProviderTests(unittest.TestCase):
def setUp(self):
mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__dirPath = os.path.join(self.__cachePath, "dictionaries")
configPath = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
configName = "site_info_configuration"
self.__configName = configName
self.__contentInfoConfigName = "content_info_helper_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName,
mockTopPath=mockTopPath)
dictLocatorMap = self.__cfgOb.get(
"DICT_LOCATOR_CONFIG_MAP",
sectionName=self.__contentInfoConfigName)
schemaName = "pdbx_core"
self.__dictLocators = [
self.__cfgOb.getPath(configLocator, sectionName=self.__configName)
for configLocator in dictLocatorMap[schemaName]
]
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def tearDown(self):
endTime = time.time()
logger.debug("Completed %s at %s (%.4f seconds)", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()),
endTime - self.__startTime)
def testResourceCache(self):
"""Test case - generate and check dictonary artifact and api caches
"""
try:
logger.debug("Dictionary locators %r", self.__dictLocators)
dp = DictionaryApiProvider(dirPath=self.__dirPath, useCache=False)
dApi = dp.getApi(self.__dictLocators)
ok = dApi.testCache()
self.assertTrue(ok)
title = dApi.getDictionaryTitle()
logger.debug("Title %r", title)
self.assertEqual(
title, "mmcif_pdbx.dic,rcsb_mmcif_ext.dic,vrpt_mmcif_ext.dic")
# revL = dApi.getDictionaryHistory()
numRev = dApi.getDictionaryRevisionCount()
logger.debug("Number of dictionary revisions (numRev) %r", numRev)
self.assertGreater(numRev, 220)
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def setUp(self):
self.__numProc = 2
# self.__fileLimit = 200
self.__fileLimit = None
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example-ihm.yml")
configName = "site_info_configuration"
self.__configName = configName
self.__cfgOb = ConfigUtil(configPath=self.__configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath)
self.__mU = MarshalUtil(workPath=self.__cachePath)
#self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=False, rebuildFlag=True)
self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=True)
self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb, numProc=self.__numProc, fileLimit=self.__fileLimit, cachePath=self.__cachePath)
#
self.__birdRepoPath = self.__cfgOb.getPath("BIRD_REPO_PATH", sectionName=configName)
#
self.__fTypeRow = "drop-empty-attributes|drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs"
self.__fTypeCol = "drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs"
self.__verbose = True
#
self.__modulePathMap = self.__cfgOb.get("DICT_METHOD_HELPER_MODULE_PATH_MAP", sectionName=configName)
self.__testDirPath = os.path.join(HERE, "test-output", "pdbx-files")
self.__testIhmDirPath = os.path.join(HERE, "test-output", "ihm-files")
self.__export = True
#
#self.__extraOpts = None
# The following for extended parent/child info -
self.__extraOpts = 'addParentRefs|addPrimaryKey'
#
self.__alldatabaseNameD = {
"ihm_dev": ["ihm_dev"],
"pdbx": ["pdbx", "pdbx_ext"],
"pdbx_core": ["pdbx_core_entity", "pdbx_core_entry", "pdbx_core_assembly", "pdbx_core_entity_instance", "pdbx_core_entity_instance_validation"],
"bird": ["bird"],
"bird_family": ["family"],
"chem_comp": ["chem_comp"],
"bird_chem_comp": ["bird_chem_comp"],
"bird_chem_comp_core": ["bird_chem_comp_core"],
}
self.__databaseNameD = {
"pdbx_core": ["pdbx_core_entity", "pdbx_core_entry", "pdbx_core_assembly", "pdbx_core_entity_instance", "pdbx_core_entity_instance_validation"],
"bird_chem_comp_core": ["bird_chem_comp_core"],
}
self.__mergeContentTypeD = {"pdbx_core": ["vrpt"]}
# self.__databaseNameD = {"chem_comp_core": ["chem_comp_core"], "bird_chem_comp_core": ["bird_chem_comp_core"]}
# self.__databaseNameD = {"ihm_dev_full": ["ihm_dev_full"]}
# self.__databaseNameD = {"pdbx_core": ["pdbx_core_entity_instance_validation"]}
# self.__databaseNameD = {"pdbx_core": ["pdbx_core_entity_monomer"]}
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def testWriteFromIniConfig(self):
try:
cfgOb = ConfigUtil(configPath=self.__inpPathConfigIni,
mockTopPath=self.__dataPath)
ok = cfgOb.writeConfig(self.__outPathConfigIni, configFormat="ini")
ok = cfgOb.writeConfig(self.__outPathConfigYaml,
configFormat="yaml")
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
return {}
def setUp(self):
configPath = os.path.join(HERE, "test-data", "drugbank-config-example.yml")
configName = "site_info_configuration"
cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName)
self.__user = cfgOb.get("_DRUGBANK_AUTH_USERNAME", sectionName=configName)
self.__pw = cfgOb.get("_DRUGBANK_AUTH_PASSWORD", sectionName=configName)
self.__cachePath = os.path.join(HERE, "test-output", "CACHE")
self.__fastaPath = os.path.join(HERE, "test-output", "drugbank-targets.fa")
self.__taxonPath = os.path.join(HERE, "test-output", "drugbank-targets-taxon.tdd")
#
self.__seqMatchResultsPath = os.path.join(HERE, "test-data", "drugbank-vs-pdbprent-filtered-results.json.gz")
self.__startTime = time.time()
logger.info("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def __init__(self, **kwargs):
# Configuration Details
configPath = kwargs.get("configPath", "exdb-config-example.yml")
self.__configName = kwargs.get("configName", "site_info_configuration")
mockTopPath = kwargs.get("mockTopPath", None)
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=self.__configName,
mockTopPath=mockTopPath)
#
self.__cachePath = kwargs.get("cachePath", ".")
self.__cachePath = os.path.abspath(self.__cachePath)
self.__debugFlag = kwargs.get("debugFlag", False)
if self.__debugFlag:
logger.setLevel(logging.DEBUG)
def setUp(self):
configPath = os.path.join(HERE, "test-data",
"pharos-config-example.yml")
self.__configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=self.__configName)
self.__user = self.__cfgOb.get("_MYSQL_DB_USER_NAME",
sectionName=self.__configName)
self.__pw = self.__cfgOb.get("_MYSQL_DB_PASSWORD",
sectionName=self.__configName)
self.__cachePath = os.path.join(HERE, "test-output", "CACHE")
self.__dirPath = os.path.join(self.__cachePath, "Pharos-targets")
self.__dataPath = os.path.join(HERE, "test-data")
#
self.__pharosFixture()
def testReadYamlConfigWithAutoAppend(self):
try:
cfgOb = ConfigUtil(
configPath=self.__inpPathConfigAutoYaml,
configFormat="yaml",
mockTopPath=self.__mockTopPath,
defaultSectionName="site_info_1",
cachePath=None,
useCache=False,
)
ok = cfgOb.appendConfig(self.__inpPathConfigAppendYaml,
configFormat="yaml")
self.assertTrue(ok)
#
for sName in [
"section_appended_1", "section_appended_2", "Section1",
"Section2"
]:
pathBird = cfgOb.getPath("BIRD_REPO_PATH", sectionName=sName)
pathPdbx = cfgOb.getPath("PDBX_REPO_PATH", sectionName=sName)
#
self.assertEqual(
pathBird, os.path.join(self.__mockTopPath,
"MOCK_BIRD_REPO"))
self.assertEqual(
pathPdbx,
os.path.join(self.__mockTopPath, "MOCK_PDBX_SANDBOX"))
#
#
cfgOb = ConfigUtil(
configPath=self.__inpPathConfigAutoYaml,
configFormat="yaml",
mockTopPath=self.__mockTopPath,
defaultSectionName="site_info_1",
cachePath=None,
useCache=True,
)
ok = cfgOb.appendConfig(self.__inpPathConfigAppendYaml,
configFormat="yaml")
self.assertTrue(ok)
#
for sName in [
"section_appended_1", "section_appended_2", "Section1",
"Section2"
]:
pathBird = cfgOb.getPath("BIRD_REPO_PATH", sectionName=sName)
pathPdbx = cfgOb.getPath("PDBX_REPO_PATH", sectionName=sName)
#
self.assertEqual(
pathBird, os.path.join(self.__mockTopPath,
"MOCK_BIRD_REPO"))
self.assertEqual(
pathPdbx,
os.path.join(self.__mockTopPath, "MOCK_PDBX_SANDBOX"))
#
except Exception as e:
logger.error("Failing with %s", str(e))
self.fail()
def setUp(self):
self.__verbose = True
self.__numProc = 2
self.__fileLimit = 100
self.__chunkSize = 0
self.__workPath = os.path.join(HERE, "test-output")
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName)
self.__resourceName = "CRATE_DB"
self.__schP = SchemaProvider(self.__cfgOb,
self.__workPath,
useCache=True)
self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb,
numProc=self.__numProc,
fileLimit=self.__fileLimit,
cachePath=self.__workPath)
#
#
self.__tableIdSkipD = {
"ATOM_SITE": True,
"ATOM_SITE_ANISOTROP": True,
"__LOAD_STATUS__": True
}
self.__ioObj = IoAdapter(verbose=self.__verbose)
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
self.__isMac = platform.system() == "Darwin"
self.__excludeType = None if self.__isMac else "optional"
self.__verbose = True
#
fileLimit = 100
numProc = 2
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__workPath = os.path.join(HERE, "test-output")
mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
#
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName,
mockTopPath=mockTopPath)
self.__resourceName = "MYSQL_DB"
#
self.__schP = SchemaProvider(self.__cfgOb,
self.__cachePath,
useCache=True)
self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb,
numProc=numProc,
fileLimit=fileLimit,
cachePath=self.__cachePath)
#
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
self.__cachePath = os.path.join(HERE, "test-output", "CACHE")
#
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
self.__configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml")
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=self.__configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath)
def setUp(self):
self.__dbName = "test_database"
self.__collectionName = "test_collection"
#
configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml")
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName)
self.__resourceName = "MONGO_DB"
self.__connectD = self.__assignResource(self.__cfgOb, resourceName=self.__resourceName, sectionName=configName)
# self.__cObj = self.__open(self.__connectD)
#
self.__mongoSchema = {
"bsonType": "object",
"required": ["strField1", "intField1", "enumField1", "dblField1"],
"properties": {
"strField1": {"bsonType": "string", "description": "must be a string and is required"},
"strField2": {"bsonType": "string", "description": "must be a string and is not required"},
"strField3": {"bsonType": "string", "description": "must be a string and is not required"},
"intField1": {"bsonType": "int", "minimum": 1, "maximum": 100, "exclusiveMaximum": False, "description": "must be an integer in [ 1, 100 ] and is required"},
"enumField1": {"enum": ["v1", "v2", "v3", "v4", None], "description": "can only be one of the enum values and is required"},
"dblField1": {"bsonType": ["double"], "minimum": 0, "description": "must be a double and is required"},
"dateField1": {"bsonType": "date", "description": "must be a date and is not required"},
},
}
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
self.__isMac = platform.system() == "Darwin"
self.__doLoad = True if self.__isMac else False
#
#
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config",
"dbload-setup-example.yml")
#
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName,
mockTopPath=self.__mockTopPath)
#
self.__cachePath = os.path.join(TOPDIR, "CACHE")
#
self.__mU = MarshalUtil()
#
self.__readBackCheck = True
self.__numProc = 2
self.__chunkSize = 10
self.__documentLimit = None
self.__debugFlag = False
self.__loadType = "full"
self.__useCache = True
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def testRoundTripYaml(self):
try:
cfgOb = ConfigUtil(configPath=self.__inpPathConfigYaml,
configFormat="yaml",
mockTopPath=self.__mockTopPath,
roundTrip=True)
ok = cfgOb.writeConfig(self.__outPathConfigIni, configFormat="ini")
self.assertTrue(ok)
ok = cfgOb.writeConfig(self.__outPathConfigYaml,
configFormat="yaml")
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
return {}
def setUp(self):
self.__verbose = True
mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
configName = "site_info_configuration"
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__cfgOb = ConfigUtil(configPath=pathConfig,
defaultSectionName=configName,
mockTopPath=mockTopPath)
self.__schP = SchemaProvider(self.__cfgOb,
self.__cachePath,
useCache=False)
#
self.__validationLevels = self.__cfgOb.getList(
"VALIDATION_LEVELS_TEST",
sectionName="database_catalog_configuration")
self.__encodingTypes = self.__cfgOb.getList(
"ENCODING_TYPES_TEST",
sectionName="database_catalog_configuration")
#
buildAll = True
if buildAll:
self.__databaseNameList = self.__cfgOb.getList(
"DATABASE_NAMES_DEPLOYED",
sectionName="database_catalog_configuration")
self.__dataTypingList = self.__cfgOb.getList(
"DATATYPING_DEPLOYED",
sectionName="database_catalog_configuration")
#
else:
self.__databaseNameList = self.__cfgOb.getList(
"DATABASE_NAMES_TEST",
sectionName="database_catalog_configuration")
# self.__databaseNameList = ["repository_holdings"]
self.__dataTypingList = self.__cfgOb.getList(
"DATATYPING_TEST",
sectionName="database_catalog_configuration")
# self.__databaseNameList = ["sequence_clusters"]
self.__saveSchema = True
self.__compareDefSchema = False
self.__compareSchema = False
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
class CitationUtilsTests(unittest.TestCase):
def __init__(self, methodName="runTest"):
super(CitationUtilsTests, self).__init__(methodName)
self.__verbose = True
def setUp(self):
#
#
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config",
"dbload-setup-example.yml")
#
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName,
mockTopPath=self.__mockTopPath)
#
self.__cachePath = os.path.join(TOPDIR, "CACHE")
#
self.__cacheKwargs = {"fmt": "json", "indent": 3}
self.__exdbDirPath = os.path.join(
self.__cachePath,
self.__cfgOb.get("EXDB_CACHE_DIR", sectionName=configName))
#
self.__mU = MarshalUtil()
self.__entryLimitTest = 20
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def tearDown(self):
unitS = "MB" if platform.system() == "Darwin" else "GB"
rusageMax = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
logger.info("Maximum resident memory size %.4f %s", rusageMax / 10**6,
unitS)
endTime = time.time()
logger.info("Completed %s at %s (%.4f seconds)", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()),
endTime - self.__startTime)
def testEntryCitationAccess(self):
"""Test case - extract entry citations"""
try:
ce = CitationUtils(self.__cfgOb,
exdbDirPath=self.__exdbDirPath,
useCache=True,
cacheKwargs=self.__cacheKwargs,
entryLimit=self.__entryLimitTest)
eCount = ce.getCitationEntryCount()
self.assertGreaterEqual(eCount, self.__entryLimitTest)
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
class RepoHoldingsDataPrepTests(unittest.TestCase):
def setUp(self):
self.__verbose = True
#
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
self.__pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__updateId = "2019_25"
#
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=self.__pathConfig,
defaultSectionName=configName,
mockTopPath=self.__mockTopPath)
self.__sandboxPath = self.__cfgOb.getPath("RCSB_EXCHANGE_SANDBOX_PATH",
sectionName=configName)
#
self.__startTime = time.time()
logger.info("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def tearDown(self):
endTime = time.time()
logger.info("Completed %s at %s (%.4f seconds)", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()),
endTime - self.__startTime)
def testProcessLegacyFiles(self):
"""Test loading and processing operations for repository holdings and status echange data."""
try:
rhdp = RepoHoldingsDataPrep(cfgOb=self.__cfgOb,
sandboxPath=self.__sandboxPath,
cachePath=self.__cachePath)
rL = rhdp.getHoldingsUpdateEntry(updateId=self.__updateId)
logger.info("update data length %r", len(rL))
self.assertGreaterEqual(len(rL), 10)
#
rL = rhdp.getHoldingsCurrentEntry(updateId=self.__updateId)
self.assertGreaterEqual(len(rL), 10)
logger.info("holdings data length %r", len(rL))
#
rL = rhdp.getHoldingsUnreleasedEntry(updateId=self.__updateId)
self.assertGreaterEqual(len(rL), 10)
logger.info("unreleased data length %r", len(rL))
#
rL = rhdp.getHoldingsRemovedEntry(updateId=self.__updateId)
self.assertGreaterEqual(len(rL), 10)
logger.info("removed data length %r", len(rL))
rL = rhdp.getHoldingsCombinedEntry(updateId=self.__updateId)
self.assertGreaterEqual(len(rL), 10)
logger.info("combined data length %r", len(rL))
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testEntryInfoRemote(self):
minCount = 182000
configName = "site_info_remote_configuration"
cfgOb = ConfigUtil(configPath=self.__configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath)
eiP = EntryInfoProvider(cachePath=self.__cachePath, useCache=True)
ok = eiP.testCache(minCount=minCount)
ok = eiP.update(cfgOb, fmt="json", indent=3)
self.assertTrue(ok)
riD = eiP.getEntryInfo("1kip")
logger.info("riD (%d)", len(riD))
self.assertGreaterEqual(len(riD), 1)
configName = "site_info_configuration"
cfgOb = ConfigUtil(configPath=self.__configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath)
ok = eiP.backup(cfgOb, configName, useStash=False, useGit=True)
self.assertTrue(ok)
ok = eiP.restore(cfgOb, configName, useStash=False, useGit=True)
self.assertTrue(ok)
eiP = EntryInfoProvider(cachePath=self.__cachePath, useCache=True)
ok = eiP.testCache(minCount=minCount)
def setUp(self):
#
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml")
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath)
#
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
self.__verbose = True
#
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
self.__pathConfig = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
self.__cachePath = os.path.join(TOPDIR, "CACHE")
#
self.__configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=self.__pathConfig,
defaultSectionName=self.__configName,
mockTopPath=self.__mockTopPath)
self.__schP = SchemaProvider(self.__cfgOb,
self.__cachePath,
useCache=True)
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
#
#
mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName,
mockTopPath=mockTopPath)
# self.__cfgOb.dump()
self.__resourceName = "MONGO_DB"
self.__failedFilePath = os.path.join(HERE, "test-output",
"failed-list.txt")
self.__readBackCheck = True
self.__numProc = 2
self.__chunkSize = 10
self.__documentLimit = 1000
#
# sample data set
self.__dataSetId = "2018_23"
self.__pathClusterData = self.__cfgOb.getPath(
"RCSB_SEQUENCE_CLUSTER_DATA_PATH", sectionName=configName)
self.__levels = ["100", "95", "90", "70", "50", "30"]
#
self.__workPath = os.path.join(HERE, "test-output")
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__pathSaveStyleCif = os.path.join(HERE, "test-output",
"cluster-data-cif.json")
self.__pathSaveStyleDocSequence = os.path.join(
HERE, "test-output", "cluster-data-doc-sequence.json")
self.__pathSaveStyleDocCluster = os.path.join(
HERE, "test-output", "cluster-data-doc-cluster.json")
#
self.__entitySchemaName = "rcsb_entity_sequence_cluster_list"
self.__clusterSchemaName = "rcsb_entity_sequence_cluster_identifer_list"
self.__provKeyName = "rcsb_entity_sequence_cluster_prov"
#
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def setUp(self):
configPath = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName)
self.__resourceName = "MONGO_DB"
self.__startTime = time.time()
logger.debug("Starting at %s",
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
