def __setup(self, defSiteId=None, sessionId=None):
"""Simulate the web application environment for managing session storage of temporaty data files."""
self.__siteId = getSiteId(defaultSiteId=defSiteId)
#
self.__cI = ConfigInfo(self.__siteId)
self.__topPath = self.__cI.get("SITE_WEB_APPS_TOP_PATH")
self.__topSessionPath = self.__cI.get(
"SITE_WEB_APPS_TOP_SESSIONS_PATH")
#
self.__reqObj = InputRequest({},
verbose=self.__verbose,
log=self.__lfh)
self.__reqObj.setValue("TopSessionPath", self.__topSessionPath)
self.__reqObj.setValue("TopPath", self.__topPath)
self.__reqObj.setValue("WWPDB_SITE_ID", self.__siteId)
#
self.__reqObj.setValue("SITE_DA_INTERNAL_DB_USER",
os.environ["SITE_DA_INTERNAL_DB_USER"])
self.__reqObj.setValue("SITE_DA_INTERNAL_DB_PASSWORD",
os.environ["SITE_DA_INTERNAL_DB_PASSWORD"])
os.environ["WWPDB_SITE_ID"] = self.__siteId
if sessionId is not None:
self.__reqObj.setValue("sessionid", sessionId)
# retained due to side effects
_sessionObj = self.__reqObj.newSessionObj() # noqa: F841
self.__reqObj.printIt(ofh=self.__lfh)
def get_config(var):
ci = ConfigInfo()
valsect = ci.get("VALIDATION_SERVICES")
if valsect is not None and var in valsect:
return valsect[var]
else:
return ci.get(var)
def __init__(self, siteId=None, tokenPrefix=None, **kwargs):
"""
Base class supporting application token management in persistent store.
kwargs allows for overriding ConfigInfo for testing
"""
#
self._cI = ConfigInfo(siteId)
#
if tokenPrefix is not None:
fn = tokenPrefix + "_TOKEN_STORE.pic"
else:
fn = "ANONYMOUSWS_TOKEN_STORE.pic"
#
self.__filePath = kwargs.get("site_service_registration_dir_path")
if not self.__filePath:
self.__filePath = os.path.join(
self._cI.get("SITE_SERVICE_REGISTRATION_DIR_PATH"), fn)
#
logger.debug("Assigning token store file path %r", self.__filePath)
self.__lockDirPath = kwargs.get(
"site_service_registration_lockdir_path")
if not self.__lockDirPath:
self.__lockDirPath = self._cI.get(
"SITE_SERVICE_REGISTRATION_LOCKDIR_PATH", ".")
#
self.__tokenD = {}
self.__emailD = {}
self.__tokenPrefix = tokenPrefix if tokenPrefix else "WS"
self.__pickleProtocol = 0
#
lockutils.set_defaults(self.__lockDirPath)
#
self.deserialize()
def __init__(self, config_file, noop):
self.__configfile = config_file
self.__noop = noop
self.__ci = ConfigInfo()
self.__ci_common = ConfigInfoAppCommon()
self.__extraconf = self.get_variable("ADMIN_EXTRA_CONF",
environment='INSTALL_ENVIRONMENT')
self.__confvars = {}
self.__extraconfdir = None
if self.__extraconf is not None:
self.__extraconfdir = os.path.abspath(
os.path.dirname(self.__extraconf))
self.__confvars["extraconfdir"] = self.__extraconfdir
# Infer topdir from where running from
topdir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
cdict = {'topdir': topdir}
self.__cparser = ConfigParser(cdict)
cfiles = self.__configfile
if self.__extraconf is not None:
cfiles = [self.__configfile, self.__extraconf]
self.__cparser.read(cfiles)
self.web_apps_path = self.get_variable('TOP_WWPDB_WEBAPPS_DIR')
self.resources_ro_path = self.get_variable('RO_RESOURCE_PATH')
def __init__(self):
self.__site_id = getSiteId()
self.__cI = ConfigInfo(siteId=self.__site_id)
self.__packagePath = self.__cI.get('SITE_PACKAGES_PATH')
self.node_path = os.path.join(self.__packagePath, 'node', 'bin',
'node')
self.volume_server_pack = self.__cI.get('VOLUME_SERVER_PACK')
self.volume_server_query = self.__cI.get('VOLUME_SERVER_QUERY')
def __init__(self, verbose=False, log=sys.stderr):
self.__verbose = verbose
self.__lfh = log
self.__debug = True
self.__cI = ConfigInfo(siteId=None, verbose=self.__verbose)
self.__cICommon = ConfigInfoAppDepUI()
self.__serviceD = self.__cI.get("PROJECT_DEPOSIT_SERVICE_DICTIONARY")
self.__siteAccessD = None
def __setup(self, siteId=None):
self.__siteId = siteId
self.__cI = ConfigInfo(self.__siteId)
self.__sessionPath = None
self.__pI = PathInfo(siteId=self.__siteId,
sessionPath=self.__sessionPath,
verbose=self.__verbose,
log=self.__lfh)
def setUp(self):
self.__lfh = sys.stderr
self.__verbose = True
# Get siteId from the environment -
self.__siteId = getSiteId()
self.__cI = ConfigInfo(siteId=self.__siteId)
# In test mode, no deletions are performed -
self.__testMode = True
#
# An data set ID list must be set --
self.__idListPath = "RELEASED.LIST"
#
self.__milestoneL = []
self.__milestoneL.append(None)
self.__milestoneL.extend(self.__cI.get("CONTENT_MILESTONE_LIST"))
self.__cTBD = self.__cI.get("CONTENT_TYPE_BASE_DICTIONARY")
self.__cTD = self.__cI.get("CONTENT_TYPE_DICTIONARY")
self.__cTL = sorted(self.__cTBD.keys())
# Example list of candidate content types for purging -- this is based on system V15x for X-ray content types
self.__cTypesOtherL = [
"assembly-assign",
"assembly-model",
"assembly-model-xyz",
"assembly-report",
"chem-comp-assign",
"chem-comp-assign-details",
"chem-comp-assign-final",
"chem-comp-depositor-info",
"chem-comp-link",
"component-image",
"dcc-report",
"dict-check-report",
"dict-check-report-r4",
"format-check-report",
"geometry-check-report",
"merge-xyz-report",
"misc-check-report",
"notes-from-annotator",
"polymer-linkage-distances",
"polymer-linkage-report",
"secondary-structure-topology",
"seq-align-data",
"seq-assign",
"seq-data-stats",
"seqdb-match",
"sequence-fasta",
"sf-convert-report",
"site-assign",
"special-position-report",
"validation-data",
"validation-report",
"validation-report-depositor",
"validation-report-full",
"validation-report-slider",
]
#
# Test snapshot directory required for recovery tests -
self.__snapShotPath = "/net/wwpdb_da_data_archive/.snapshot/nightly.1/data"
def __init__(self, siteId=None, verbose=True, log=sys.stderr):
self._cI = ConfigInfo(siteId=siteId, verbose=verbose, log=log)
self._resourcedir = None
self._rwresourcedir = None
self._referencedir = None
self._site_archive_dir = None
self._site_local_apps_path = None
self._top_webapps_path = None
self._top_sessions_path = None
def __init__(self, siteId=None):
"""
Limited set of token methods required to read and validate a JWT tokens.
"""
self._cI = ConfigInfo(siteId)
serviceKey = self._cI.get("SITE_SERVICE_REGISTRATION_KEY",
default=None)
self.__serviceKey = serviceKey if serviceKey else "secretvalue"
self.__tokenErrorCode = 401
def __init__(self, siteId=None):
self.__siteId = siteId
self.__cI = ConfigInfo(siteId=self.__siteId)
self.ftp_pdb_root = self.__cI.get("SITE_PDB_FTP_ROOT_DIR")
self.ftp_emdb_root = self.__cI.get("SITE_EMDB_FTP_ROOT_DIR")
self.__mapping = {
"model": "mmCIF",
"structure_factors": "structure_factors",
"chemical_shifts": "nmr_chemical_shifts",
"nmr_data": "nmr_data",
}
def __init__(self, config_file, noop, build_version='v-5200'):
self.__configfile = config_file
self.__noop = noop
self.__build_version = build_version
# Infer topdir from where running from
topdir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
cdict = {'topdir': topdir}
self.__cparser = ConfigParser(cdict)
self.__cparser.read(self.__configfile)
self.__ci = ConfigInfo()
pass
def __init__(self, siteId=None, sessionPath=".", verbose=False, log=None):
""""""
self.__verbose = verbose
self.__lfh = log
#
self.__debug = False # pylint: disable=unused-private-member
self.__siteId = siteId if siteId is not None else getSiteId(defaultSiteId=siteId)
self.__sessionPath = sessionPath
self.__sessionDownloadPath = None
if self.__sessionPath is not None:
self.__sessionDownloadPath = os.path.join(self.__sessionPath, "downloads")
#
self.__cI = ConfigInfo(siteId=self.__siteId, verbose=self.__verbose, log=self.__lfh)
def __init__(self, schemaDefObj, verbose=False, log=sys.stderr):
self.__verbose = verbose
self.__lfh = log
self.__debug = False
#
self.__siteId = getSiteId(defaultSiteId="WWPDB_DEPLOY_MACOSX")
self.__cI = ConfigInfo(self.__siteId)
#
self.__sd = schemaDefObj
self.__databaseName = self.__sd.getDatabaseName()
self.__dbCon = None
self.__defaultD = {}
self.__attributeParameterMap = {}
self.__attributeConstraintParameterMap = {}
def __setup(self, siteId=None):
if siteId is not None:
self.__siteId = siteId
else:
self.__siteId = self.__reqObj.getValue("WWPDB_SITE_ID")
#
self.__lfh.write("+FileUtils.__setup() starting with entryId %r adjusted WWPDB_SITE_ID %r\n" % (self.__entryId, self.__siteId))
#
self.__sObj = self.__reqObj.getSessionObj()
self.__sessionId = self.__sObj.getId()
self.__sessionPath = self.__sObj.getPath()
self.__pI = PathInfo(siteId=self.__siteId, sessionPath=self.__sessionPath, verbose=self.__verbose, log=self.__lfh)
self.__cI = ConfigInfo(self.__siteId)
self.__msL = self.__cI.get("CONTENT_MILESTONE_LIST")
def __init__(self, siteId=None, verbose=False, log=sys.stderr): # pylint: disable=unused-argument
#
self.__siteId = siteId
self._cI = ConfigInfo(self.__siteId)
self._dbCon = None
self.__authD = {}
self.__databaseName = None
self.__dbHost = None
self.__dbUser = None
self.__dbPw = None
self.__dbSocket = None
self.__dbPort = None
self.__dbPort = 3306
self.__dbServer = "mysql"
def testGetSiteLocation(self):
"""Test case - return site location"""
try:
for siteId in self.__siteIdList:
ci = ConfigInfo(siteId=siteId,
verbose=self.__verbose,
log=self.__lfh)
siteName = ci.get("SITE_NAME", default=None)
siteLoc = ci.get("WWPDB_SITE_LOC", default=None)
logger.info(" siteId %-30s siteName %s siteLoc %s", siteId,
siteName, siteLoc)
except Exception as e: # pragma: no cover
logger.exception("Unable to get group site location %s", str(e))
self.fail()
def __init__(self, *args, **kwargs): # pylint: disable=unused-argument
self._raiseExceptions = kwargs.get("raiseExceptions", False)
self._siteId = kwargs.get("siteId", getSiteId())
self._serverId = kwargs.get("serverId", None)
self.__cI = ConfigInfo(siteId=getSiteId())
#
cD = self.__cI.get(self._serverId, {})
self._hostName = cD.get("HOST_NAME", None)
self._userName = cD.get("HOST_USERNAME", None)
self._password = cD.get("HOST_PASSWORD", None)
self._hostPort = int(cD.get("HOST_PORT", 22))
self._protocol = cD.get("HOST_PROTOCOL", None)
self._rootPath = cD.get("HOST_ROOT_PATH", None)
self._keyFilePath = cD.get("HOST_KEY_FILE_PATH", None)
self._keyFileType = cD.get("HOST_KEY_FILE_TYPE", None)
def __init__(self, reqObj=None, verbose=False, log=sys.stderr):
"""
Base class supporting web application worker methods.
Performs URL -> application mapping for this module.
"""
self._verbose = verbose
self.__debug = False
self._lfh = log
self._reqObj = reqObj
self._sObj = None
self._sessionId = None
self._sessionPath = None
self._rltvSessionPath = None
#
self._siteId = self._reqObj.getValue("WWPDB_SITE_ID")
self._cI = ConfigInfo(self._siteId)
#
self._uds = None
# UtilDataStore prefix for general session data -- used by _getSession()
self._udsPrefix = None
#
self.__appPathD = {}
def __init__(self, verbose=False, log=sys.stderr): # pylint: disable=unused-argument
self.__verbose = verbose
self.__debug = True
self.__cI = ConfigInfo(siteId=None, verbose=self.__verbose)
self.__cIDepUI = ConfigInfoAppDepUI(siteId=getSiteId())
# Default data set id range assignments
self.__depIdAssignments = self.__cI.get(
"SITE_DATASET_ID_ASSIGNMENT_DICTIONARY")
self.__depTestIdAssignments = self.__cI.get(
"SITE_DATASET_TEST_ID_ASSIGNMENT_DICTIONARY")
self.__siteBackupD = self.__cI.get("SITE_BACKUP_DICT", default={})
self.__dsLocD = None
#
self.__lockDirPath = self.__cI.get(
"SITE_SERVICE_REGISTRATION_LOCKDIR_PATH", "/tmp")
lockutils.set_defaults(self.__lockDirPath)
def __init__(self, log=sys.stderr, verbose=False):
""" """
self.__lfh = log
self.__verbose = verbose
self.__debug = True
self.__siteId = getSiteId()
cI = ConfigInfo()
cIcommon = ConfigInfoAppCommon(self.__siteId)
self.__schemaPath = cIcommon.get_site_da_internal_schema_path()
self.__dbHost = cI.get("SITE_DB_HOST_NAME")
self.__dbUser = cI.get("SITE_DB_USER_NAME")
self.__dbPw = cI.get("SITE_DB_PASSWORD")
self.__dbPort = str(cI.get("SITE_DB_PORT_NUMBER"))
self.__dbSocket = cI.get("SITE_DB_SOCKET")
self.__archivePath = cI.get("SITE_ARCHIVE_STORAGE_PATH")
self.__dbName = "da_internal"
self.__workPath = os.path.join(self.__archivePath, "archive")
self.__mysql = "mysql "
self.__dbLoader = cIcommon.get_db_loader_path()
self.__mapping = self.__schemaPath
self.__workDir = "dbdata"
self._dbCon = None
def setUp(self):
self.__siteId = getSiteId(defaultSiteId=None)
logger.info("\nTesting with site environment for: %s\n" % self.__siteId)
#
self.__cI = ConfigInfo(self.__siteId)
#
self.__tmpPath = TESTOUTPUT
self.__testFilePath = os.path.join(TOPDIR, 'wwpdb', 'mock-data', 'dp-utils')
self.__testFileCif = '1xbb.cif'
self.__testFileAnnotSS = '4deq.cif'
self.__testFileAnnotSSTop = 'topology.txt'
#
self.__testFileAnnotLink = '3rij.cif'
self.__testFileAnnotCisPeptide = '5hoh.cif'
self.__testFileAnnotSolvent = '4ec0.cif'
self.__testFileAnnotValidate = '3rij.cif'
self.__testFileAnnotNA = '1o3q.cif'
self.__testFileAnnotSite = '1xbb.cif'
self.__testIdAnnotSite = '1xbb'
#
self.__testFileAnnotSiteAlt = '4p00.cif'
self.__testIdAnnotSiteAlt = '4P00'
## OK JDW
self.__testFileAnnotRcsb = '3of4.cif'
#
self.__testFilePdbPisa = '1xbb.pdb'
self.__testFileCifPisa = '1xbb.cif'
#
self.__testFileStarCs = "2MMZ-cs.str"
self.__testFileCsRelatedCif = "cor_16703_test.cif"
#
self.__testFileValidateXyz = "1cbs.cif"
self.__testFileValidateSf = "1cbs-sf.cif"
self.__testValidateIdList = ["1cbs", "3of4", "3oqp"]
#
self.__testFileMtzBad = "mtz-bad.mtz"
self.__testFileMtzGood = "mtz-good.mtz"
self.__testFileMtzRunaway = "bad-runaway.mtz"
self.__testFileXyzRunaway = "bad-runaway.cif"
self.__testMapNormal = "normal.map"
self.__testMapSpider = "testmap.spi"
## OK JDW
self.__testFilePrdSearch = 'D_1200000237_model_P1.cif.V1'
self.__testValidateXrayIdList = ['1cbs']
self.__testValidateNmrIdList = ['2MM4', '2MMZ']
self.__testDccModelId = '4wpo'
self.__testSpecialPosition = '5uee.cif'
self.__testDepAssembly = "1cbs.cif"
class ConfigInfoGroupDataSet(object):
"""
Provides accessors for the correspondence between group deposition data identifiers and
deposition and annotation sites (e.g. wwpdb_site_id).
"""
def __init__(self, verbose=False, log=sys.stderr): # pylint: disable=unused-argument
self.__verbose = verbose
self.__debug = True
self.__cI = ConfigInfo(siteId=None, verbose=self.__verbose)
self.__groupIdAssignments = self.__cI.get(
"SITE_GROUP_DATASET_ID_ASSIGNMENT_DICTIONARY")
def getDefaultGroupIdRange(self, siteId):
"""Return the default upper and lower group deposition data set identifier codes
assigned to the input siteId.
Any site lacking a default range will get return tuple (-1,-1)
Returns: (lower bound, upper bound) for data set identifiers (int)
"""
if siteId in self.__groupIdAssignments:
GID_START, GID_STOP = self.__groupIdAssignments[siteId]
# elif 'UNASSIGNED' in self.__groupIdAssignments:
# GID_START, GID_STOP = self.__groupIdAssignments['UNASSIGNED']
else:
GID_START, GID_STOP = (-1, -1)
return (GID_START, GID_STOP)
def getDefaultSiteId(self, groupId):
"""Get the default site assignment for the input group data set id."""
return self.__getSiteIdForGroup(groupId)
def __getSiteIdForGroup(self, groupId):
"""Return the siteId to which the input groupId is within the default
code assignment range.
Input may be either a string "G_xxxxxxx" or an integer/string "xxxxxx".
"""
# check default group range assignment --
try:
if str(groupId).startswith("G_"):
idVal = int(str(groupId)[2:])
else:
idVal = int(str(groupId))
#
for ky in self.__groupIdAssignments.keys():
idMin, idMax = self.__groupIdAssignments[ky]
if (idVal >= idMin) and (idVal <= idMax):
return ky
except Exception as e:
if self.__debug:
logger.exception("failed checking group range for %r with %s",
groupId, str(e))
return None
def testParseCutoff(self):
"""Tests if common built in definitions are set"""
cI = ConfigInfo()
val = cI.get("PROJECT_VAL_REL_CUTOFF")
self.assertEqual(len(val), 2)
time_t = self._parseTime(val["start"])
self.assertEqual(time_t.hour, 9)
self.assertEqual(time_t.minute, 0)
self.assertEqual(time_t.second, 0)
self.assertEqual(time_t.isoweekday(), 5)
time_t = self._parseTime(val["end"])
self.assertEqual(time_t.hour, 23)
self.assertEqual(time_t.minute, 59)
self.assertEqual(time_t.second, 59)
self.assertEqual(time_t.isoweekday(), 5)
def __setup(self, siteId=None):
if siteId is not None:
self.__siteId = siteId
else:
self.__siteId = self.__reqObj.getValue("WWPDB_SITE_ID")
self.__sessionObj = self.__reqObj.getSessionObj()
self.__sessionPath = self.__sessionObj.getPath()
self.__cI = ConfigInfo(self.__siteId)
self.__pI = PathInfo(siteId=self.__siteId, sessionPath=self.__sessionPath, verbose=self.__verbose, log=self.__lfh)
#
if self.__debug:
self.__lfh.write("+DataExchange.__setup() - session id %s\n" % (self.__sessionObj.getId()))
self.__lfh.write("+DataExchange.__setup() - session path %s\n" % (self.__sessionObj.getPath()))
self.__lfh.write("+DataExchange.__setup() - data set %s instance %s file source %s\n" % (self.__depDataSetId, self.__wfInstanceId, self.__fileSource))
self.__pI.setDebugFlag(flag=self.__debug)
def setUp(self):
self.__tmpPath = TESTOUTPUT
#
self.__siteId = getSiteId(defaultSiteId=None)
logger.info("\nTesting with site environment for: %s\n" %
self.__siteId)
#
self.__cI = ConfigInfo(self.__siteId)
self.__testFilePath = os.path.join(TOPDIR, 'wwpdb', 'mock-data',
'dp-utils')
self.__testFileCif = '1xbb.cif'
def __setResource(self, resource):
"""Loads resources for access"""
cI = ConfigInfo(self.__siteId)
if resource == "DA_INTERNAL":
self.__dbServer = cI.get("SITE_DB_SERVER")
self.__dbHost = cI.get("SITE_DB_HOST_NAME")
self.__dbUser = cI.get("SITE_DB_USER_NAME")
self.__dbPw = cI.get("SITE_DB_PASSWORD")
self.__dbPort = str(cI.get("SITE_DB_PORT_NUMBER"))
self.__dbSocket = cI.get("SITE_DB_SOCKET")
self.__dbName = cI.get("SITE_DA_INTERNAL_DB_NAME")
else:
raise NameError("Unknown resource %s" % resource)
def setUp(self):
# Pick up site information from the environment or failover to the development site id.
self.__siteId = getSiteId(defaultSiteId=None)
logger.info("\nTesting with site environment for: %s\n" % self.__siteId)
#
self.__cI = ConfigInfo(self.__siteId)
self.__testFilePath = os.path.join(TOPDIR, 'wwpdb', 'mock-data', 'dp-utils')
self.__tmpPath = TESTOUTPUT
#
self.__testFilePdbPisa = '3rer.pdb'
self.__testFileCifPisa = '3rer.cif'
#
logger.info("\nTest file path %s\n" % (self.__testFilePath))
logger.info("\nCIF file path %s\n" % (self.__testFileCifPisa))
def setUp(self):
self.__lfh = sys.stderr
self.__verbose = False
#
self.__serverId = "BACKUP_SERVER_RDI2"
self.__cI = ConfigInfo(siteId=getSiteId(),
verbose=self.__verbose,
log=self.__lfh)
cD = self.__cI.get(self.__serverId, {})
self.__hostName = cD.get("HOST_NAME")
self.__userName = cD.get("HOST_USERNAME")
self.__hostPort = int(cD.get("HOST_PORT"))
# self.__protocol = cD.get("HOST_PROTOCOL")
self.__rootPath = cD.get("HOST_ROOT_PATH")
self.__keyFilePath = cD.get("HOST_KEY_FILE_PATH")
self.__keyFileType = cD.get("HOST_KEY_FILE_TYPE")
#
self.__testLocalFilePath = "./data/TEST-FILE.DAT"
self.__testLocalOutputFilePath = "./JUNK.JUNK"
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
class DensityWrapper:
def __init__(self):
self.__site_id = getSiteId()
self.__cI = ConfigInfo(siteId=self.__site_id)
self.__packagePath = self.__cI.get('SITE_PACKAGES_PATH')
self.node_path = os.path.join(self.__packagePath, 'node', 'bin',
'node')
self.volume_server_pack = self.__cI.get('VOLUME_SERVER_PACK')
self.volume_server_query = self.__cI.get('VOLUME_SERVER_QUERY')
def convert_xray_density_map(self, coord_file, in_2fofc_map, in_fofc_map,
out_binary_cif, working_dir):
xray_conversion = XrayVolumeServerMap(
coord_path=coord_file,
binary_map_out=out_binary_cif,
node_path=self.node_path,
volume_server_query_path=self.volume_server_query,
volume_server_pack_path=self.volume_server_pack,
two_fofc_mmcif_map_coeff_in=in_2fofc_map,
fofc_mmcif_map_coeff_in=in_fofc_map,
working_dir=working_dir)
return xray_conversion.run_process()
def convert_em_volume(self, in_em_volume, out_binary_volume, working_dir):
logging.debug(working_dir)
rdb = RcsbDpUtility(tmpPath=working_dir,
siteId=self.__site_id,
verbose=True)
rdb.imp(in_em_volume)
rdb.op('em-density-bcif')
rdb.exp(out_binary_volume)
if out_binary_volume:
if os.path.exists(out_binary_volume):
return True
return False
