def testBasicInit(self):
"""
_testBasicInit_
Test init values
"""
mc = MemoryCacheStruct(0, lambda x: x, initCacheValue=Foo())
self.assertIsInstance(mc.data, Foo)
mc1 = MemoryCacheStruct(0, lambda x: sum(x), [], kwargs={'x': [1, 2]})
self.assertEqual(mc1.data, [])
after = mc1.getData()
self.assertEqual(after, 3)
mc2 = MemoryCacheStruct(0,
lambda x: x, {},
kwargs={'x': {
'one': 1,
'two': 2
}})
self.assertEqual(mc2.data, {})
after = mc2.getData()
self.assertItemsEqual(after.keys(), ['one', 'two'])
return
def getRequestByStatusFromMemoryCache(self, statusList, expire=0):
return MemoryCacheStruct(expire=expire,
func=self.getRequestByStatus,
initCacheValue=[],
kwargs={
'statusList': statusList,
"detail": False
})
def _getDataFromMemoryCache(self, callname):
cache = MemoryCacheStruct(expire=0,
func=self._getResult,
initCacheValue={},
kwargs={
'callname': callname,
"verb": "GET"
})
return cache.getData()
def testExists(self):
"""
Tests whether a given cache already exists
"""
mc = MemoryCacheStruct(1, lambda x: int(time.time()), kwargs={'x': 1})
self.assertFalse(GenericDataCache.cacheExists("tCache"))
GenericDataCache.registerCache("tCache", mc)
self.assertTrue(GenericDataCache.cacheExists("tCache"))
self.assertFalse(GenericDataCache.cacheExists("tCache2"))
def getAbortedAndForceCompleteRequestsFromMemoryCache(self, expire=0):
"""
_getAbortedAndForceCompleteRequestsFromMemoryCache_
"""
# imports here to avoid the dependency not using this function
from WMCore.Cache.GenericDataCache import MemoryCacheStruct
maskStates = ["aborted", "aborted-completed", "force-complete"]
return MemoryCacheStruct(expire=expire,
func=self.getRequestByStatus,
initCacheValue=[],
kwargs={
'statusList': maskStates,
"detail": False
})
def _validateDatatier(datatier, dbsUrl, expiration=3600):
"""
_validateDatatier_
Provided a list of datatiers extracted from the outputDatasets, checks
whether they all exist in DBS.
"""
cacheName = "dataTierList_" + md5(dbsUrl).hexdigest()
if not GenericDataCache.cacheExists(cacheName):
mc = MemoryCacheStruct(expiration,
getDataTiers,
kwargs={'dbsUrl': dbsUrl})
GenericDataCache.registerCache(cacheName, mc)
cacheData = GenericDataCache.getCacheData(cacheName)
dbsTiers = cacheData.getData()
badTiers = list(set(datatier) - set(dbsTiers))
if badTiers:
raise InvalidSpecParameterValue(
"Bad datatier(s): %s not available in DBS." % badTiers)
def testBasic(self):
"""
_testBasic_
Basic stuff.
"""
mc = MemoryCacheStruct(1, lambda x: int(time.time()), {'x':1})
self.assertEqual(mc.lastUpdated, -1)
GenericDataCache.registerCache("test", mc)
with self.assertRaises(CacheExistException):
GenericDataCache.registerCache("test", mc)
with self.assertRaises(CacheWithWrongStructException):
GenericDataCache.registerCache("test2", {'a': 1})
mc2 = GenericDataCache.getCacheData('test')
before = mc2.getData()
time.sleep(2)
after = mc2.getData()
self.assertFalse(before == after)
self.assertFalse(mc2.lastUpdate == -1)
return
def sites():
"Return known CMS site list from CRIC"
try:
# Download a list of all the sites from CRIC
cric = CRIC()
site_list = sorted(cric.getAllPSNs())
except Exception as exc:
msg = "ERROR: Could not retrieve sites from CRIC, reason: %s" % str(
exc)
raise Exception(msg)
return site_list
# create a site cache and pnn cache 2 hour duration
SITE_CACHE = MemoryCacheStruct(5200, sites)
def pnns():
"""
Returns all PhEDEx node names, excluding Buffer endpoints
"""
cric = CRIC()
try:
pnn_list = sorted(cric.getAllPhEDExNodeNames(excludeBuffer=True))
except Exception as exc:
msg = "ERROR: Could not retrieve PNNs from CRIC, reason: %s" % str(exc)
raise Exception(msg)
return pnn_list
def __init__(self, app, config, mount):
self.base = config.base
self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
if config and not isinstance(config, dict):
web_config = config.dictionary_()
if not config:
web_config = {'base': self.base}
TemplatedPage.__init__(self, web_config)
imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images')
self.imgdir = web_config.get('imgdir', imgdir)
cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css')
self.cssdir = web_config.get('cssdir', cssdir)
jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js')
self.jsdir = web_config.get('jsdir', jsdir)
spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs')
self.spdir = web_config.get('spdir', spdir)
# read scripts area and initialize data-ops scripts
self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts')
self.sdir = web_config.get('sdir', self.sdir)
self.sdict_thr = web_config.get('sdict_thr',
600) # put reasonable 10 min interval
self.sdict = {'ts': time.time()} # placeholder for data-ops scripts
self.update_scripts(force=True)
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
self.std_specs = spec_list(std_specs_dir)
self.std_specs.sort()
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += [
'application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript'
]
cherryconf.update({
'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
# initialize access to reqmgr2 APIs
self.reqmgr_url = config.reqmgr.reqmgr2_url
self.reqmgr = ReqMgr(self.reqmgr_url)
# only gets current view (This might cause to reponse time much longer,
# If upto date view is not needed overwrite Fale)
self.reqmgr._noStale = True
# get fields which we'll use in templates
cdict = config.reqmgr.dictionary_()
self.couch_url = cdict.get('couch_host', '')
self.couch_dbname = cdict.get('couch_reqmgr_db', '')
self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
self.acdc_url = cdict.get('acdc_host', '')
self.acdc_dbname = cdict.get('acdc_db', '')
self.configcache_url = cdict.get('couch_config_cache_url',
self.couch_url)
self.dbs_url = cdict.get('dbs_url', '')
self.dqm_url = cdict.get('dqm_url', '')
self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1')
self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493')
# LogDB holder
centralurl = cdict.get("central_logdb_url", "")
identifier = cdict.get("log_reporter", "reqmgr2")
self.logdb = LogDB(centralurl, identifier)
# local team cache which will request data from wmstats
base, uri = self.reqmgr_url.split('://')
base_url = '%s://%s' % (base, uri.split('/')[0])
self.wmstatsurl = cdict.get('wmstats_url',
'%s/wmstatsserver' % base_url)
if not self.wmstatsurl:
raise Exception(
'ReqMgr2 configuration file does not provide wmstats url')
# cache team information for 2 hours to limit wmstatsserver API calls
self.TEAM_CACHE = MemoryCacheStruct(7200, self.refreshTeams)
# fetch assignment arguments specification from StdBase
self.assignArgs = StdBase().getWorkloadAssignArgs()
self.assignArgs = {
key: val['default']
for key, val in viewitems(self.assignArgs)
}