def _configCacheId(self, label):
"""Return config cache id for given config label"""
key, cert = self.__class__.reqmgr['requests'].getKeyCert()
configCache = ConfigCache(self.__class__.endpoint + '/couchdb',
'reqmgr_config_cache',
ckey=key,
cert=cert)
try:
configCacheId = configCache.getIDFromLabel(label)
except:
configCacheId = None
if configCacheId:
return configCacheId
# The following will fail if FWCore.ParameterSet not in PYTHONPATH
from PSetTweaks.WMTweak import makeTweak
configCache.createUserGroup('test', 'test')
configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data',
'configs')
configCache.addConfig(os.path.join(configDir, label + '.py'))
configCache.setLabel(label)
configCache.setDescription(label)
modPath = imp.find_module(label, [configDir])
loadedConfig = imp.load_module(label, modPath[0], modPath[1],
modPath[2])
configCache.setPSetTweaks(
makeTweak(loadedConfig.process).jsondictionary())
configCache.save()
return configCache.getIDFromLabel(label)
def __init__(self, config, userConfig=None, logger=None):
self.config = config
self.logger = logger
self.fullConfig = None
self.outputFile = None
#TODO: Deal with user parameters (pycfg_params)
if userConfig:
cfgBaseName = os.path.basename(userConfig).replace(".py", "")
cfgDirName = os.path.dirname(userConfig)
if not os.path.isfile( userConfig ):
msg = "Cannot find file %s" % userConfig
raise ConfigException( msg )
self.logger.debug("Importing CMSSW config %s" % userConfig)
modPath = imp.find_module(cfgBaseName, [cfgDirName])
pyCfgParams = getattr(self.config.JobType, 'pyCfgParams', [])
if pyCfgParams:
originalArgv = sys.argv
sys.argv = [userConfig]
sys.argv.extend(pyCfgParams)
self.fullConfig = imp.load_module(cfgBaseName, modPath[0],
modPath[1], modPath[2])
if pyCfgParams: # Restore original sys.argv
sys.argv = originalArgv
self.tweakJson = makeTweak(self.fullConfig.process).jsonise()
def __init__(self, config, userConfig=None, logger=None):
self.config = config
self.logger = logger
self.fullConfig = None
self.outputFile = None
#TODO: Deal with user parameters (pycfg_params)
if userConfig:
cfgBaseName = os.path.basename(userConfig).replace(".py", "")
cfgDirName = os.path.dirname(userConfig)
if not os.path.isfile(userConfig):
msg = "Cannot find file %s" % userConfig
raise ConfigException(msg)
self.logger.debug("Importing CMSSW config %s" % userConfig)
modPath = imp.find_module(cfgBaseName, [cfgDirName])
pyCfgParams = getattr(self.config.JobType, 'pyCfgParams', [])
if pyCfgParams:
originalArgv = sys.argv
sys.argv = [userConfig]
sys.argv.extend(pyCfgParams)
self.fullConfig = imp.load_module(cfgBaseName, modPath[0],
modPath[1], modPath[2])
if pyCfgParams: # Restore original sys.argv
sys.argv = originalArgv
self.tweakJson = makeTweak(self.fullConfig.process).jsonise()
def upload_to_couch(cfg_name,
section_name,
user_name,
group_name,
test_mode=False,
url=None):
if test_mode:
return "00000000000000000"
if not os.path.exists(cfg_name):
raise RuntimeError("Error: Can't locate config file %s." % cfg_name)
# create a file with the ID inside to avoid multiple injections
oldID = cfg_name + '.couchID'
if os.path.exists(oldID):
f = open(oldID)
the_id = f.readline().replace('\n', '')
f.close()
print cfg_name, 'already uploaded with ID', the_id, 'from', oldID
return the_id
try:
loadedConfig = __loadConfig(cfg_name)
except:
#just try again !!
time.sleep(2)
loadedConfig = __loadConfig(cfg_name)
where = COUCH_DB_ADDRESS
if url:
where = url
configCache = ConfigCache(where, DATABASE_NAME)
configCache.createUserGroup(group_name, user_name)
configCache.addConfig(cfg_name)
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.setLabel(section_name)
configCache.setDescription(section_name)
configCache.save()
print "Added file to the config cache:"
print " DocID: %s" % configCache.document["_id"]
print " Revision: %s" % configCache.document["_rev"]
f = open(oldID, "w")
f.write(configCache.document["_id"])
f.close()
return configCache.document["_id"]
def upload_to_couch(cfg_name, section_name, user_name, group_name, test_mode=False, url=None):
if test_mode:
return "00000000000000000"
if not os.path.exists(cfg_name):
raise RuntimeError("Error: Can't locate config file %s." % cfg_name)
# create a file with the ID inside to avoid multiple injections
oldID = cfg_name + '.couchID'
if os.path.exists(oldID):
f = open(oldID)
the_id = f.readline().replace('\n','')
f.close()
print cfg_name, 'already uploaded with ID', the_id, 'from', oldID
return the_id
try:
loadedConfig = __loadConfig(cfg_name)
except:
#just try again !!
time.sleep(2)
loadedConfig = __loadConfig(cfg_name)
where = COUCH_DB_ADDRESS
if url:
where = url
configCache = ConfigCache(where, DATABASE_NAME)
configCache.createUserGroup(group_name, user_name)
configCache.addConfig(cfg_name)
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.setLabel(section_name)
configCache.setDescription(section_name)
configCache.save()
print "Added file to the config cache:"
print " DocID: %s" % configCache.document["_id"]
print " Revision: %s" % configCache.document["_rev"]
f = open(oldID,"w")
f.write(configCache.document["_id"])
f.close()
return configCache.document["_id"]
def upload(self, configModule, group, userDN,
label="Autogenerated by YAAT",
description="Autogenerated by YAAT",
url="http://se2.accre.vanderbilt.edu:5984",
database="wmagent_configcache"):
if url.endswith('/'):
raise RuntimeError, "URL shouldn't have a trailing slash"
configCache = CMSConfigCache(url, database)
configCache.createUserGroup(group, userDN)
tweaks = makeTweak(configModule.process).jsondictionary()
requiredFields = (('dataTier','USER'),('filterName',''))
for currentModule in tweaks['process']['outputModules_']:
if not 'dataset' in tweaks['process'][currentModule]:
tweaks['process'][currentModule]['dataset'] = {
"parameters_": [
"dataTier",
"filterName"
],
"filterName": "",
"dataTier": "USER"
}
continue
for reqKey, defaultValue in (('dataTier','USER'),('filterName','')):
if not reqKey in tweaks['process'][currentModule]['dataset']['parameters_']:
tweaks['process'][currentModule]['dataset']['parameters_'].append(reqKey)
tweaks['process'][currentModule]['dataset'][reqKey] = defaultValue
_, fileName = tempfile.mkstemp()
try:
filename = self.writeFile(configModule, fileName)
configCache.addConfig(filename)
configCache.setPSetTweaks(tweaks)
configCache.setLabel(label)
configCache.setDescription(description)
configCache.save()
finally:
if os.path.exists(fileName):
os.unlink(fileName)
targetUrl = "%s/%s/%s/configFile" % (url, database, configCache.document["_id"])
return targetUrl
def __init__(self, config, userConfig=None, logger=None):
global configurationCache
self.config = config
self.logger = logger
self.fullConfig = None
self.outputFile = None
#TODO: Deal with user parameters (pycfg_params)
if userConfig:
cfgBaseName = os.path.basename(userConfig).replace(".py", "")
cfgDirName = os.path.dirname(os.path.abspath(userConfig))
if not os.path.isfile(userConfig):
msg = "Cannot find file %s in %s" % (userConfig, os.getcwd())
raise ConfigException(msg)
self.logger.debug("Importing CMSSW config %s" % userConfig)
pyCfgParams = getattr(self.config.JobType, 'pyCfgParams', [])
originalArgv = sys.argv
sys.argv = [userConfig]
if pyCfgParams:
sys.argv.extend(pyCfgParams)
self.logger.debug("Extended parameters are %s" % pyCfgParams)
file, pathname, description = imp.find_module(
cfgBaseName, [cfgDirName])
cacheLine = (tuple(sys.path), tuple(pathname), tuple(sys.argv))
if cacheLine in configurationCache:
self.fullConfig = configurationCache[cacheLine]
file.close()
else:
sys.path.append(os.getcwd())
try:
self.fullConfig = imp.load_module(cfgBaseName, file,
pathname, description)
finally:
file.close()
configurationCache[cacheLine] = self.fullConfig
sys.argv = originalArgv
self.tweakJson = makeTweak(self.fullConfig.process).jsonise()
def __init__(self, config, userConfig=None, logger=None):
global configurationCache
self.config = config
self.logger = logger
self.fullConfig = None
self.outputFile = None
#TODO: Deal with user parameters (pycfg_params)
if userConfig:
cfgBaseName = os.path.basename(userConfig).replace(".py", "")
cfgDirName = os.path.dirname(os.path.abspath(userConfig))
if not os.path.isfile(userConfig):
msg = "Cannot find CMSSW configuration file %s in %s" % (userConfig, os.getcwd())
raise ConfigurationException(msg)
self.logger.debug("Importing CMSSW configuration %s" % (userConfig))
pyCfgParams = getattr(self.config.JobType, 'pyCfgParams', [])
originalArgv = sys.argv
sys.argv = [userConfig]
if pyCfgParams:
sys.argv.extend(pyCfgParams)
msg = "Additional parameters for the CMSSW configuration are: %s" % (pyCfgParams)
self.logger.debug(msg)
file, pathname, description = imp.find_module(cfgBaseName, [cfgDirName])
cacheLine = (tuple(sys.path), tuple(pathname), tuple(sys.argv))
if cacheLine in configurationCache:
self.fullConfig = configurationCache[cacheLine]
file.close()
else:
sys.path.append(os.getcwd())
try:
self.fullConfig = imp.load_module(cfgBaseName, file, pathname, description)
finally:
file.close()
configurationCache[cacheLine] = self.fullConfig
sys.argv = originalArgv
self.tweakJson = makeTweak(self.fullConfig.process).jsonise()
def _configCacheId(self, label):
"""Return config cache id for given config label"""
key, cert = self.__class__.reqmgr['requests'].getKeyCert()
configCache = ConfigCache(self.__class__.endpoint + '/couchdb', 'reqmgr_config_cache', ckey = key, cert = cert)
try:
configCacheId = configCache.getIDFromLabel(label)
except:
configCacheId = None
if configCacheId:
return configCacheId
# The following will fail if FWCore.ParameterSet not in PYTHONPATH
from PSetTweaks.WMTweak import makeTweak
configCache.createUserGroup('test', 'test')
configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data', 'configs')
configCache.addConfig(os.path.join(configDir, label + '.py'))
configCache.setLabel(label)
configCache.setDescription(label)
modPath = imp.find_module(label, [configDir])
loadedConfig = imp.load_module(label, modPath[0], modPath[1], modPath[2])
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.save()
return configCache.getIDFromLabel(label)
