def _configCacheId(self, label): """Return config cache id for given config label""" key, cert = self.__class__.reqmgr['requests'].getKeyCert() configCache = ConfigCache(self.__class__.endpoint + '/couchdb', 'reqmgr_config_cache', ckey=key, cert=cert) try: configCacheId = configCache.getIDFromLabel(label) except: configCacheId = None if configCacheId: return configCacheId # The following will fail if FWCore.ParameterSet not in PYTHONPATH from PSetTweaks.WMTweak import makeTweak configCache.createUserGroup('test', 'test') configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data', 'configs') configCache.addConfig(os.path.join(configDir, label + '.py')) configCache.setLabel(label) configCache.setDescription(label) modPath = imp.find_module(label, [configDir]) loadedConfig = imp.load_module(label, modPath[0], modPath[1], modPath[2]) configCache.setPSetTweaks( makeTweak(loadedConfig.process).jsondictionary()) configCache.save() return configCache.getIDFromLabel(label)
def __init__(self, config, userConfig=None, logger=None): self.config = config self.logger = logger self.fullConfig = None self.outputFile = None #TODO: Deal with user parameters (pycfg_params) if userConfig: cfgBaseName = os.path.basename(userConfig).replace(".py", "") cfgDirName = os.path.dirname(userConfig) if not os.path.isfile( userConfig ): msg = "Cannot find file %s" % userConfig raise ConfigException( msg ) self.logger.debug("Importing CMSSW config %s" % userConfig) modPath = imp.find_module(cfgBaseName, [cfgDirName]) pyCfgParams = getattr(self.config.JobType, 'pyCfgParams', []) if pyCfgParams: originalArgv = sys.argv sys.argv = [userConfig] sys.argv.extend(pyCfgParams) self.fullConfig = imp.load_module(cfgBaseName, modPath[0], modPath[1], modPath[2]) if pyCfgParams: # Restore original sys.argv sys.argv = originalArgv self.tweakJson = makeTweak(self.fullConfig.process).jsonise()
def __init__(self, config, userConfig=None, logger=None): self.config = config self.logger = logger self.fullConfig = None self.outputFile = None #TODO: Deal with user parameters (pycfg_params) if userConfig: cfgBaseName = os.path.basename(userConfig).replace(".py", "") cfgDirName = os.path.dirname(userConfig) if not os.path.isfile(userConfig): msg = "Cannot find file %s" % userConfig raise ConfigException(msg) self.logger.debug("Importing CMSSW config %s" % userConfig) modPath = imp.find_module(cfgBaseName, [cfgDirName]) pyCfgParams = getattr(self.config.JobType, 'pyCfgParams', []) if pyCfgParams: originalArgv = sys.argv sys.argv = [userConfig] sys.argv.extend(pyCfgParams) self.fullConfig = imp.load_module(cfgBaseName, modPath[0], modPath[1], modPath[2]) if pyCfgParams: # Restore original sys.argv sys.argv = originalArgv self.tweakJson = makeTweak(self.fullConfig.process).jsonise()
def upload_to_couch(cfg_name, section_name, user_name, group_name, test_mode=False, url=None): if test_mode: return "00000000000000000" if not os.path.exists(cfg_name): raise RuntimeError("Error: Can't locate config file %s." % cfg_name) # create a file with the ID inside to avoid multiple injections oldID = cfg_name + '.couchID' if os.path.exists(oldID): f = open(oldID) the_id = f.readline().replace('\n', '') f.close() print cfg_name, 'already uploaded with ID', the_id, 'from', oldID return the_id try: loadedConfig = __loadConfig(cfg_name) except: #just try again !! time.sleep(2) loadedConfig = __loadConfig(cfg_name) where = COUCH_DB_ADDRESS if url: where = url configCache = ConfigCache(where, DATABASE_NAME) configCache.createUserGroup(group_name, user_name) configCache.addConfig(cfg_name) configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary()) configCache.setLabel(section_name) configCache.setDescription(section_name) configCache.save() print "Added file to the config cache:" print " DocID: %s" % configCache.document["_id"] print " Revision: %s" % configCache.document["_rev"] f = open(oldID, "w") f.write(configCache.document["_id"]) f.close() return configCache.document["_id"]
def upload_to_couch(cfg_name, section_name, user_name, group_name, test_mode=False, url=None): if test_mode: return "00000000000000000" if not os.path.exists(cfg_name): raise RuntimeError("Error: Can't locate config file %s." % cfg_name) # create a file with the ID inside to avoid multiple injections oldID = cfg_name + '.couchID' if os.path.exists(oldID): f = open(oldID) the_id = f.readline().replace('\n','') f.close() print cfg_name, 'already uploaded with ID', the_id, 'from', oldID return the_id try: loadedConfig = __loadConfig(cfg_name) except: #just try again !! time.sleep(2) loadedConfig = __loadConfig(cfg_name) where = COUCH_DB_ADDRESS if url: where = url configCache = ConfigCache(where, DATABASE_NAME) configCache.createUserGroup(group_name, user_name) configCache.addConfig(cfg_name) configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary()) configCache.setLabel(section_name) configCache.setDescription(section_name) configCache.save() print "Added file to the config cache:" print " DocID: %s" % configCache.document["_id"] print " Revision: %s" % configCache.document["_rev"] f = open(oldID,"w") f.write(configCache.document["_id"]) f.close() return configCache.document["_id"]
def upload(self, configModule, group, userDN, label="Autogenerated by YAAT", description="Autogenerated by YAAT", url="http://se2.accre.vanderbilt.edu:5984", database="wmagent_configcache"): if url.endswith('/'): raise RuntimeError, "URL shouldn't have a trailing slash" configCache = CMSConfigCache(url, database) configCache.createUserGroup(group, userDN) tweaks = makeTweak(configModule.process).jsondictionary() requiredFields = (('dataTier','USER'),('filterName','')) for currentModule in tweaks['process']['outputModules_']: if not 'dataset' in tweaks['process'][currentModule]: tweaks['process'][currentModule]['dataset'] = { "parameters_": [ "dataTier", "filterName" ], "filterName": "", "dataTier": "USER" } continue for reqKey, defaultValue in (('dataTier','USER'),('filterName','')): if not reqKey in tweaks['process'][currentModule]['dataset']['parameters_']: tweaks['process'][currentModule]['dataset']['parameters_'].append(reqKey) tweaks['process'][currentModule]['dataset'][reqKey] = defaultValue _, fileName = tempfile.mkstemp() try: filename = self.writeFile(configModule, fileName) configCache.addConfig(filename) configCache.setPSetTweaks(tweaks) configCache.setLabel(label) configCache.setDescription(description) configCache.save() finally: if os.path.exists(fileName): os.unlink(fileName) targetUrl = "%s/%s/%s/configFile" % (url, database, configCache.document["_id"]) return targetUrl
def __init__(self, config, userConfig=None, logger=None): global configurationCache self.config = config self.logger = logger self.fullConfig = None self.outputFile = None #TODO: Deal with user parameters (pycfg_params) if userConfig: cfgBaseName = os.path.basename(userConfig).replace(".py", "") cfgDirName = os.path.dirname(os.path.abspath(userConfig)) if not os.path.isfile(userConfig): msg = "Cannot find file %s in %s" % (userConfig, os.getcwd()) raise ConfigException(msg) self.logger.debug("Importing CMSSW config %s" % userConfig) pyCfgParams = getattr(self.config.JobType, 'pyCfgParams', []) originalArgv = sys.argv sys.argv = [userConfig] if pyCfgParams: sys.argv.extend(pyCfgParams) self.logger.debug("Extended parameters are %s" % pyCfgParams) file, pathname, description = imp.find_module( cfgBaseName, [cfgDirName]) cacheLine = (tuple(sys.path), tuple(pathname), tuple(sys.argv)) if cacheLine in configurationCache: self.fullConfig = configurationCache[cacheLine] file.close() else: sys.path.append(os.getcwd()) try: self.fullConfig = imp.load_module(cfgBaseName, file, pathname, description) finally: file.close() configurationCache[cacheLine] = self.fullConfig sys.argv = originalArgv self.tweakJson = makeTweak(self.fullConfig.process).jsonise()
def __init__(self, config, userConfig=None, logger=None): global configurationCache self.config = config self.logger = logger self.fullConfig = None self.outputFile = None #TODO: Deal with user parameters (pycfg_params) if userConfig: cfgBaseName = os.path.basename(userConfig).replace(".py", "") cfgDirName = os.path.dirname(os.path.abspath(userConfig)) if not os.path.isfile(userConfig): msg = "Cannot find CMSSW configuration file %s in %s" % (userConfig, os.getcwd()) raise ConfigurationException(msg) self.logger.debug("Importing CMSSW configuration %s" % (userConfig)) pyCfgParams = getattr(self.config.JobType, 'pyCfgParams', []) originalArgv = sys.argv sys.argv = [userConfig] if pyCfgParams: sys.argv.extend(pyCfgParams) msg = "Additional parameters for the CMSSW configuration are: %s" % (pyCfgParams) self.logger.debug(msg) file, pathname, description = imp.find_module(cfgBaseName, [cfgDirName]) cacheLine = (tuple(sys.path), tuple(pathname), tuple(sys.argv)) if cacheLine in configurationCache: self.fullConfig = configurationCache[cacheLine] file.close() else: sys.path.append(os.getcwd()) try: self.fullConfig = imp.load_module(cfgBaseName, file, pathname, description) finally: file.close() configurationCache[cacheLine] = self.fullConfig sys.argv = originalArgv self.tweakJson = makeTweak(self.fullConfig.process).jsonise()
def _configCacheId(self, label): """Return config cache id for given config label""" key, cert = self.__class__.reqmgr['requests'].getKeyCert() configCache = ConfigCache(self.__class__.endpoint + '/couchdb', 'reqmgr_config_cache', ckey = key, cert = cert) try: configCacheId = configCache.getIDFromLabel(label) except: configCacheId = None if configCacheId: return configCacheId # The following will fail if FWCore.ParameterSet not in PYTHONPATH from PSetTweaks.WMTweak import makeTweak configCache.createUserGroup('test', 'test') configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data', 'configs') configCache.addConfig(os.path.join(configDir, label + '.py')) configCache.setLabel(label) configCache.setDescription(label) modPath = imp.find_module(label, [configDir]) loadedConfig = imp.load_module(label, modPath[0], modPath[1], modPath[2]) configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary()) configCache.save() return configCache.getIDFromLabel(label)