def _wmAgentConfig(configFile): config = Configuration() config.section_("General") config.General.logdb_name = "unittest_logdb" config.General.central_logdb_url = "http://localhost/central_logdb" config.General.ReqMgr2ServiceURL = "http://localhost/reqmgr2" config.section_("JobStateMachine") # Waring setting couchDB to None will cause the ERROR: # but that should be ignored, if you want to test couchDB # set the real couchDB information here config.JobStateMachine.couchurl = os.getenv("COUCHURL") config.JobStateMachine.couchDBName = os.getenv("COUCHDB") config.JobStateMachine.jobSummaryDBName = "wmagent_summary_test" config.JobStateMachine.summaryStatsDBName = "stat_summary_test" config.section_("Agent") # User specific parameter config.Agent.hostName = "cmssrv52.fnal.gov" # User specific parameter config.Agent.contact = "*****@*****.**" # User specific parameter config.Agent.teamName = "DMWM" # User specific parameter config.Agent.agentName = "WMAgentCommissioning" config.Agent.useMsgService = False config.Agent.useTrigger = False config.Agent.isDocker = False # BossAir setup config.section_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'SimpleCondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' saveConfigurationFile(config, configFile)
def _wmAgentConfig(configFile): config = Configuration() config.section_("General") config.General.logdb_name = "unittest_logdb" config.General.central_logdb_url = "http://localhost/central_logdb" config.General.ReqMgr2ServiceURL = "http://localhost/reqmgr2" config.section_("JobStateMachine") # Waring setting couchDB to None will cause the ERROR: # but that should be ignored, if you want to test couchDB # set the real couchDB information here config.JobStateMachine.couchurl = os.getenv("COUCHURL") config.JobStateMachine.couchDBName = os.getenv("COUCHDB") config.JobStateMachine.jobSummaryDBName = "wmagent_summary_test" config.JobStateMachine.summaryStatsDBName = "stat_summary_test" config.section_("Agent") # User specific parameter config.Agent.hostName = "cmssrv52.fnal.gov" # User specific parameter config.Agent.contact = "*****@*****.**" # User specific parameter config.Agent.teamName = "DMWM" # User specific parameter config.Agent.agentName = "WMAgentCommissioning" config.Agent.useMsgService = False config.Agent.useTrigger = False # BossAir setup config.section_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'SimpleCondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' saveConfigurationFile(config, configFile)
def _wmAgentConfig(configFile): config = Configuration() config.section_("JobStateMachine") #Waring setting couchDB to None will cause the ERROR: # but that should be ignored, if you want to test couchDB # set the real couchDB information here config.JobStateMachine.couchurl = os.getenv("COUCHURL") config.JobStateMachine.couchDBName = os.getenv("COUCHDB") config.section_("Agent") # User specific parameter config.Agent.hostName = "cmssrv52.fnal.gov" # User specific parameter config.Agent.contact = "*****@*****.**" # User specific parameter config.Agent.teamName = "DMWM" # User specific parameter config.Agent.agentName = "WMAgentCommissioning" config.Agent.useMsgService = False config.Agent.useTrigger = False # BossAir setup config.section_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' saveConfigurationFile(config, configFile)
def _wmAgentConfig(configFile): config = Configuration() config.section_("JobStateMachine") # Waring setting couchDB to None will cause the ERROR: # but that should be ignored, if you want to test couchDB # set the real couchDB information here config.JobStateMachine.couchurl = os.getenv("COUCHURL") config.JobStateMachine.couchDBName = os.getenv("COUCHDB") config.JobStateMachine.jobSummaryDBName = "wmagent_summary_test" config.JobStateMachine.summaryStatsDBName = "stat_summary_test" config.section_("Agent") # User specific parameter config.Agent.hostName = "cmssrv52.fnal.gov" # User specific parameter config.Agent.contact = "*****@*****.**" # User specific parameter config.Agent.teamName = "DMWM" # User specific parameter config.Agent.agentName = "WMAgentCommissioning" config.Agent.useMsgService = False config.Agent.useTrigger = False # BossAir setup config.section_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'SimpleCondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' # TaskArchive setup (JobSubmitter needs this) config.component_("TaskArchiver") config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2" saveConfigurationFile(config, configFile)
def _wmAgentConfig(configFile): config = Configuration() config.section_("JobStateMachine") #Waring setting couchDB to None will cause the ERROR: # but that should be ignored, if you want to test couchDB # set the real couchDB information here config.JobStateMachine.couchurl = os.getenv("COUCHURL") config.JobStateMachine.couchDBName = os.getenv("COUCHDB") config.JobStateMachine.jobSummaryDBName = "wmagent_summary_test" config.JobStateMachine.summaryStatsDBName = "stat_summary_test" config.section_("Agent") # User specific parameter config.Agent.hostName = "cmssrv52.fnal.gov" # User specific parameter config.Agent.contact = "*****@*****.**" # User specific parameter config.Agent.teamName = "DMWM" # User specific parameter config.Agent.agentName = "WMAgentCommissioning" config.Agent.useMsgService = False config.Agent.useTrigger = False # BossAir setup config.section_("BossAir") config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' #TaskArchive setup (JobSubmitter needs this) config.component_("TaskArchiver") config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2" saveConfigurationFile(config, configFile)
def _emulatorCofig(phedex, dbs, siteDB, requestMgr, configFile): config = Configuration() config.section_("Emulator") config.Emulator.PhEDEx = phedex config.Emulator.DBSReader = dbs config.Emulator.RequestMgr = requestMgr config.Emulator.SiteDB = siteDB saveConfigurationFile(config, configFile) print "create config file:%s, PhEDEx: %s, DBS: %s, RequestManager: %s, SiteDB %s with flag" \ % (configFile, phedex, dbs, siteDB, requestMgr)
def testE(self): """test save/load """ testValues = [ "string", 123, 123.456, ["list", 789, 10.1 ], { "dict1" : "value", "dict2" : 10.0 } ] config = Configuration() for x in range(0, 5): config.section_("Section%s" % x) config.component_("Component%s" % x) sect = getattr(config, "Section%s" % x) comp = getattr(config, "Component%s" % x) sect.document_("This is Section%s" % x) comp.document_("This is Component%s" % x) for i in range(0, 5): setattr(comp, "Parameter%s" % i, testValues[i]) setattr(sect, "Parameter%s" % i, testValues[i]) comp.document_("This is Parameter%s" % i, "Parameter%s" %i) sect.document_("This is Parameter%s" %i, "Parameter%s" %i) stringSave = str(config) documentSave = config.documentedString_() commentSave = config.commentedString_() saveConfigurationFile(config, self.normalSave) saveConfigurationFile(config, self.docSave, document = True) saveConfigurationFile(config, self.commentSave, comment = True) plainConfig = loadConfigurationFile(self.normalSave) docConfig = loadConfigurationFile(self.docSave) commentConfig = loadConfigurationFile(self.commentSave)
def testE(self): """test save/load """ testValues = [ "string", 123, 123.456, ["list", 789, 10.1 ], { "dict1" : "value", "dict2" : 10.0 } ] config = Configuration() for x in range(0, 5): config.section_("Section%s" % x) config.component_("Component%s" % x) sect = getattr(config, "Section%s" % x) comp = getattr(config, "Component%s" % x) sect.document_("This is Section%s" % x) comp.document_("This is Component%s" % x) for i in range(0, 5): setattr(comp, "Parameter%s" % i, testValues[i]) setattr(sect, "Parameter%s" % i, testValues[i]) comp.document_("This is Parameter%s" % i, "Parameter%s" %i) sect.document_("This is Parameter%s" %i, "Parameter%s" %i) dummyStringSave = str(config) dummyDocumentSave = config.documentedString_() dummyCommentSave = config.commentedString_() saveConfigurationFile(config, self.normalSave) saveConfigurationFile(config, self.docSave, document = True) saveConfigurationFile(config, self.commentSave, comment = True) dummyPlainConfig = loadConfigurationFile(self.normalSave) dummyDocConfig = loadConfigurationFile(self.docSave) dummyCommentConfig = loadConfigurationFile(self.commentSave)
def SubmitJob(key, value): doAll = False doTest = False allSelKeys = selSubmitKey.split() if selSubmitKey.find('NONE') != -1: print "Nothing to be done!" sys.exit() if selSubmitKey.find('ALL') != -1: doAll = True if selSubmitKey.find('TEST') != -1: doTest = True doThis = doAll if not doAll: for selKey in allSelKeys: if key.find(selKey) != -1: doThis = True break; if not doThis: return tempconfig = copy.deepcopy(config) tempconfig.General.requestName = key tempconfig.General.workArea = workArea tempconfig.Data.outputDatasetTag = Pubname + "_" + key tempconfig.Data.outLFNDirBase = outDir if len(value) <3: print "Not enough argument for %s" % key raise AssertionError() if value[0]: # Data if key.find('Run2015C') != -1: tempconfig.JobType.pyCfgParams = ['mcInfo=0', 'GlobalTag=74X_dataRun2_v4', 'specialFix=JEC', 'jecDBname=Summer15_25nsV6_DATA', 'externalFilterList=csc2015_Dec01.txt.tar.gz,ecalscn1043093_Dec01.txt.tar.gz,badResolutionTrack_Jan13.txt.tar.gz,muonBadTrack_Jan13.txt.tar.gz'] tempconfig.JobType.inputFiles = [json_25ns, 'Summer15_25nsV6_DATA.db', 'csc2015_Dec01.txt.tar.gz', 'ecalscn1043093_Dec01.txt.tar.gz', 'csc2015_Dec01.txt', 'ecalscn1043093_Dec01.txt', 'badResolutionTrack_Jan13.txt', 'muonBadTrack_Jan13.txt'] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns elif key.find('Run2015D-05Oct2015') != -1: tempconfig.JobType.pyCfgParams = ['mcInfo=0', 'GlobalTag=74X_dataRun2_reMiniAOD_v0', 'specialFix=JEC', 'jecDBname=Summer15_25nsV6_DATA', 'externalFilterList=csc2015_Dec01.txt.tar.gz,ecalscn1043093_Dec01.txt.tar.gz,badResolutionTrack_Jan13.txt.tar.gz,muonBadTrack_Jan13.txt.tar.gz'] tempconfig.JobType.inputFiles = [json_25ns, 'Summer15_25nsV6_DATA.db', 'csc2015_Dec01.txt.tar.gz', 'ecalscn1043093_Dec01.txt.tar.gz', 'csc2015_Dec01.txt', 'ecalscn1043093_Dec01.txt', 'badResolutionTrack_Jan13.txt', 'muonBadTrack_Jan13.txt'] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns elif key.find('Run2015D-PromptReco') != -1: tempconfig.JobType.pyCfgParams = ['mcInfo=0', 'GlobalTag=74X_dataRun2_Prompt_v4', 'specialFix=JEC', 'jecDBname=Summer15_25nsV6_DATA', 'externalFilterList=csc2015_Dec01.txt.tar.gz,ecalscn1043093_Dec01.txt.tar.gz,badResolutionTrack_Jan13.txt.tar.gz,muonBadTrack_Jan13.txt.tar.gz'] tempconfig.JobType.inputFiles = [json_25ns, 'Summer15_25nsV6_DATA.db', 'csc2015_Dec01.txt.tar.gz', 'ecalscn1043093_Dec01.txt.tar.gz', 'csc2015_Dec01.txt', 'ecalscn1043093_Dec01.txt', 'badResolutionTrack_Jan13.txt', 'muonBadTrack_Jan13.txt'] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns else: pass else: if key.find('FastSim') != -1: tempconfig.JobType.pyCfgParams = ['mcInfo=1', 'GlobalTag=74X_mcRun2_asymptotic_v2', 'specialFix=JEC', 'jecDBname=MCRUN2_74_V9', 'fastsim=1'] tempconfig.JobType.inputFiles = ['MCRUN2_74_V9.db'] tempconfig.Data.splitting = 'FileBased' else: tempconfig.JobType.pyCfgParams = ['mcInfo=1', 'GlobalTag=74X_mcRun2_asymptotic_v2', 'specialFix=JEC', 'jecDBname=Summer15_25nsV6_MC'] tempconfig.JobType.inputFiles = ['Summer15_25nsV6_MC.db'] tempconfig.Data.splitting = 'FileBased' tempconfig.Data.inputDataset = value[1].strip() tempconfig.Data.unitsPerJob = value[2] if value[0] and len(value) > 3: tempconfig.Data.lumiMask = value[3] # Submitting jobs if doTest: saveConfigurationFile(tempconfig, workArea+"/test/"+key+"_test_cfg.py") tasklist["crab_"+key] = key else: results = crabCommand('submit', config = tempconfig) tasklist[results['uniquerequestname']] = key del tempconfig
def SubmitJob(key, value): doAll = False doTest = False allSelKeys = selSubmitKey.split() if selSubmitKey.find('NONE') != -1: print "Nothing to be done!" sys.exit() if selSubmitKey.find('ALL') != -1: doAll = True if selSubmitKey.find('TEST') != -1: doTest = True doThis = doAll if not doAll: for selKey in allSelKeys: if key.find(selKey) != -1: doThis = True break if not doThis: return tempconfig = copy.deepcopy(config) tempconfig.General.requestName = key tempconfig.General.workArea = workArea tempconfig.Data.outputDatasetTag = Pubname + "_" + key tempconfig.Data.outLFNDirBase = outDir if len(value) < 3: print "Not enough argument for %s" % key raise AssertionError() if value[0]: # Data if key.find('Run2015C') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=0', 'GlobalTag=74X_dataRun2_v4', 'specialFix=JEC', 'jecDBname=Summer15_25nsV6_DATA', 'externalFilterList=csc2015_Dec01.txt.tar.gz,ecalscn1043093_Dec01.txt.tar.gz,badResolutionTrack_Jan13.txt.tar.gz,muonBadTrack_Jan13.txt.tar.gz' ] tempconfig.JobType.inputFiles = [ json_25ns, 'Summer15_25nsV6_DATA.db', 'csc2015_Dec01.txt.tar.gz', 'ecalscn1043093_Dec01.txt.tar.gz', 'csc2015_Dec01.txt', 'ecalscn1043093_Dec01.txt', 'badResolutionTrack_Jan13.txt', 'muonBadTrack_Jan13.txt' ] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns elif key.find('Run2015D-05Oct2015') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=0', 'GlobalTag=74X_dataRun2_reMiniAOD_v0', 'specialFix=JEC', 'jecDBname=Summer15_25nsV6_DATA', 'externalFilterList=csc2015_Dec01.txt.tar.gz,ecalscn1043093_Dec01.txt.tar.gz,badResolutionTrack_Jan13.txt.tar.gz,muonBadTrack_Jan13.txt.tar.gz' ] tempconfig.JobType.inputFiles = [ json_25ns, 'Summer15_25nsV6_DATA.db', 'csc2015_Dec01.txt.tar.gz', 'ecalscn1043093_Dec01.txt.tar.gz', 'csc2015_Dec01.txt', 'ecalscn1043093_Dec01.txt', 'badResolutionTrack_Jan13.txt', 'muonBadTrack_Jan13.txt' ] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns elif key.find('Run2015D-PromptReco') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=0', 'GlobalTag=74X_dataRun2_Prompt_v4', 'specialFix=JEC', 'jecDBname=Summer15_25nsV6_DATA', 'externalFilterList=csc2015_Dec01.txt.tar.gz,ecalscn1043093_Dec01.txt.tar.gz,badResolutionTrack_Jan13.txt.tar.gz,muonBadTrack_Jan13.txt.tar.gz' ] tempconfig.JobType.inputFiles = [ json_25ns, 'Summer15_25nsV6_DATA.db', 'csc2015_Dec01.txt.tar.gz', 'ecalscn1043093_Dec01.txt.tar.gz', 'csc2015_Dec01.txt', 'ecalscn1043093_Dec01.txt', 'badResolutionTrack_Jan13.txt', 'muonBadTrack_Jan13.txt' ] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns else: pass else: if key.find('FastSim') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=1', 'GlobalTag=74X_mcRun2_asymptotic_v2', 'specialFix=JEC', 'jecDBname=MCRUN2_74_V9', 'fastsim=1' ] tempconfig.JobType.inputFiles = ['MCRUN2_74_V9.db'] tempconfig.Data.splitting = 'FileBased' else: tempconfig.JobType.pyCfgParams = [ 'mcInfo=1', 'GlobalTag=74X_mcRun2_asymptotic_v2', 'specialFix=JEC', 'jecDBname=Summer15_25nsV6_MC' ] tempconfig.JobType.inputFiles = ['Summer15_25nsV6_MC.db'] tempconfig.Data.splitting = 'FileBased' tempconfig.Data.inputDataset = value[1].strip() tempconfig.Data.unitsPerJob = value[2] if value[0] and len(value) > 3: tempconfig.Data.lumiMask = value[3] # Submitting jobs if doTest: saveConfigurationFile(tempconfig, workArea + "/test/" + key + "_test_cfg.py") tasklist["crab_" + key] = key else: results = crabCommand('submit', config=tempconfig) tasklist[results['uniquerequestname']] = key del tempconfig
def SubmitJob(key, value): doAll = False doTest = False allSelKeys = selSubmitKey.split() if selSubmitKey.find('NONE') != -1: print "Nothing to be done!" sys.exit() if selSubmitKey.find('ALL') != -1: doAll = True if selSubmitKey.find('TEST') != -1: doTest = True doThis = doAll if not doAll: for selKey in allSelKeys: if key.find(selKey) != -1: doThis = True break if not doThis: return tempconfig = copy.deepcopy(config) tempconfig.General.requestName = key tempconfig.General.workArea = workArea tempconfig.Data.outputDatasetTag = Pubname + "_" + key tempconfig.Data.outLFNDirBase = outDir if len(value) < 3: print "Not enough argument for %s" % key raise AssertionError() if value[ 0]: # Data : note the version number after 23Sep2016 is complicated, so removing them if key.find('Run2016B-23Sep2016') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=0', 'GlobalTag=80X_dataRun2_2016SeptRepro_v3', 'specialFix=JEC BADMUON', 'jecDBname=Summer16_23Sep2016AllV3_DATA' ] tempconfig.JobType.inputFiles = [ json_25ns, 'Summer16_23Sep2016AllV3_DATA.db' ] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns elif key.find('Run2016C-23Sep2016') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=0', 'GlobalTag=80X_dataRun2_2016SeptRepro_v3', 'specialFix=JEC BADMUON', 'jecDBname=Summer16_23Sep2016AllV3_DATA' ] tempconfig.JobType.inputFiles = [ json_25ns, 'Summer16_23Sep2016AllV3_DATA.db' ] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns elif key.find('Run2016D-23Sep2016') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=0', 'GlobalTag=80X_dataRun2_2016SeptRepro_v3', 'specialFix=JEC BADMUON', 'jecDBname=Summer16_23Sep2016AllV3_DATA' ] tempconfig.JobType.inputFiles = [ json_25ns, 'Summer16_23Sep2016AllV3_DATA.db' ] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns elif key.find('Run2016E-23Sep2016') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=0', 'GlobalTag=80X_dataRun2_2016SeptRepro_v3', 'specialFix=JEC BADMUON', 'jecDBname=Summer16_23Sep2016AllV3_DATA' ] tempconfig.JobType.inputFiles = [ json_25ns, 'Summer16_23Sep2016AllV3_DATA.db' ] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns elif key.find('Run2016F-23Sep2016') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=0', 'GlobalTag=80X_dataRun2_2016SeptRepro_v3', 'specialFix=JEC BADMUON', 'jecDBname=Summer16_23Sep2016AllV3_DATA' ] tempconfig.JobType.inputFiles = [ json_25ns, 'Summer16_23Sep2016AllV3_DATA.db' ] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns elif key.find('Run2016G-23Sep2016') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=0', 'GlobalTag=80X_dataRun2_2016SeptRepro_v3', 'specialFix=JEC BADMUON', 'jecDBname=Summer16_23Sep2016AllV3_DATA' ] tempconfig.JobType.inputFiles = [ json_25ns, 'Summer16_23Sep2016AllV3_DATA.db' ] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns elif key.find('Run2016H-PromptReco') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=0', 'GlobalTag=80X_dataRun2_Prompt_v14', 'specialFix=JEC BADMUON', 'jecDBname=Summer16_23Sep2016AllV3_DATA' ] tempconfig.JobType.inputFiles = [ json_25ns, 'Summer16_23Sep2016AllV3_DATA.db' ] tempconfig.Data.splitting = 'LumiBased' tempconfig.Data.lumiMask = json_25ns else: pass else: if key.find('FastSim') != -1: tempconfig.JobType.pyCfgParams = [ 'mcInfo=1', 'GlobalTag=80X_mcRun2_asymptotic_2016_miniAODv2_v0', 'specialFix=JEC BADMUON', 'jecDBname=Spring16_25nsFastSimMC_V1', 'fastsim=1' ] tempconfig.JobType.inputFiles = ['Spring16_25nsFastSimMC_V1.db'] tempconfig.Data.splitting = 'FileBased' else: tempconfig.JobType.pyCfgParams = [ 'mcInfo=1', 'GlobalTag=80X_mcRun2_asymptotic_2016_TrancheIV_v6', 'specialFix=JEC BADMUON', 'jecDBname=Summer16_23Sep2016V3_MC' ] tempconfig.JobType.inputFiles = ['Summer16_23Sep2016V3_MC.db'] tempconfig.Data.splitting = 'FileBased' tempconfig.Data.inputDataset = value[1].strip() tempconfig.Data.unitsPerJob = value[2] if value[0] and len(value) > 3: tempconfig.Data.lumiMask = value[3] # Submitting jobs if doTest: saveConfigurationFile(tempconfig, workArea + "/test/" + key + "_test_cfg.py") tasklist["crab_" + key] = key else: results = crabCommand('submit', config=tempconfig) tasklist[results['uniquerequestname']] = key del tempconfig