def prepareObjects():
#Creting the input parameters
requestname = '130911_093053_mmascher_crab_tmp2'
task = {'tm_taskname':requestname} #just using some locations for this dataset
locations = {'/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9':[u'ganymede.hep.kbfi.ee', u'srm.ihepa.ufl.edu', u'storm-se-01.ba.infn.it']}
#and just using some files
datasetfiles = {}
datasetfiles['/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/626F3DC8-1EAE-E111-A64C-0025902CB6B0.root'] = {'NumberOfEvents':1700L, 'BlockName':'/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9',
'Lumis':{1L:[670994L, 670986L, 670958L, 671001L, 670995L, 671018L, 670988L, 671009L, 671038L, 671028L, 670942L, 670960L, 670948L, 670987L,
670991L, 671006L, 671012L, 671014L, 671040L, 670998L, 670983L, 670999L, 671027L, 671052L, 670984L, 671003L, 670996L, 671031L,
671007L, 671011L, 671022L, 670985L, 670962L, 671021L],
2L:[1L, 2L, 3L, 8L, 9L],
4L:[5L, 6L, 7L, 1L, 2L, 100L]},
'Parents':['/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/C880E78D-1EAE-E111-897B-0025902CB6AE.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8C7BB8D5-1EAE-E111-B269-0025901AD638.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/8215258E-1EAE-E111-8A4E-0025904B11D4.root'],
'Checksums':{'Checksum':'922282544', 'Adler32':'39f2938b', 'Md5':'NOTSET'}, 'Size':975355735L}
datasetfiles['/store/mc/HC/GenericTTbar/GEN-SIM-RECO/CMSSW_5_3_1_START53_V5-v1/0011/7A5634E8-03AE-E111-9363-5404A63886EC.root'] = {'NumberOfEvents':1700L, 'BlockName':'/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO#289639b0-146f-411b-a220-fcdff0a13cd9',
'Lumis':{1L:[670165L, 670174L, 670199L, 670081L, 670192L, 670170L, 670029L, 670093L, 670173L, 670164L, 670146L, 670079L, 670200L, 670065L,
670062L, 670216L, 670202L, 670150L, 670176L, 670207L, 670087L, 670179L, 670191L, 670185L, 670160L, 670182L, 670177L, 670203L, 670154L,
670187L, 670138L, 670162L, 670066L, 670085L],
2L:[4L, 5L, 20L, 21L, 22L],
3L:[11L, 12L, 13L]},
'Parents':['/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/BAC6F21E-04AE-E111-A591-003048D3C90E.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/781547E6-03AE-E111-B1C0-BCAEC5329717.root',
'/store/mc/HC/GenericTTbar/GEN-SIM-RAW/CMSSW_5_3_1_START53_V5-v1/0011/38694250-03AE-E111-B618-003048D37580.root'],
'Checksums':{'Checksum':'2628876232', 'Adler32':'21d69fc7', 'Md5':'NOTSET'}, 'Size':974920274L}
#Instantiate DataDiscovery
config = Configuration()
config.section_("MyProxy")
config.MyProxy.serverhostcert, config.MyProxy.serverhostkey = Requests().getKeyCert()
obj = DataDiscovery(config, None, None)
return obj, task, requestname, datasetfiles, locations
def testInit(self):
wrongconfig = Configuration()
wrongconfig.section_("BossAir")
try:
mp = MockPlugin(wrongconfig)
except BossAirPluginException:
# The config does not contain MockPlugin section
pass
else:
fail("Expected exception")
wrongconfig.BossAir.section_("MockPlugin")
try:
mp = MockPlugin(wrongconfig)
except BossAirPluginException:
# The config does not contain fakeReport parameter
pass
else:
fail("Expected exception")
wrongconfig.BossAir.MockPlugin.fakeReport = "asdf"
try:
mp = MockPlugin(wrongconfig)
except BossAirPluginException:
# The fakeReport does not exist
pass
else:
fail("Expected exception")
mp = MockPlugin(config)
def configure(self, configfile, service):
cfg = loadConfigurationFile(configfile)
wconfig = cfg.section_("Webtools")
app = wconfig.application
appconfig = cfg.section_(app)
dbsconfig = getattr(appconfig.views.active, service)
databasecore = cfg.CoreDatabase
# Eitehr we change formatter
# OR change the 'Accept' type to application/json (which we don't know how to do at thi moment)
dbsconfig.formatter.object="WMCore.WebTools.RESTFormatter"
config = Configuration()
config.section_("CoreDatabase")
config.CoreDatabase = databasecore
config.component_('DBS')
config.DBS.application = app
config.DBS.model = dbsconfig.model
config.DBS.formatter = dbsconfig.formatter
config.DBS.version = dbsconfig.version
config.DBS.default_expires = 300
# DBS uses owner name, directly from app section at the moment (does not pick it from CoreDatabse)
config.DBS.dbowner = databasecore.dbowner
# Add the CoreDatabase section to DBS
config.DBS.database = config.CoreDatabase
return config
def testLumiSubmit(self):
"""
Test submission with the lumiMask parameter"
"""
sections = ["General", "User", "Data", "Site" , "JobType"]
lumiConf = Configuration()
for sec in sections:
lumiConf.section_(sec)
lumiConf.General.serverUrl = "localhost:8518"
lumiConf.JobType.externalPluginFile = os.path.join( os.path.dirname(__file__), "TestPlugin.py")
lumiConf.Site.storageSite = 'T2_XXX'
lumiInput = os.path.join( os.path.dirname(__file__), "../../../data/lumiInput.json")
lumiConf.Data.splitting = 'LumiBased'
lumiConf.Data.lumiMask = 'lumiInput.json'
sub = submit(self.logger, self.maplistopt + ["-c", lumiConf,
"-p", "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=mmascher/CN=720897/CN=Marco Mascheroni",
"-s", "127.0.0.1:8518"])
res = sub()
expRes = CommandResult(0, None)
self.assertEquals(res, expRes)
def testB(self):
"""add settings"""
config = Configuration()
config.section_("Section1")
section1 = getattr(config, "Section1", None)
self.failUnless(section1 != None)
config.section_("Section2")
section2 = getattr(config, "Section2", None)
self.failUnless(section2 != None)
self.assertRaises(AttributeError, getattr, config, "Section3")
# basic types
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
self.assertEqual(config.Section1.Parameter1, True)
self.assertEqual(config.Section1.Parameter2, "string")
self.assertEqual(config.Section1.Parameter3, 123)
self.assertEqual(config.Section1.Parameter4, 123.456)
# dictionary format:
try:
section1Dict = config.Section1.dictionary_()
except Exception, ex:
msg = "Error converting section to dictionary:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
def testAgentConfigurationRetrieving(self):
"""
Test that getting some agent details (config values from config.Agent
section) will be correctly propagated into Alert instances.
Alert instance is obtained via API.getPredefinedAlert factory.
"""
d = dict(Additional = "detail")
# instantiate just plain Alert, no configuration to take
# into account at this point
a = Alert(**d)
self.assertEqual(a["HostName"], None)
self.assertEqual(a["Contact"], None)
self.assertEqual(a["TeamName"], None)
self.assertEqual(a["AgentName"], None)
self.assertEqual(a["Additional"], "detail")
# instantiate via factory which reads configuration instance
config = Configuration()
config.section_("Agent")
config.Agent.hostName = "some1"
config.Agent.contact = "some2"
config.Agent.teamName = "some3"
config.Agent.agentName = "some4"
a = alertAPI.getPredefinedAlert(**d)
self.assertEqual(a["HostName"], "some1")
self.assertEqual(a["Contact"], "some2")
self.assertEqual(a["TeamName"], "some3")
self.assertEqual(a["AgentName"], "some4")
self.assertEqual(a["Additional"], "detail")
class AlertProcessorTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.Agent.Database',
"WMCore.ResourceControl"],
useDefault = False)
self.testDir = self.testInit.generateWorkDir()
self.config = Configuration()
self.config.section_("Agent")
self.config.Agent.useMsgService = False
self.config.Agent.useTrigger = False
self.config.component_("AlertProcessor")
self.config.AlertProcessor.componentDir = self.testDir
self.config.AlertProcessor.address = "tcp://127.0.0.1:5557"
self.config.AlertProcessor.controlAddr = "tcp://127.0.0.1:5559"
self.config.section_("CoreDatabase")
self.config.CoreDatabase.socket = os.environ.get("DBSOCK")
self.config.CoreDatabase.connectUrl = os.environ.get("DATABASE")
self.config.AlertProcessor.section_("critical")
self.config.AlertProcessor.section_("soft")
self.config.AlertProcessor.critical.level = 5
self.config.AlertProcessor.soft.level = 0
self.config.AlertProcessor.soft.bufferSize = 3
self.config.AlertProcessor.critical.section_("sinks")
self.config.AlertProcessor.soft.section_("sinks")
def tearDown(self):
self.testInit.clearDatabase()
self.testInit.delWorkDir()
def testAlertProcessorBasic(self):
alertProcessor = AlertProcessor(self.config)
try:
# alertProcessor.startComponent() causes the flow to stop, Harness.py
# the method just calls prepareToStart() and waits for ever
# alertProcessor.startDaemon() no good for this either ... puts everything
# on background
alertProcessor.prepareToStart()
except Exception, ex:
print ex
self.fail(str(ex))
logging.debug("AlertProcessor and its sub-components should be running now ...")
logging.debug("Going to stop the component ...")
# stop via component method
try:
alertProcessor.stopAlertProcessor()
except Exception, ex:
print ex
self.fail(str(ex))
def crabConfig():
config = Configuration()
# -- General
config.section_('General')
config.General.transferOutputs = True
config.General.requestName = ''
config.General.workArea = 'crab_projects'
# -- JobType
config.section_('JobType')
config.JobType.psetName = 'ntuplizer.py'
config.JobType.pluginName = 'Analysis'
config.JobType.outputFiles = ['ntuple.root']
# -- Data
config.section_('Data')
config.Data.publication = False
config.Data.unitsPerJob = 1
config.Data.totalUnits = -1
config.Data.splitting = 'FileBased'
config.Data.outLFNDirBase= ''
# -- User
config.section_('User')
config.User.voGroup = 'dcms'
# -- Site
config.section_('Site')
config.Site.storageSite = 'T2_DE_DESY'
return config
def testG_testStaticReferenceToConfigurationInstance(self):
"""
test Configuration.getInstance() which returns reference
to the Configuration object instance.
"""
config = Configuration()
instance = Configuration.getInstance()
self.assertFalse(hasattr(instance, "testsection"))
config.section_("testsection")
self.assertTrue(hasattr(instance, "testsection"))
config.testsection.var = 10
self.assertEquals(instance.testsection.var, 10)
def testInit(self):
wrongconfig = Configuration()
wrongconfig.section_('BossAir')
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
wrongconfig.BossAir.section_('MockPlugin')
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
#The config does not contain fakeReport parameter
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
#The fakeReport does not exist
wrongconfig.BossAir.MockPlugin.fakeReport = 'asdf'
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
def _emulatorCofig(phedex, dbs, siteDB, requestMgr, configFile):
config = Configuration()
config.section_("Emulator")
config.Emulator.PhEDEx = phedex
config.Emulator.DBSReader = dbs
config.Emulator.RequestMgr = requestMgr
config.Emulator.SiteDB = siteDB
saveConfigurationFile(config, configFile)
print "create config file:%s, PhEDEx: %s, DBS: %s, RequestManager: %s, SiteDB %s with flag" \
% (configFile, phedex, dbs, siteDB, requestMgr)
def _wmAgentConfig(configFile):
config = Configuration()
config.section_("General")
config.General.logdb_name = "unittest_logdb"
config.General.central_logdb_url = "http://localhost/central_logdb"
config.General.ReqMgr2ServiceURL = "http://localhost/reqmgr2"
config.section_("JobStateMachine")
# Waring setting couchDB to None will cause the ERROR:
# but that should be ignored, if you want to test couchDB
# set the real couchDB information here
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = os.getenv("COUCHDB")
config.JobStateMachine.jobSummaryDBName = "wmagent_summary_test"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.section_("Agent")
# User specific parameter
config.Agent.hostName = "cmssrv52.fnal.gov"
# User specific parameter
config.Agent.contact = "*****@*****.**"
# User specific parameter
config.Agent.teamName = "DMWM"
# User specific parameter
config.Agent.agentName = "WMAgentCommissioning"
config.Agent.useMsgService = False
config.Agent.useTrigger = False
# BossAir setup
config.section_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'SimpleCondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
saveConfigurationFile(config, configFile)
def make_crab_config(self):
if self.misc["crab_config"] is not None:
self.do_log("crab config already made, not remaking")
return
config = Configuration()
config.section_('General')
config.General.workArea = self.misc["pfx_crab"] # all crab output goes into crab/
config.General.transferOutputs = True
config.General.transferLogs = True
config.General.requestName = self.sample["crab"]["requestname"]
config.section_('JobType')
config.JobType.inputFiles = params.jecs
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = "%s/%s_cfg.py" % (self.misc["pfx_pset"], self.sample["shortname"])
config.section_('Data')
config.Data.allowNonValidInputDataset = True
config.Data.publication = False
config.Data.inputDataset = self.sample["dataset"]
config.Data.unitsPerJob = 1
config.Data.ignoreLocality = True
config.Data.splitting = 'FileBased'
config.Data.inputDBS = "phys03" if self.sample["dataset"].endswith("/USER") else "global"
config.section_('User')
config.section_('Site')
config.Site.storageSite = 'T2_US_UCSD'
config.Site.whitelist = ['T2_US_*']
self.misc["crab_config"] = config
def testD(self):
"""test documentation"""
config = Configuration()
config.section_("Section1")
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
config.Section1.Parameter5 = {"test1" : "test2", "test3" : 123}
config.Section1.document_("""This is Section1""")
config.Section1.document_("""This is Section1.Parameter1""",
"Parameter1")
config.Section1.document_("""This is Section1.Parameter2""",
"Parameter2")
config.Section1.document_("""This is Section1.Parameter3\n with multiline comments""",
"Parameter3")
try:
config.Section1.documentedString_()
except Exception as ex:
msg = "Error calling ConfigSection.documentedString_:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
try:
config.Section1.commentedString_()
except Exception as ex:
msg = "Error calling ConfigSection.commentedString_:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
try:
config.documentedString_()
except Exception as ex:
msg = "Error calling Configuration.documentedString_:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
try:
config.commentedString_()
except Exception as ex:
msg = "Error calling Configuration.commentedString_:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
def setup_test_server(module_name, class_name, app_name = None, authz_key_file=None, port=8888):
"""Helper function to set up a :class:`~.RESTMain` server from given
module and class. Creates a fake server configuration and instantiates
the server application from it.
:arg str module_name: module from which to import test class.
:arg str class_type: name of the server test class.
:arg str app_name: optional test application name, 'test' by default.
:returns: tuple with the server object and authz hmac signing key."""
if authz_key_file:
test_authz_key = authz_key_file
else:
test_authz_key = fake_authz_key_file()
cfg = Configuration()
main = cfg.section_('main')
main.application = app_name or 'test'
main.silent = True
main.index = 'top'
main.authz_defaults = { 'role': None, 'group': None, 'site': None }
main.section_('tools').section_('cms_auth').key_file = test_authz_key.name
app = cfg.section_(app_name or 'test')
app.admin = '*****@*****.**'
app.description = app.title = 'Test'
views = cfg.section_('views')
top = views.section_('top')
top.object = module_name + "." + class_name
server = RESTMain(cfg, os.getcwd())
server.validate_config()
server.setup_server()
server.install_application()
cherrypy.config.update({'server.socket_port': port})
cherrypy.config.update({'server.socket_host': '127.0.0.1'})
cherrypy.config.update({'request.show_tracebacks': True})
cherrypy.config.update({'environment': 'test_suite'})
for app in cherrypy.tree.apps.values():
if '/' in app.config:
app.config["/"]["request.show_tracebacks"] = True
return server, test_authz_key
def setup_dummy_server(module_name, class_name, app_name=None, authz_key_file=None, port=8888):
"""Helper function to set up a :class:`~.RESTMain` server from given
module and class. Creates a fake server configuration and instantiates
the server application from it.
:arg str module_name: module from which to import test class.
:arg str class_type: name of the server test class.
:arg str app_name: optional test application name, 'test' by default.
:returns: tuple with the server object and authz hmac signing key."""
if authz_key_file:
test_authz_key = authz_key_file
else:
test_authz_key = fake_authz_key_file()
cfg = Configuration()
main = cfg.section_("main")
main.application = app_name or "test"
main.silent = True
main.index = "top"
main.authz_defaults = {"role": None, "group": None, "site": None}
main.section_("tools").section_("cms_auth").key_file = test_authz_key.name
app = cfg.section_(app_name or "test")
app.admin = "*****@*****.**"
app.description = app.title = "Test"
views = cfg.section_("views")
top = views.section_("top")
top.object = module_name + "." + class_name
server = RESTMain(cfg, os.getcwd())
server.validate_config()
server.setup_server()
server.install_application()
cherrypy.config.update({"server.socket_port": port})
cherrypy.config.update({"server.socket_host": "127.0.0.1"})
cherrypy.config.update({"request.show_tracebacks": True})
cherrypy.config.update({"environment": "test_suite"})
for app in cherrypy.tree.apps.values():
if "/" in app.config:
app.config["/"]["request.show_tracebacks"] = True
return server, test_authz_key
def config():
"""
Return a Configuration object containing all the sections that CRAB recognizes.
"""
config = Configuration()
config.section_("General")
config.section_("JobType")
config.section_("Data")
config.section_("Site")
config.section_("User")
config.section_("Debug")
return config
def workqueueConfig(couchdb = localCouchDb):
"""
Returns an usable workqueue config
"""
config = Configuration()
config.section_("Agent")
config.Agent.hostName = gethostname()
config.component_("WorkQueueManager")
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
config.WorkQueueManager.couchurl = couchdb
config.WorkQueueManager.dbname = workqueueDBName
config.WorkQueueManager.wmstatDBName = wmstatsDBName
config.WorkQueueManager.inboxDatabase = workqueueInboxDbName
config.WorkQueueManager.level = "GlobalQueue"
config.WorkQueueManager.queueParams = {'WMStatsCouchUrl': "%s/%s" % (couchdb, wmstatsDBName)}
config.WorkQueueManager.queueParams['QueueURL'] = '%s/%s' % (couchdb, workqueueDBName)
config.WorkQueueManager.reqMgrConfig = {}
config.WorkQueueManager.reqMgrConfig['endpoint'] = reqMgrEndpoint
return config
def testCallableConfigParams(self):
"""ctor"""
def f():
return True
config = Configuration()
config.section_("SectionF")
#creating field for the following test
config.SectionF.aFunction = ''
#Cannot set a function for plain Configuration objects
#config.SectionF.__setattr__('aFunction', f)
self.assertRaises(RuntimeError, config.SectionF.__setattr__, config.SectionF.aFunction, f)
config = ConfigurationEx()
config.section_("SectionF")
#No failures with configurationEx
config.SectionF.aFunction = f
#However ConfigurationEx instances cannot be saved
self.assertRaises(RuntimeError, saveConfigurationFile, config, self.functionSave)
def _wmAgentConfig(configFile):
config = Configuration()
config.section_("JobStateMachine")
#Waring setting couchDB to None will cause the ERROR:
# but that should be ignored, if you want to test couchDB
# set the real couchDB information here
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = os.getenv("COUCHDB")
config.section_("Agent")
# User specific parameter
config.Agent.hostName = "cmssrv52.fnal.gov"
# User specific parameter
config.Agent.contact = "*****@*****.**"
# User specific parameter
config.Agent.teamName = "DMWM"
# User specific parameter
config.Agent.agentName = "WMAgentCommissioning"
config.Agent.useMsgService = False
config.Agent.useTrigger = False
# BossAir setup
config.section_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
saveConfigurationFile(config, configFile)
def _wmAgentConfig(configFile):
config = Configuration()
config.section_("JobStateMachine")
#Waring setting couchDB to None will cause the ERROR:
# but that should be ignored, if you want to test couchDB
# set the real couchDB information here
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = os.getenv("COUCHDB")
config.JobStateMachine.jobSummaryDBName = "wmagent_summary_test"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.section_("Agent")
# User specific parameter
config.Agent.hostName = "cmssrv52.fnal.gov"
# User specific parameter
config.Agent.contact = "*****@*****.**"
# User specific parameter
config.Agent.teamName = "DMWM"
# User specific parameter
config.Agent.agentName = "WMAgentCommissioning"
config.Agent.useMsgService = False
config.Agent.useTrigger = False
# BossAir setup
config.section_("BossAir")
config.BossAir.pluginNames = ['TestPlugin', 'CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
#TaskArchive setup (JobSubmitter needs this)
config.component_("TaskArchiver")
config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"
saveConfigurationFile(config, configFile)
def testSubmit(self):
#Delete workdir
if os.path.isdir("crab_" + self.TestConfig.config.General.requestName):
shutil.rmtree("crab_" + self.TestConfig.config.General.requestName)
#2) The config file is not found
sub = submit(self.logger, self.maplistopt + ["-c", "asdf", "-p", "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=mmascher/CN=720897/CN=Marco Mascheroni", \
"-s", "127.0.0.1:8518"])
res = sub()
self.assertEqual(res[0], 1)
#3) Is the client chacking the configurations?
#If a mandatory section is not there => fail!
sections = ["General", "User", "Data", "Site" , "JobType"]#mandatory sections
emptyConf = Configuration()
for sec in sections:
sub = submit(self.logger, self.maplistopt + ["-c", "asdf", "-p", "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=mmascher/CN=720897/CN=Marco Mascheroni", \
"-s", "127.0.0.1:8518"])
res = sub()
self.assertEqual(res[0], 1)
emptyConf.section_(sec)
emptyConf.General.serverUrl = "localhost:8518"
emptyConf.General.requestName = 'TestAnalysisSubmit'
emptyConf.JobType.externalPluginFile = os.path.join( os.path.dirname(__file__), "TestPlugin.py")
emptyConf.Site.storageSite = 'T2_XXX'
expRes = CommandResult(0, None)
sub = submit(self.logger, self.maplistopt + ["-c", emptyConf, "-p", "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=mmascher/CN=720897/CN=Marco Mascheroni", \
"-s", "127.0.0.1:8518"])
res = sub()
self.assertEquals(res, expRes)
shutil.rmtree("./crab_TestAnalysisSubmit")
emptyConf.Data.runWhitelist = '1,3,9-13'
emptyConf.Data.runBlacklist = '1,3,9-13'
expRes = CommandResult(0, None)
sub = submit(self.logger, self.maplistopt + ["-c", emptyConf, "-p", "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=mmascher/CN=720897/CN=Marco Mascheroni", \
"-s", "127.0.0.1:8518"])
res = sub()
self.assertEquals(res, expRes)
def createTier0Config():
"""
_createTier0Config_
Create a configuration object to hold the Tier0 configuration. Currently,
the configuration has two sections: Streams and Global.
"""
tier0Config = Configuration()
tier0Config.section_("Streams")
tier0Config.section_("Datasets")
tier0Config.section_("Global")
tier0Config.Global.ScramArches = {}
tier0Config.Global.Backfill = None
tier0Config.Global.ProcessingSite = "T2_CH_CERN_AI"
tier0Config.Global.BulkInjectNode = "T0_CH_CERN_Disk"
tier0Config.Global.ExpressInjectNode = "T0_CH_CERN_Disk"
tier0Config.Global.ExpressSubscribeNode = None
tier0Config.Global.DQMDataTier = "DQMIO"
tier0Config.Global.BaseRequestPriority = 150000
return tier0Config
def createTier0Config():
"""
_createTier0Config_
Create a configuration object to hold the Tier0 configuration. Currently,
the configuration has two sections: Streams and Global.
"""
tier0Config = Configuration()
tier0Config.section_("Streams")
tier0Config.section_("Datasets")
tier0Config.section_("Global")
tier0Config.Global.InjectRuns = None
tier0Config.Global.InjectMinRun = None
tier0Config.Global.InjectMaxRun = None
tier0Config.Global.ScramArches = {}
tier0Config.Global.Backfill = None
tier0Config.Global.ProcessingSite = "T0_CH_CERN"
tier0Config.Global.StreamerPNN = "T0_CH_CERN_Disk"
tier0Config.Global.DQMDataTier = "DQMIO"
tier0Config.Global.BaseRequestPriority = 150000
tier0Config.Global.EnableUniqueWorkflowName = False
return tier0Config
def testE(self):
"""test save/load """
testValues = [
"string", 123, 123.456,
["list", 789, 10.1 ],
{ "dict1" : "value", "dict2" : 10.0 }
]
config = Configuration()
for x in range(0, 5):
config.section_("Section%s" % x)
config.component_("Component%s" % x)
sect = getattr(config, "Section%s" % x)
comp = getattr(config, "Component%s" % x)
sect.document_("This is Section%s" % x)
comp.document_("This is Component%s" % x)
for i in range(0, 5):
setattr(comp, "Parameter%s" % i, testValues[i])
setattr(sect, "Parameter%s" % i, testValues[i])
comp.document_("This is Parameter%s" % i,
"Parameter%s" %i)
sect.document_("This is Parameter%s" %i,
"Parameter%s" %i)
stringSave = str(config)
documentSave = config.documentedString_()
commentSave = config.commentedString_()
saveConfigurationFile(config, self.normalSave)
saveConfigurationFile(config, self.docSave, document = True)
saveConfigurationFile(config, self.commentSave, comment = True)
plainConfig = loadConfigurationFile(self.normalSave)
docConfig = loadConfigurationFile(self.docSave)
commentConfig = loadConfigurationFile(self.commentSave)
def getBasicConfig():
config = Configuration()
config.section_("General")
config.section_("JobType")
config.section_("Data")
config.section_("Site")
config.section_("User")
return config
def main():
options = getOptions()
from WMCore.Configuration import Configuration
config = Configuration()
from CRABAPI.RawCommand import crabCommand
from httplib import HTTPException
# We want to put all the CRAB project directories from the tasks we submit here into one common directory.
# That's why we need to set this parameter (here or above in the configuration file, it does not matter, we will not overwrite it).
config.section_("General")
config.General.workArea = options.dir
config.General.transferLogs = True
config.section_("JobType")
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = options.config
config.section_("Data")
config.Data.inputDataset = None
config.Data.splitting = 'FileBased'
config.Data.unitsPerJob = 20
config.Data.ignoreLocality = True
config.Data.publication = False
config.Data.inputDBS = 'phys03'
config.Data.outputDatasetTag = 'AK8_200'
config.section_("Site")
config.Site.storageSite = 'T3_US_FNALLPC'
print 'Using config ' + options.config
print 'Writing to directory ' + options.dir
def submit(config):
try:
crabCommand('submit', config = config)
except HTTPException, hte:
print 'Cannot execute commend'
print hte.headers
def prepare_job_config(self, job):
""" Generates a CRAB config object from the Ganga job configuration. """
from WMCore.Configuration import Configuration
job_config = Configuration()
for section in job.backend.CRABConfig._schema.datadict.keys():
section_config = getattr(job.backend.CRABConfig, section)
ganga_section_config = Config.getConfig('CRABConfig_%s' % section)
task_section_config = job_config.section_(section)
for parameter_name, parameter_type in section_config._schema.allItems():
parameter_value = getattr(section_config, parameter_name)
if parameter_value not in (None, [None]):
# CRAB Config doesn't like Ganga sequence type instead of Lists
if parameter_type._meta['sequence']:
parameter_value = list(parameter_value)
task_section_config.__setattr__(parameter_name, parameter_value)
# Updating configuration in case of Ganga inline options specified
ganga_option = ganga_section_config[parameter_name]
if ganga_option:
# CRAB Config doesn't like Ganga sequence (or tuples) type instead of Lists
# Passing sequance with Ganga inline options makes it a tuple.
if parameter_type._meta['sequence']:
# Ugly but we need this because otherwise tuple of 1 element with a string would
# be transformed in a list of chars ( ('ab') --> ['a', 'b'] )
import json
ganga_option = json.loads(json.dumps(ganga_option))
if type(ganga_option) != list:
ganga_option = [ganga_option]
# loads makes strings 'utf' type, CRAB3 Server wants 'str' type
ganga_option = map(lambda x: str(x), ganga_option)
task_section_config.__setattr__(parameter_name, ganga_option)
# Some internal configuration
job_config.General.workArea = job.outputdir
return job_config
def getConfig(self):
"""
_getConfig_
Get a test configuration for
the JobUpdater tests
"""
config = Configuration()
config.section_('Agent')
config.Agent.agentName = 'testAgent'
config.section_('CoreDatabase')
config.CoreDatabase.connectUrl = os.environ['DATABASE']
config.CoreDatabase.socket = os.getenv('DBSOCK')
# JobTracker
config.component_('JobUpdater')
config.JobUpdater.reqMgrUrl = 'https://cmsweb-dev.cern.ch/reqmgr/reqMgr'
# JobStateMachine
config.section_('JobStateMachine')
config.JobStateMachine.couchDBName = 'bogus'
# BossAir
config.section_('BossAir')
config.BossAir.pluginNames = ['MockPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.BossAir.multicoreTaskTypes = ['MultiProcessing', 'MultiProduction']
config.BossAir.nCondorProcesses = 1
config.BossAir.section_('MockPlugin')
config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
'WMComponent_t/JobAccountant_t/fwjrs',
'MergedSkimSuccess.pkl')
# WorkQueue
config.component_('WorkQueueManager')
config.WorkQueueManager.couchurl = os.environ['COUCHURL']
config.WorkQueueManager.dbname = 'workqueue_t'
config.WorkQueueManager.inboxDatabase = 'workqueue_inbox_t'
return config
def createTier0Config():
"""
_createTier0Config_
Create a configuration object to hold the Tier0 configuration. Currently,
the configuration has two sections: Streams and Global.
"""
tier0Config = Configuration()
tier0Config.section_("Streams")
tier0Config.section_("Datasets")
tier0Config.section_("Global")
return tier0Config
def main():
options = getOptions()
print "IsData? ", options.isData
datasetsFile = open(options.datasets)
jobsLines = datasetsFile.readlines()
jobs = []
from WMCore.Configuration import Configuration
config = Configuration()
from CRABAPI.RawCommand import crabCommand
from httplib import HTTPException
config.section_("General")
config.General.workArea = options.dir
config.section_("JobType")
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = options.config
config.JobType.allowUndistributedCMSSW = True
# config.JobType.pyCfgParams = ['isData=' + str(options.isData), 'changeJECs=True', 'channel=wzjets']
config.JobType.pyCfgParams = [
'isData=' + str(options.isData), 'changeJECs=True'
]
#config.JobType.inputFiles = ['Fall15_25nsV2_DATA.db', 'Fall15_25nsV2_MC.db']
#config.JobType.inputFiles = ["Fall15_25nsV2_MC_L1FastJet_AK4PFchs.txt", "Fall15_25nsV2_MC_L1RC_AK4PFchs.txt","Fall15_25nsV2_MC_L2Relative_AK4PFchs.txt", "Fall15_25nsV2_MC_L3Absolute_AK4PFchs.txt","Fall15_25nsV2_MC_L2L3Residual_AK4PFchs.txt", "Fall15_25nsV2_DATA_L1FastJet_AK4PFchs.txt","Fall15_25nsV2_DATA_L1RC_AK4PFchs.txt","Fall15_25nsV2_DATA_L2Relative_AK4PFchs.txt","Fall15_25nsV2_DATA_L3Absolute_AK4PFchs.txt", "Fall15_25nsV2_DATA_L2L3Residual_AK4PFchs.txt"]
config.JobType.inputFiles = [
'Spring16_25nsV10_MC_PtResolution_AK8PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_L1FastJet_AK4PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_L1FastJet_AK8PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_L1RC_AK4PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_L1RC_AK8PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_L2L3Residual_AK4PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_L2L3Residual_AK8PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_L2Relative_AK4PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_L2Relative_AK8PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_L2Residual_AK4PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_L3Absolute_AK4PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_L3Absolute_AK8PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_UncertaintySources_AK4PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_UncertaintySources_AK8PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_Uncertainty_AK4PFchs.txt',
'Summer16_23Sep2016BCDV4_DATA_Uncertainty_AK8PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_L1FastJet_AK4PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_L1FastJet_AK8PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_L1RC_AK4PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_L1RC_AK8PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_L2L3Residual_AK4PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_L2L3Residual_AK8PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_L2Relative_AK4PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_L2Relative_AK8PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_L2Residual_AK4PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_L3Absolute_AK4PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_L3Absolute_AK8PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_UncertaintySources_AK4PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_UncertaintySources_AK8PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_Uncertainty_AK4PFchs.txt',
'Summer16_23Sep2016EFV4_DATA_Uncertainty_AK8PFchs.txt',
'Summer16_23Sep2016GV4_DATA_L1FastJet_AK4PFchs.txt',
'Summer16_23Sep2016GV4_DATA_L1FastJet_AK8PFchs.txt',
'Summer16_23Sep2016GV4_DATA_L1RC_AK4PFchs.txt',
'Summer16_23Sep2016GV4_DATA_L1RC_AK8PFchs.txt',
'Summer16_23Sep2016GV4_DATA_L2L3Residual_AK4PFchs.txt',
'Summer16_23Sep2016GV4_DATA_L2L3Residual_AK8PFchs.txt',
'Summer16_23Sep2016GV4_DATA_L2Relative_AK4PFchs.txt',
'Summer16_23Sep2016GV4_DATA_L2Relative_AK8PFchs.txt',
'Summer16_23Sep2016GV4_DATA_L2Residual_AK4PFchs.txt',
'Summer16_23Sep2016GV4_DATA_L3Absolute_AK4PFchs.txt',
'Summer16_23Sep2016GV4_DATA_L3Absolute_AK8PFchs.txt',
'Summer16_23Sep2016GV4_DATA_UncertaintySources_AK4PFchs.txt',
'Summer16_23Sep2016GV4_DATA_UncertaintySources_AK8PFchs.txt',
'Summer16_23Sep2016GV4_DATA_Uncertainty_AK4PFchs.txt',
'Summer16_23Sep2016GV4_DATA_Uncertainty_AK8PFchs.txt',
'Summer16_23Sep2016HV4_DATA_L1FastJet_AK4PFchs.txt',
'Summer16_23Sep2016HV4_DATA_L1FastJet_AK8PFchs.txt',
'Summer16_23Sep2016HV4_DATA_L1RC_AK4PFchs.txt',
'Summer16_23Sep2016HV4_DATA_L1RC_AK8PFchs.txt',
'Summer16_23Sep2016HV4_DATA_L2L3Residual_AK4PFchs.txt',
'Summer16_23Sep2016HV4_DATA_L2L3Residual_AK8PFchs.txt',
'Summer16_23Sep2016HV4_DATA_L2Relative_AK4PFchs.txt',
'Summer16_23Sep2016HV4_DATA_L2Relative_AK8PFchs.txt',
'Summer16_23Sep2016HV4_DATA_L2Residual_AK4PFchs.txt',
'Summer16_23Sep2016HV4_DATA_L3Absolute_AK4PFchs.txt',
'Summer16_23Sep2016HV4_DATA_L3Absolute_AK8PFchs.txt',
'Summer16_23Sep2016HV4_DATA_UncertaintySources_AK4PFchs.txt',
'Summer16_23Sep2016HV4_DATA_UncertaintySources_AK8PFchs.txt',
'Summer16_23Sep2016HV4_DATA_Uncertainty_AK4PFchs.txt',
'Summer16_23Sep2016HV4_DATA_Uncertainty_AK8PFchs.txt',
'Summer16_23Sep2016V4_MC_L1FastJet_AK4PFchs.txt',
'Summer16_23Sep2016V4_MC_L1FastJet_AK8PFchs.txt',
'Summer16_23Sep2016V4_MC_L1RC_AK4PFchs.txt',
'Summer16_23Sep2016V4_MC_L1RC_AK8PFchs.txt',
'Summer16_23Sep2016V4_MC_L2L3Residual_AK4PFchs.txt',
'Summer16_23Sep2016V4_MC_L2L3Residual_AK8PFchs.txt',
'Summer16_23Sep2016V4_MC_L2Relative_AK4PFchs.txt',
'Summer16_23Sep2016V4_MC_L2Relative_AK8PFchs.txt',
'Summer16_23Sep2016V4_MC_L3Absolute_AK4PFchs.txt',
'Summer16_23Sep2016V4_MC_L3Absolute_AK8PFchs.txt',
'Summer16_23Sep2016V4_MC_Uncertainty_AK4PFchs.txt',
'Summer16_23Sep2016V4_MC_Uncertainty_AK8PFchs.txt',
]
config.section_("Data")
config.Data.ignoreLocality = True
config.Data.inputDataset = None
config.Data.inputDBS = 'phys03'
config.Data.splitting = 'FileBased'
config.Data.unitsPerJob = 1
config.Data.publication = False
config.Data.outLFNDirBase = '/store/user/decosa/Bprime_LHCP17_jer/'
config.section_("Site")
config.Site.storageSite = 'T2_CH_CSCS'
config.Site.whitelist = ['T2_CH_CERN', 'T2_IT_*', 'T2_DE_*', 'T2_CH_*']
#config.Site.blacklist = ['T2_DE_RWTH']
print 'Using config ' + options.config
print 'Writing to directory ' + options.dir
def submit(config):
try:
crabCommand('submit', config=config)
except HTTPException, hte:
print 'Cannot execute command'
print hte.headers
from WMCore.Configuration import Configuration
import re
import socket
myhost = socket.getfqdn().lower()
conf = Configuration()
main = conf.section_('main')
srv = main.section_('server')
srv.thread_pool = 5
main.application = 'crabserver'
main.port = 8270
main.index = 'data'
main.authz_defaults = { 'role': None, 'group': None, 'site': None }
main.section_('tools').section_('cms_auth').key_file = "%s/auth/wmcore-auth/header-auth-key" % __file__.rsplit('/', 3)[0]
app = conf.section_('crabserver')
app.admin = '*****@*****.**'
app.description = 'CRABServer RESTFull API'
app.title = 'CRABRESTFull'
views = conf.section_('views')
data = views.section_('data')
data.object = 'CRABInterface.RESTBaseAPI.RESTBaseAPI'
data.phedexurl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod/'
data.dbsurl = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
data.defaultBlacklist = ['T0_CH_CERN']
data.serverhostcert = "%s/auth/crabserver/dmwm-service-cert.pem" % __file__.rsplit('/', 3)[0]
data.serverhostkey = "%s/auth/crabserver/dmwm-service-key.pem" % __file__.rsplit('/', 3)[0]
import socket
import time
from WMCore.Configuration import Configuration
from os import path
HOST = socket.gethostname().lower()
BASE_URL = "@@BASE_URL@@"
DBS_INS = "@@DBS_INS@@"
COUCH_URL = "%s/couchdb" % BASE_URL
LOG_DB_URL = "%s/wmstats_logdb" % COUCH_URL
LOG_REPORTER = "wmstats"
ROOTDIR = __file__.rsplit('/', 3)[0]
config = Configuration()
main = config.section_("main")
srv = main.section_("server")
srv.thread_pool = 30
main.application = "wmstatsserver"
main.application_dir = "reqmon"
main.port = 8249 # main application port it listens on
main.index = "ui"
# Defaults to allow any CMS authenticated user. Write APIs should require
# additional roles in SiteDB (i.e. "Admin" role for the "ReqMgr" group)
main.authz_defaults = {"role": None, "group": None, "site": None}
#set default logging (prevent duplicate)
main.log_screen = True
sec = main.section_("tools").section_("cms_auth")
sec.key_file = "%s/auth/wmcore-auth/header-auth-key" % ROOTDIR
def testB(self):
"""add settings"""
config = Configuration()
config.section_("Section1")
section1 = getattr(config, "Section1", None)
self.assertTrue(section1 != None)
config.section_("Section2")
section2 = getattr(config, "Section2", None)
self.assertTrue(section2 != None)
self.assertRaises(AttributeError, getattr, config, "Section3")
# basic types
config.Section1.Parameter1 = True
config.Section1.Parameter2 = "string"
config.Section1.Parameter3 = 123
config.Section1.Parameter4 = 123.456
self.assertEqual(config.Section1.Parameter1, True)
self.assertEqual(config.Section1.Parameter2, "string")
self.assertEqual(config.Section1.Parameter3, 123)
self.assertEqual(config.Section1.Parameter4, 123.456)
# dictionary format:
try:
section1Dict = config.Section1.dictionary_()
except Exception as ex:
msg = "Error converting section to dictionary:\n"
msg += "%s\n" % str(ex)
self.fail(msg)
self.assertTrue( "Parameter1" in section1Dict)
self.assertTrue( "Parameter2" in section1Dict)
self.assertTrue( "Parameter3" in section1Dict)
self.assertTrue( "Parameter4" in section1Dict)
self.assertEqual(section1Dict['Parameter1'],
config.Section1.Parameter1)
self.assertEqual(section1Dict['Parameter2'],
config.Section1.Parameter2)
self.assertEqual(section1Dict['Parameter3'],
config.Section1.Parameter3)
self.assertEqual(section1Dict['Parameter4'],
config.Section1.Parameter4)
# compound types
config.Section2.List = ["string", 123, 123.456, False]
config.Section2.Dictionary = { "string" : "string",
"int" : 123,
"float" : 123.456,
"bool" : False}
config.Section2.Tuple = ("string", 123, 123.456, False)
self.assertEqual(config.Section2.List,
["string", 123, 123.456, False])
self.assertEqual(config.Section2.Tuple,
("string", 123, 123.456, False))
class DummyObject:
pass
# unsupported parameter type
self.assertRaises(
RuntimeError, setattr,
config.Section2, "BadObject", DummyObject())
# unsupported data type in compound type
badList = [ DummyObject(), DummyObject()]
self.assertRaises(
RuntimeError, setattr,
config.Section2, "BadList", badList)
badDict = { "dict" : {}, "list": [DummyObject()], "tuple" : () }
self.assertRaises(
RuntimeError, setattr,
config.Section2, "BadDict", badDict)
goodDict = { "dict" : {}, "list": [], "tuple" : () }
config.Section2.GoodDict = goodDict
class WMDataMining(CherryPyPeriodicTask):
def __init__(self, rest, config):
CherryPyPeriodicTask.__init__(self, config)
def setConcurrentTasks(self, config):
"""
sets the list of functions which
"""
self.concurrentTasks = [{
'func': sayBye,
'duration': config.activeDuration
}, {
'func': Hello(),
'duration': config.archiveDuration
}]
if __name__ == '__main__':
import cherrypy
from WMCore.Configuration import Configuration
config = Configuration()
config.section_("wmmining")
config.wmmining.activeDuration = 5
config.wmmining.archiveDuration = 30
#helloTask = PeriodicWorker(sayHello, config.wmmining)
WMDataMining(None, config.wmmining)
cherrypy.quickstart()
from WMCore.Configuration import Configuration
config = Configuration()
config.section_("General")
config.General.requestName = 'ntReminiaod_RunH_PR_v2'
config.General.transferOutputs = True
config.General.transferLogs = True
config.General.workArea = 'NtMaker_Reminiaod'
config.section_("JobType")
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = '/hcp/data/data02/jelee/Wprime13TeV/Moriond17/CMSSW_8_0_26_patch2/src/MakeNpKNU/makeNpKNU_80X_reMiniAOD_runH.py'
config.JobType.disableAutomaticOutputCollection = True
config.JobType.outputFiles = ['ntNpKNUdata.root', 'histFileData.root']
config.JobType.maxMemoryMB = 2500
config.section_("Data")
config.Data.inputDataset = '/SingleMuon/Run2016H-03Feb2017_ver2-v1/MINIAOD'
config.Data.inputDBS = 'global'
config.Data.splitting = 'LumiBased'
config.Data.unitsPerJob = 10
config.Data.lumiMask = 'https://cms-service-dqm.web.cern.ch/cms-service-dqm/CAF/certification/Collisions16/13TeV/ReReco/Final/Cert_271036-284044_13TeV_23Sep2016ReReco_Collisions16_JSON.txt' #1115
config.Data.runRange = '280919-284044'
config.Data.publication = False
config.Data.outLFNDirBase = '/store/user/jelee/'
config.section_('User')
config.section_("Site")
config.Site.storageSite = 'T2_KR_KNU'
from WMCore.Configuration import Configuration
name = 'idle'
steam_dir = 'xulyu'
config = Configuration()
config.section_("General")
config.General.requestName = 'newM4500_R0-7_off'
config.General.transferLogs = True
config.section_("JobType")
config.JobType.pluginName = 'Analysis'
config.JobType.inputFiles = ['Summer16_07Aug2017_V11_MC_L1FastJet_AK4PFchs.txt','Summer16_07Aug2017_V11_MC_L2Relative_AK4PFchs.txt','Summer16_07Aug2017_V11_MC_L3Absolute_AK4PFchs.txt','Summer16_07Aug2017_V11_MC_L1FastJet_AK8PFchs.txt','Summer16_07Aug2017_V11_MC_L2Relative_AK8PFchs.txt','Summer16_07Aug2017_V11_MC_L3Absolute_AK8PFchs.txt','Summer16_07Aug2017_V11_MC_L1FastJet_AK8PFPuppi.txt','Summer16_07Aug2017_V11_MC_L2Relative_AK8PFPuppi.txt','Summer16_07Aug2017_V11_MC_L3Absolute_AK8PFPuppi.txt','Summer16_07Aug2017_V11_MC_L1FastJet_AK4PFPuppi.txt','Summer16_07Aug2017_V11_MC_L2Relative_AK4PFPuppi.txt','Summer16_07Aug2017_V11_MC_L3Absolute_AK4PFPuppi.txt','L1PrefiringMaps_new.root']
#config.JobType.inputFiles = ['PHYS14_25_V2_All_L1FastJet_AK4PFchs.txt','PHYS14_25_V2_All_L2Relative_AK4PFchs.txt','PHYS14_25_V2_All_L3Absolute_AK4PFchs.txt','PHYS14_25_V2_All_L1FastJet_AK8PFchs.txt','PHYS14_25_V2_All_L2Relative_AK8PFchs.txt','PHYS14_25_V2_All_L3Absolute_AK8PFchs.txt']
# Name of the CMSSW configuration file
#config.JobType.psetName = 'bkg_ana.py'
config.JobType.psetName = 'analysis.py'
#config.JobType.allowUndistributedCMSSW = True
config.JobType.allowUndistributedCMSSW = True
config.section_("Data")
#config.Data.inputDataset = '/WJetsToLNu_13TeV-madgraph-pythia8-tauola/Phys14DR-PU20bx25_PHYS14_25_V1-v1/MINIAODSIM'
config.Data.inputDataset = '/WkkToWRadionToWWW_M4500-R0-7-TuneCUEP8M1_13TeV-madgraph/RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6-v1/MINIAODSIM'
#config.Data.inputDBS = 'global'
config.Data.inputDBS = 'global'
config.Data.splitting = 'FileBased'
config.Data.unitsPerJob =50
config.Data.totalUnits = -1
config.Data.publication = False
#config.Data.outLFNDirBase = '/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/STEAM/' + steam_dir + '/' + name + '/'
# This string is used to construct the output dataset name
config.Data.outputDatasetTag = 'newM4500_R0-7_off'
HOST = "cmsweb-dev.cern.ch"
REQMGR = "https://%s/reqmgr/reqMgr" % HOST
REQMGR2 = "https://%s/reqmgr2" % HOST
COUCH = "https://%s/couchdb" % HOST
WEBURL = "%s/%s" % (COUCH, workqueueDBName)
LOG_DB_URL = "%s/wmstats_logdb" % COUCH
LOG_REPORTER = "global_workqueue"
root = __file__.rsplit('/', 4)[0]
cache_dir = os.path.join(root, 'state', 'workqueue', 'cache')
os.environ['WMCORE_CACHE_DIR'] = cache_dir
# Nothing after this point should need to be changed.
config = Configuration()
config.section_("Agent")
config.Agent.hostName = HOST
config.component_("WorkQueueManager")
config.WorkQueueManager.namespace = "WMComponent.WorkQueueManager.WorkQueueManager"
config.WorkQueueManager.couchurl = COUCH
config.WorkQueueManager.dbname = workqueueDBName
config.WorkQueueManager.inboxDatabase = workqueueInboxDbName
config.WorkQueueManager.wmstatDBName = wmstatDBName
config.WorkQueueManager.level = "GlobalQueue"
config.WorkQueueManager.queueParams = {'WMStatsCouchUrl': "%s/%s" % (COUCH, wmstatDBName)}
config.WorkQueueManager.queueParams['QueueURL'] = WEBURL
config.WorkQueueManager.queueParams['ReqMgrServiceURL'] = REQMGR2
config.WorkQueueManager.queueParams['RequestDBURL'] = "%s/%s" % (COUCH, reqmgrCouchDB)
config.WorkQueueManager.queueParams['central_logdb_url'] = LOG_DB_URL
config.WorkQueueManager.queueParams['log_reporter'] = LOG_REPORTER
import socket
import time
from WMCore.Configuration import Configuration
from os import path
HOST = socket.gethostname().lower()
BASE_URL = "@@BASE_URL@@"
DBS_INS = "@@DBS_INS@@"
COUCH_URL = "%s/couchdb" % BASE_URL
LOG_DB_URL = "%s/wmstats_logdb" % COUCH_URL
ROOTDIR = __file__.rsplit('/', 3)[0]
config = Configuration()
main = config.section_("main")
srv = main.section_("server")
srv.thread_pool = 30
main.application = "reqmgr2"
main.port = 8246 # main application port it listens on
main.index = "ui"
# Defaults to allow any CMS authenticated user. Write APIs should require
# additional roles in SiteDB (i.e. "Admin" role for the "ReqMgr" group)
main.authz_defaults = {"role": None, "group": None, "site": None}
sec = main.section_("tools").section_("cms_auth")
sec.key_file = "%s/auth/wmcore-auth/header-auth-key" % ROOTDIR
# this is where the application will be mounted, where the REST API
# is reachable and this features in CMS web frontend rewrite rules
app = config.section_(main.application) # string containing "reqmgr2"
'''
Created on Feb 15, 2010
'''
from WMCore.Configuration import Configuration
from WMCore.WMBase import getWMBASE
import os.path
config = Configuration()
config.section_("Emulator")
config.Emulator.PhEDEx = True
config.Emulator.DBSReader = True
config.Emulator.ReqMgr = True
Unittests for CMSSW config files
"""
import logging
import os
import unittest
from CRABClient.JobType.CMSSWConfig import CMSSWConfig
from CRABClient.JobType.ScramEnvironment import ScramEnvironment
from WMCore.Configuration import Configuration
#### Test WMCore.Configuration
testWMConfig = Configuration()
testWMConfig.section_("JobType")
testWMConfig.JobType.pluginName = 'CMSSW'
testWMConfig.section_("Data")
testWMConfig.Data.inputDataset = '/cms/data/set'
testWMConfig.section_("General")
testWMConfig.General.serverUrl = 'crabas.lnl.infn.it:8888'
testWMConfig.section_("User")
testWMConfig.User.group = 'Analysis'
#### Test CMSSW python config
testCMSSWConfig = """
import FWCore.ParameterSet.Config as cms
process = cms.Process("ANALYSIS")
process.load("FWCore.MessageLogger.MessageLogger_cfi")
if BASE_URL == "https://cmsweb.cern.ch":
RULE_LIFETIME=None
RUCIO_AUTH_URL="https://cms-rucio-auth.cern.ch"
RUCIO_URL="http://cms-rucio.cern.ch"
else:
RULE_LIFETIME=24 * 60 * 60 # 24h
RUCIO_AUTH_URL="https://cmsrucio-auth-int.cern.ch"
RUCIO_URL="http://cmsrucio-int.cern.ch"
# load AMQ credentials
sys.path.append(path.join(ROOTDIR, 'auth/reqmgr2ms'))
from ReqMgr2MSSecrets import USER_AMQ, PASS_AMQ, AMQ_TOPIC
config = Configuration()
main = config.section_("main")
srv = main.section_("server")
srv.thread_pool = 30
main.application = "ms-transferor"
main.port = 8247 # main application port it listens on
main.index = 'ui' # Configuration requires index attribute
# Security configuration
main.authz_defaults = {"role": None, "group": None, "site": None}
#set default logging (prevent duplicate)
main.log_screen = True
sec = main.section_("tools").section_("cms_auth")
sec.key_file = "%s/auth/wmcore-auth/header-auth-key" % ROOTDIR
# this is where the application will be mounted, where the REST API
# database. The GroupUser and ACDC couchapps needs to be install into the
# acdc database.
couchURL = "http://*****:*****@COUCHSERVER:5984"
workqueueDBName = 'workqueue'
workqueueInboxDbName = 'workqueue_inbox'
wmstatDBName = "wmstats"
# Agent name and team name.
agentName = "WMAgent"
teamName = "cmsdataops"
contactName = "*****@*****.**"
# Nothing after this point should need to be changed.
config = Configuration()
config.section_("Agent")
config.Agent.hostName = serverHostName
config.Agent.teamName = teamName
config.Agent.agentName = agentName
config.Agent.useMsgService = False
config.Agent.useTrigger = False
config.Agent.useHeartbeat = False
config.Agent.contact = contactName
config.section_("General")
config.General.workDir = workDirectory
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = databaseUrl
config.CoreDatabase.socket = databaseSocket
from WMCore.Configuration import Configuration
config = Configuration()
config.section_('General')
config.General.transferLogs = True
config.General.requestName = 'HXX_400_50_400_DGNTuple'
config.General.instance = 'prod'
config.section_('JobType')
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = 'runDGAnalysis_cfg.py'
config.JobType.disableAutomaticOutputCollection = True
config.JobType.outputFiles = ['DGNTuple.root']
config.JobType.maxMemoryMB = 2500
config.section_('Data')
config.Data.inputDBS = 'phys03'
#config.Data.unitsPerJob = 1
config.Data.splitting = 'FileBased'
config.Data.unitsPerJob = 10
#config.Data.totalUnits = 50
config.Data.inputDataset = '/H2ToLLPXToLeptons_MH_400_MX_50_ctau_400mm_TuneCP2_13TeV_pythia8_80X_13082019-1313/fernance-400-50-400_HXX_RunIISummer16MiniAODv3_230220-1650-bd3e7bcff6c9bcad356ea4ed7e4f08b4/USER'
config.Data.publication = False
config.Data.outLFNDirBase = '/store/user/fernance/'
config.section_('Site')
config.Site.storageSite = 'T2_ES_IFCA'
import os
from WMCore.Configuration import Configuration
config = Configuration()
# Set variables
production_label = os.environ["CRAB_PROD_LABEL"]
dataset = os.environ["CRAB_DATASET"]
cfgFile = os.environ["CMSRUN_CFG"]
unitsPerJob = int(os.environ["CRAB_UNITS_PER_JOB"])
publish = True if os.environ["CRAB_PUBLISH"] == 'True' else False
runOnNonValid = True if os.environ['CRAB_RUNONNONVALID'] == 'True' else False
inputDBS = os.environ["CRAB_input_DBS"]
if "CRAB_TOTAL_UNITS" in os.environ: totalUnits = os.environ["CRAB_TOTAL_UNITS"]
config.section_("General")
config.General.transferLogs = True
config.General.requestName = production_label
config.General.workArea = 'crab_' + os.environ['ORIG_PROD_LABEL'] + "_" + os.environ["MAOD_SAMPLE_NAME"]# config.General.requestName
config.section_("JobType")
config.JobType.allowUndistributedCMSSW = True
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = cfgFile
config.JobType.outputFiles = ['nanoAOD.root']
#config.JobType.numCores = 2
config.JobType.maxMemoryMB = 4000
config.section_("Data")
config.Data.inputDataset = dataset
from WMCore.Configuration import Configuration
config = Configuration()
config.section_("General")
config.General.requestName = 'runHitsRelValSingleMuFlatPt0p7to10noPU-v2V7_r3' ## change//Name of the directory that crab will create
config.section_("JobType")
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = 'runHits_cfg.py' ###Config file that crab will use for the job
config.JobType.pyCfgParams = [
'useMTDTrack=True', 'crysLayout=barzflat', 'output=DumpHits.root',
'dumpRecHits=True'
] # BBT, 01-22-19
config.General.transferOutputs = True
config.General.transferLogs = False
#config.JobType.allowUndistributedCMSSW = False
config.section_("Data") ###Might need to chance to MC
config.Data.inputDataset = None
#config.Data.inputDBS = 'phys03'
config.Data.splitting = 'FileBased'
config.Data.unitsPerJob = 1
#config.Data.outLFNDirBase = '/store/user/btannenw/MTD/10_4_0_mtd3/SingleMu_FlatPt_BTL_barz_v2/'
config.Data.outLFNDirBase = '/store/user/btannenw/'
config.Data.publication = False
config.Data.outputDatasetTag = '10_4_0_mtd3_runHitsRelValSingleMuFlatPt0p7to10noPU-v2V7_r3'
config.Data.allowNonValidInputDataset = True
config.Data.useParent = True
config.Data.inputDataset = '/RelValSingleMuFlatPt_0p7to10_pythia8/meridian-RelValSingleMuFlatPt0p7to10pythia8CMSSW1040mtd2patch1-103Xupgrade2023realisticv22023D35noPU-v2V7-45dc70b98f9b98b0807197db02f5776e/USER'
### V7 chi2=500, use only bestChi2, clusterTimeThreshold=10
# '/RelValSingleProtonFlatPt_0p7to10/meridian-RelValSingleProtonFlatPt0p7to10CMSSW1040mtd2patch1-103Xupgrade2023realisticv22023D35noPU-v1V7-45dc70b98f9b98b0807197db02f5776e/USER',
LOG_DB_URL = "%s/wmstats_logdb" % COUCH_URL
ROOTDIR = __file__.rsplit('/', 3)[0]
AMQ_HOST_PORT = [('cms-mb.cern.ch', 61313)]
# load AMQ credentials
sys.path.append(path.join(ROOTDIR, 'auth/reqmgr2ms'))
from ReqMgr2MSSecrets import USER_AMQ, PASS_AMQ, AMQ_TOPIC
if BASE_URL == "https://cmsweb.cern.ch":
RUCIO_ACCT = "wma_prod"
else:
RUCIO_ACCT = "wma_test"
config = Configuration()
main = config.section_("main")
srv = main.section_("server")
srv.thread_pool = 30
main.application = "ms-monitor"
main.port = 8248 # main application port it listens on
main.index = 'ui' # Configuration requires index attribute
# Security configuration
main.authz_defaults = {"role": None, "group": None, "site": None}
#set default logging (prevent duplicate)
main.log_screen = True
sec = main.section_("tools").section_("cms_auth")
sec.key_file = "%s/auth/wmcore-auth/header-auth-key" % ROOTDIR
# this is where the application will be mounted, where the REST API
Template for a locale configuration that contains all the stuff that agent operators want to customise
on a regular basis
Created by Dave Evans on 2011-05-20.
Copyright (c) 2011 Fermilab. All rights reserved.
"""
import sys
import os
import socket
from WMCore.Configuration import Configuration
config = Configuration()
config.section_('locale')
#
# Define the required sections
#
config.locale.section_("reqmgr")
config.locale.section_("gwq")
config.locale.section_("lwq")
config.locale.section_("agent")
config.locale.section_("crabserver")
config.locale.section_("couch")
config.locale.section_("mysql")
config.locale.section_("certificates")
# canned settings for each section
# request manager
# The amount of time to wait after a workflow has completed before archiving it.
workflowArchiveTimeout = 3600
# Global LogLevel
# For setting the general log level of the components
globalLogLevel = 'INFO'
# Contact information
# Used for email alerting
contactName = "*****@*****.**"
# Nothing beyond this point should need to be changed.
config = Configuration()
config.section_("Agent")
config.Agent.hostName = serverHostName
config.Agent.contact = contactName
config.Agent.teamName = "REPLACE_TEAM_NAME"
config.Agent.agentName = "WMAgent"
config.Agent.agentNumber = 0
config.Agent.useMsgService = False
config.Agent.useTrigger = False
config.Agent.useHeartbeat = True
config.Agent.isDocker = False
config.section_("General")
config.General.workDir = workDirectory
config.General.logdb_name = logDBName
config.General.central_logdb_url = "need to get from secrets file"
config.General.ReqMgr2ServiceURL = "ReqMgr2 rest service"
#pylint: disable=C0103,W0105,broad-except,logging-not-lazy,W0702,C0301,R0902,R0914,R0912,R0915
"""
Configuration file for CRAB standalone Publisher
"""
from __future__ import division
from WMCore.Configuration import Configuration
config = Configuration()
config.section_('General')
config.General.asoworker = 'asoprod1'
config.General.isOracle = True
config.General.oracleDB = 'cmsweb-testbed.cern.ch'
config.General.oracleFileTrans = '/crabserver/preprod/filetransfers'
config.General.oracleUserTrans = '/crabserver/preprod/fileusertransfers'
config.General.logLevel = 'INFO'
config.General.pollInterval = 1800
config.General.publish_dbs_url = 'https://cmsweb.cern.ch/dbs/prod/phys03/DBSWriter'
config.General.block_closure_timeout = 9400
config.General.serverDN = '/DC=ch/DC=cern/OU=computers/CN=vocms0118.cern.ch'
config.General.cache_area = 'https://cmsweb-testbed.cern.ch/crabserver/preprod/filemetadata'
#config.General.cache_area = 'https://cmsweb.cern.ch/crabserver/prod/filemetadata'
config.General.workflow_expiration_time = 3
config.General.serviceCert = '/data/certs/hostcert.pem'
config.General.serviceKey = '/data/certs/hostkey.pem'
config.General.logMsgFormat = '%(asctime)s:%(levelname)s:%(module)s:%(name)s: %(message)s'
config.General.max_files_per_block = 100
config.General.opsCert = '/data/certs/servicecert.pem'
config.General.opsKey = '/data/certs/servicekey.pem'
config.General.cache_path = '/crabserver/preprod/filemetadata'
"""
WMArchive configuration file.
"""
import os
import socket
from WMCore.Configuration import Configuration
HOST = socket.gethostname().lower()
# ROOTDIR = __file__.rsplit('/', 3)[0]
ROOTDIR = os.getenv('WMA_STATIC_ROOT', os.getcwd())
config = Configuration()
main = config.section_("main")
srv = main.section_("server")
srv.thread_pool = 30
main.application = "wmarchive"
main.port = 8247 # main application port it listens on
main.index = 'data' # Configuration requires index attribute
# Security configuration
#main.authz_defaults = {"role": None, "group": None, "site": None}
#sec = main.section_("tools").section_("cms_auth")
#sec.key_file = "%s/auth/wmcore-auth/header-auth-key" % ROOTDIR
# this is where the application will be mounted, where the REST API
# is reachable and this features in CMS web frontend rewrite rules
app = config.section_(main.application)
app.admin = "*****@*****.**"
app.description = "CMS data operations WMArchive."
app.title = "CMS Request Manager (WMArchive)"
msg = "Error updating document %s in CouchDB: %s" % (docid,
str(ex))
self.logger.error(msg)
return
if __name__ == "__main__":
import os
import logging
from RESTInteractions import CRABRest
from WMCore.Configuration import Configuration
logging.basicConfig(level=logging.DEBUG)
config = Configuration()
config.section_("TaskWorker")
#will use X509_USER_PROXY var for this test
config.TaskWorker.cmscert = os.environ["X509_USER_PROXY"]
config.TaskWorker.cmskey = os.environ["X509_USER_PROXY"]
server_ = CRABRest('cmsweb-testbed.cern.ch', config.TaskWorker.cmscert,
config.TaskWorker.cmskey)
server_.setDbInstance('dev')
resubmitter = DagmanResubmitter(config, server_)
resubmitter.execute(
task={
'tm_taskname':
'141129_110306_crab3test-5:atanasi_crab_test_resubmit',
'user_proxy': os.environ["X509_USER_PROXY"],
'resubmit_site_whitelist': ['T2_IT_Bari'],
'resubmit_site_blacklist': ['T2_IT_Legnaro'],
from WMCore.Configuration import Configuration
config = Configuration()
config.section_('General')
config.General.transferOutputs = True
config.General.requestName = 'Pythia8_Ze10e10Jet-HINppWinter16DR-75X_mcRun2_forest_v1'
config.section_('JobType')
config.JobType.psetName = 'runForestAOD_pp_MC_75X.py'
config.JobType.pluginName = 'Analysis'
config.section_('Data')
config.Data.inputDataset = '/Pythia8_Ze10e10Jet/HINppWinter16DR-75X_mcRun2_asymptotic_ppAt5TeV_v3-v1/AODSIM'
config.Data.inputDBS = 'global'
config.Data.splitting = "FileBased"
config.Data.unitsPerJob = 1
config.Data.totalUnits = -1
config.Data.outputDatasetTag = 'Pythia8_Ze10e10Jet-HINppWinter16DR-75X_mcRun2_forest_v1'
config.section_('User')
config.section_('Site')
#config.Site.whitelist = ['T2_US_MIT']
config.Site.storageSite = 'T2_US_MIT'
# config.section_("Debug")
# config.Debug.extraJDL = ['+CMS_ALLOW_OVERFLOW=False']
