def __init__(self, config):
self.config = config
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)
self.locationAction = daoFactory(classname="Locations.GetSiteInfo")
self.packageDir = None
if os.path.exists(os.path.join(getWMBASE(), "src/python/WMCore/WMRuntime/Unpacker.py")):
self.unpacker = os.path.join(getWMBASE(), "src/python/WMCore/WMRuntime/Unpacker.py")
else:
self.unpacker = os.path.join(getWMBASE(), "WMCore/WMRuntime/Unpacker.py")
self.agent = getattr(config.Agent, "agentName", "WMAgent")
self.sandbox = None
self.scriptFile = None
self.submitDir = None
self.removeTime = getattr(config.BossAir, "removeTime", 60)
self.multiTasks = getattr(config.BossAir, "multicoreTaskTypes", [])
# Build ourselves a pool
self.pool = []
self.input = multiprocessing.Queue()
self.result = multiprocessing.Queue()
self.nProcess = getattr(self.config.BossAir, "nCondorProcesses", 4)
return
def __init__(self, config):
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.locationAction = daoFactory(classname="Locations.GetSiteInfo")
self.packageDir = None
if os.path.exists(
os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')):
self.unpacker = os.path.join(
getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py')
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = config.JobSubmitter.submitScript
self.defaultTaskPriority = getattr(config.BossAir,
'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200)
self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500)
# Required for global pool accounting
self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser',
"cmsdataops")
# Build a requirement string. All CMS resources match DESIRED_Sites on the START
# expression side; however, there are currently some resources (T2_CH_CERN_HLT)
# that are missing the REQUIRED_OS logic. Hence, we duplicate it here.
# TODO(bbockelm): Remove reqStr once HLT has upgraded.
self.reqStr = (
'((REQUIRED_OS=?="any") || '
'(GLIDEIN_REQUIRED_OS =?= "any") || '
'stringListMember(GLIDEIN_REQUIRED_OS, REQUIRED_OS)) && '
'(AuthenticatedIdentity =!= "*****@*****.**")')
if hasattr(config.BossAir, 'condorRequirementsString'):
self.reqStr = config.BossAir.condorRequirementsString
# x509 proxy handling
proxy = Proxy({'logger': myThread.logger})
self.x509userproxy = proxy.getProxyFilename()
self.x509userproxysubject = proxy.getSubject()
self.x509userproxyfqan = proxy.getAttributeFromProxy(
self.x509userproxy)
# Remove the x509 ads if the job is matching a volunteer resource
self.x509Expr = 'ifThenElse("$$(GLIDEIN_CMSSite)" =?= "T3_CH_Volunteer",undefined,"%s")'
return
def injectIntoConfigCache(self, frameworkVersion, scramArch, initCommand,
configUrl, configLabel, couchUrl, couchDBName):
"""
_injectIntoConfigCache_
"""
logging.error("Injecting to config cache.\n");
configTempDir = tempfile.mkdtemp()
configPath = os.path.join(configTempDir, "cmsswConfig.py")
configString = urllib.urlopen(configUrl).read(-1)
configFile = open(configPath, "w")
configFile.write(configString)
configFile.close()
scramTempDir = tempfile.mkdtemp()
wmcoreBase = getWMBASE()
envPath = os.path.normpath(os.path.join(getWMBASE(), "../../../../../../../../apps/wmagent/etc/profile.d/init.sh"))
scram = Scram(version = frameworkVersion, architecture = scramArch,
directory = scramTempDir, initialise = initCommand,
envCmd = "source %s" % envPath)
scram.project()
scram.runtime()
scram("python2.6 %s/../../../bin/inject-to-config-cache %s %s PromptSkimmer cmsdataops %s %s None" % (wmcoreBase,
couchUrl,
couchDBName,
configPath,
configLabel))
shutil.rmtree(configTempDir)
shutil.rmtree(scramTempDir)
return
def __init__(self, config):
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.locationAction = daoFactory(classname="Locations.GetSiteInfo")
self.packageDir = None
# if agent is running in a container, Unpacker.py must come from a directory
# on the host so the condor schedd can see it
# config.General.workDir should always be bind mounted to the container
if getattr(config.Agent, "isDocker", False):
unpackerPath = os.path.join(config.General.workDir +
"/Docker/WMRuntime/Unpacker.py")
else:
unpackerPath = os.path.join(
getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py')
if os.path.exists(unpackerPath):
self.unpacker = unpackerPath
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = config.JobSubmitter.submitScript
self.defaultTaskPriority = getattr(config.BossAir,
'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200)
self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500)
# Required for global pool accounting
self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser',
"cmsdataops")
if hasattr(config.BossAir, 'condorRequirementsString'):
self.reqStr = config.BossAir.condorRequirementsString
else:
self.reqStr = None
# x509 proxy handling
proxy = Proxy({'logger': myThread.logger})
self.x509userproxy = proxy.getProxyFilename()
# These are added now by the condor client
#self.x509userproxysubject = proxy.getSubject()
#self.x509userproxyfqan = proxy.getAttributeFromProxy(self.x509userproxy)
return
def __init__(self, config):
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger,
dbinterface=myThread.dbi)
self.locationAction = daoFactory(classname="Locations.GetSiteInfo")
self.packageDir = None
if os.path.exists(os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')):
self.unpacker = os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = config.JobSubmitter.submitScript
self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200)
self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500)
# Required for global pool accounting
self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops")
# Build a requirement string. All CMS resources match DESIRED_Sites on the START
# expression side; however, there are currently some resources (T2_CH_CERN_HLT)
# that are missing the REQUIRED_OS logic. Hence, we duplicate it here.
# TODO(bbockelm): Remove reqStr once HLT has upgraded.
self.reqStr = ('((REQUIRED_OS=?="any") || '
'(GLIDEIN_REQUIRED_OS =?= "any") || '
'stringListMember(GLIDEIN_REQUIRED_OS, REQUIRED_OS)) && '
'(AuthenticatedIdentity =!= "*****@*****.**")')
if hasattr(config.BossAir, 'condorRequirementsString'):
self.reqStr = config.BossAir.condorRequirementsString
# x509 proxy handling
proxy = Proxy({'logger': myThread.logger})
self.x509userproxy = proxy.getProxyFilename()
self.x509userproxysubject = proxy.getSubject()
self.x509userproxyfqan = proxy.getAttributeFromProxy(self.x509userproxy)
# Remove the x509 ads if the job is matching a volunteer resource
self.x509Expr = 'ifThenElse("$$(GLIDEIN_CMSSite)" =?= "T3_CH_Volunteer",undefined,"%s")'
return
def __init__(self, config):
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.locationAction = daoFactory(classname="Locations.GetSiteInfo")
self.packageDir = None
if os.path.exists(
os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')):
self.unpacker = os.path.join(
getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py')
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = config.JobSubmitter.submitScript
self.defaultTaskPriority = getattr(config.BossAir,
'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200)
# Required for global pool accounting
self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser',
"cmsdataops")
# Build a requirement string
self.reqStr = "stringListMember(GLIDEIN_CMSSite, DESIRED_Sites) && ((REQUIRED_OS=?=\"any\") || (GLIDEIN_REQUIRED_OS=?=REQUIRED_OS)) && (TARGET.Cpus >= RequestCpus)"
if hasattr(config.BossAir, 'condorRequirementsString'):
self.reqStr = config.BossAir.condorRequirementsString
# x509 proxy handling
proxy = Proxy({'logger': myThread.logger})
self.x509userproxy = proxy.getProxyFilename()
self.x509userproxysubject = proxy.getSubject()
return
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getWMBASE(), "test/python/WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def testC_testViews(self):
"""
_testViews_
Prototype test for what should be a lot of other tests.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['attach1'] = attach
configCache.document['md5_hash'] = "somemd5"
psetPath = os.path.join(getWMBASE(), "test/python/WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache2.document['md5_hash'] = configCache.document['md5_hash']
configCache2.load()
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache2.delete()
return
def testTrivialFileCatalog(self):
"""
Run some simple tests on reading a trivialFileCatalog
"""
tfcFilename = os.path.join(getWMBASE(),
"test/python/WMCore_t/Storage_t",
"T1_US_FNAL_TrivialFileCatalog.xml")
if not os.path.exists(tfcFilename):
raise Exception("No TrivialFileCatalog found!")
tfcInstance = readTFC(tfcFilename)
self.assertEqual(type(tfcInstance), type(TrivialFileCatalog()))
# Look for similarities in each node of the TFC file
for mapping in ['lfn-to-pfn', 'pfn-to-lfn']:
for x in tfcInstance[mapping]:
self.assertEqual(x.has_key('path-match-expr'), True)
self.assertEqual(x.has_key('path-match'), True)
self.assertEqual(x.has_key('protocol'), True)
self.assertEqual(x.has_key('result'), True)
self.assertEqual(x.has_key('chain'), True)
self.assertEqual(x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'],
True, 'Could not find protocol %s' % (x['protocol']))
self.assertEqual(x['chain'], None, 'Invalid chain %s' % (x['chain']))
def injectIntoConfigCache(self, frameworkVersion, scramArch, initCommand,
configUrl, configLabel, couchUrl, couchDBName):
"""
_injectIntoConfigCache_
"""
configTempDir = tempfile.mkdtemp()
configPath = os.path.join(configTempDir, "cmsswConfig.py")
configString = urllib.urlopen(configUrl).read(-1)
configFile = open(configPath, "w")
configFile.write(configString)
configFile.close()
scramTempDir = tempfile.mkdtemp()
scram = Scram(version = frameworkVersion, architecture = scramArch,
directory = scramTempDir, initialise = initCommand)
scram.project()
scram.runtime()
wmcoreBase = getWMBASE()
scram("python2.6 %s/bin/inject-to-config-cache %s %s PromptSkimmer cmsdataops %s %s None" % (wmcoreBase,
couchUrl,
couchDBName,
configPath,
configLabel))
shutil.rmtree(configTempDir)
shutil.rmtree(scramTempDir)
return
def initialize(self):
self.config = DefaultConfig()
self.config.UnitTests.templates = getWMBASE() + '/src/templates/WMCore/WebTools'
self.config.Webtools.section_('server')
self.config.Webtools.server.socket_timeout = 1
self.urlbase = self.config.getServerUrl()
self.cache_path = tempfile.mkdtemp()
def thrashCouch():
"""
_thrashCouch_
"""
jobs = {"new": set(), "created": set(), "executing": set(),
"complete": set(), "success": set(), "cleanout": set()}
config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
changeState = ChangeState(config)
myReport = Report()
myReport.unpersist(os.path.join(getWMBASE(), "test/python/WMComponent_t/JobAccountant_t/fwjrs/LoadTest00.pkl"))
for i in range(500):
jobs = createJobs()
changeState.recordInCouch(jobs, "created", "new")
changeState.recordInCouch(jobs, "executing", "created")
changeState.recordInCouch(jobs, "complete", "executing")
for job in jobs:
job["fwjr"] = myReport
changeState.recordInCouch(jobs, "success", "complete")
for job in jobs:
job["fwjr"] = None
changeState.recordInCouch(jobs, "cleanout", "success")
#time.sleep(10)
return
def __init__(self, config):
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger,
dbinterface=myThread.dbi)
self.locationAction = daoFactory(classname="Locations.GetSiteInfo")
self.packageDir = None
if os.path.exists(os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')):
self.unpacker = os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = config.JobSubmitter.submitScript
self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200)
self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500)
# Required for global pool accounting
self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops")
# Build a requirement string
self.reqStr = ('stringListMember(GLIDEIN_CMSSite, DESIRED_Sites) '
'&& ((REQUIRED_OS=?="any") || stringListMember(GLIDEIN_REQUIRED_OS, REQUIRED_OS))'
'&& (TARGET.Cpus >= RequestCpus)')
if hasattr(config.BossAir, 'condorRequirementsString'):
self.reqStr = config.BossAir.condorRequirementsString
# x509 proxy handling
proxy = Proxy({'logger': myThread.logger})
self.x509userproxy = proxy.getProxyFilename()
self.x509userproxysubject = proxy.getSubject()
return
def testVanderbiltSiteLocalConfig(self):
"""
_testFNALSiteLocalConfig_
Verify that the FNAL site config file is parsed correctly.
"""
vandyConfigFileName = os.path.join(getWMBASE(),
"test/python/WMCore_t/Storage_t",
"T3_US_Vanderbilt_SiteLocalConfig.xml")
mySiteConfig = SiteLocalConfig(vandyConfigFileName)
assert mySiteConfig.siteName == "T3_US_Vanderbilt", \
"Error: Wrong site name."
assert len(mySiteConfig.eventData.keys()) == 1, \
"Error: Wrong number of event data keys."
assert mySiteConfig.eventData["catalog"] == "trivialcatalog_file://gpfs1/grid/grid-app/cmssoft/cms/SITECONF/local/PhEDEx/storage.xml?protocol=direct", \
"Eroror: Event data catalog is wrong."
goldenServers = ["http://cmsfrontier.cern.ch:8000/FrontierInt",
"http://cmsfrontier1.cern.ch:8000/FrontierInt",
"http://cmsfrontier2.cern.ch:8000/FrontierInt",
"http://cmsfrontier3.cern.ch:8000/FrontierInt"]
for frontierServer in mySiteConfig.frontierServers:
assert frontierServer in goldenServers, \
"Error: Unknown server: %s" % frontierServer
goldenServers.remove(frontierServer)
assert len(goldenServers) == 0, \
"Error: Missing frontier servers."
goldenProxies = ["http://se1.accre.vanderbilt.edu:3128"]
for frontierProxy in mySiteConfig.frontierProxies:
assert frontierProxy in goldenProxies, \
"Error: Unknown proxy: %s" % frontierProxy
goldenProxies.remove(frontierProxy)
assert len(goldenProxies) == 0, \
"Error: Missing proxy servers."
assert mySiteConfig.localStageOut["se-name"] == "se1.accre.vanderbilt.edu", \
"Error: Wrong se name from local stageout."
assert mySiteConfig.localStageOut["command"] == "srmv2", \
"Error: Wrong stage out command."
assert mySiteConfig.localStageOut["catalog"] == "trivialcatalog_file://gpfs1/grid/grid-app/cmssoft/cms/SITECONF/local/PhEDEx/storage.xml?protocol=srmv2", \
"Error: TFC catalog is not correct."
assert len(mySiteConfig.fallbackStageOut) == 1, \
"Error: Incorrect number of fallback stageout methods"
assert mySiteConfig.fallbackStageOut[0]["command"] == "srmv2-lcg", \
"Error: Incorrect fallback command."
assert mySiteConfig.fallbackStageOut[0]["se-name"] == "se1.accre.vanderbilt.edu", \
"Error: Incorrect fallback SE."
assert mySiteConfig.fallbackStageOut[0]["lfn-prefix"] == "srm://se1.accre.vanderbilt.edu:6288/srm/v2/server?SFN=", \
"Error: Incorrect fallback LFN prefix."
return
def testFNALSiteLocalConfig(self):
"""
_testFNALSiteLocalConfig_
Verify that the FNAL site config file is parsed correctly.
"""
fnalConfigFileName = os.path.join(getWMBASE(),
"test/python/WMCore_t/Storage_t",
"T1_US_FNAL_SiteLocalConfig.xml")
mySiteConfig = SiteLocalConfig(fnalConfigFileName)
assert mySiteConfig.siteName == "T1_US_FNAL", \
"Error: Wrong site name."
assert len(mySiteConfig.eventData.keys()) == 1, \
"Error: Wrong number of event data keys."
assert mySiteConfig.eventData["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \
"Eroror: Event data catalog is wrong."
goldenServers = ["http://cmsfrontier.cern.ch:8000/FrontierInt",
"http://cmsfrontier.cern.ch:8000/FrontierInt",
"http://cmsfrontier1.cern.ch:8000/FrontierInt",
"http://cmsfrontier2.cern.ch:8000/FrontierInt",
"http://cmsfrontier3.cern.ch:8000/FrontierInt",
"http://cmsfrontier4.cern.ch:8000/FrontierInt"]
for frontierServer in mySiteConfig.frontierServers:
assert frontierServer in goldenServers, \
"Error: Unknown server: %s" % frontierServer
goldenServers.remove(frontierServer)
assert len(goldenServers) == 0, \
"Error: Missing frontier servers."
goldenProxies = ["http://cmsfrontier1.fnal.gov:3128",
"http://cmsfrontier2.fnal.gov:3128",
"http://cmsfrontier3.fnal.gov:3128",
"http://cmsfrontier4.fnal.gov:3128"]
for frontierProxy in mySiteConfig.frontierProxies:
assert frontierProxy in goldenProxies, \
"Error: Unknown proxy: %s" % frontierProxy
goldenProxies.remove(frontierProxy)
assert len(goldenProxies) == 0, \
"Error: Missing proxy servers."
assert mySiteConfig.localStageOut["se-name"] == "cmssrm.fnal.gov", \
"Error: Wrong se name from local stageout."
assert mySiteConfig.localStageOut["command"] == "dccp-fnal", \
"Error: Wrong stage out command."
assert mySiteConfig.localStageOut["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=srmv2", \
"Error: TFC catalog is not correct."
assert mySiteConfig.fallbackStageOut == [], \
"Error: Fallback config is incorrect."
return
def submit(self, jobs, info=None):
jobscript = self.config.JobSubmitter.submitScript
unpacker = os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')
xrsl = '+\n'
wrapper = gen_wrapper_script()
for j in jobs:
logging.debug("ARCplugin.submit: got job %s" % str(j))
xrsl += '(' + gen_xrsl(j, wrapper.name, jobscript,
unpacker) + ')\n'
xrsl_file = tempfile.NamedTemporaryFile(prefix="xrslcode.")
xrsl_file.write(xrsl)
xrsl_file.flush()
cmd = "ngsub %s -c %s" % (xrsl_file.name, jobs[0]['location'])
s, output = executeCommand(cmd)
logging.debug("executed %s" % cmd)
logging.debug("with exit stat %s" % s)
logging.debug("and output %s" % output)
xrsl_file.close()
wrapper.close()
subRe = re.compile(
"Job submitted with jobid: +(\w+://([a-zA-Z0-9.-]+)(:\d+)?(/.*)?/\d+)"
)
lines = output.split('\n')
n = 0
failure = []
success = []
for j in jobs:
if n < len(lines):
m = re.match(subRe, lines[n])
else:
m = None
if m:
j['gridid'] = m.group(1)
success.append(j)
logging.debug("Successful job %s" % j['gridid'])
else:
failure.append(j)
logging.debug("Failed job")
n += 1
logging.info("%i successful job submissions, %i failures" %
(len(success), len(failure)))
return success, failure
def testLoadingConfigFromOverridenEnvVarriable(self):
"""
test SiteLocalConfig module method loadSiteLocalConfig when loading
site config from location defined by WMAGENT_SITE_CONFIG_OVERRIDE
env. variable
"""
vandyConfigFileName = os.path.join(getWMBASE(),
"test/python/WMCore_t/Storage_t",
"T3_US_Vanderbilt_SiteLocalConfig.xml")
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = vandyConfigFileName
mySiteConfig = loadSiteLocalConfig()
self.assertEqual(mySiteConfig.siteName, "T3_US_Vanderbilt",
"Error: Wrong site name.")
def injectIntoConfigCache(frameworkVersion,
scramArch,
initCommand,
configUrl,
configLabel,
couchUrl,
couchDBName,
envPath=None,
binPath=None):
"""
_injectIntoConfigCache_
"""
logging.info("Injecting to config cache.\n")
configTempDir = tempfile.mkdtemp()
configPath = os.path.join(configTempDir, "cmsswConfig.py")
configString = urllib.urlopen(fixCVSUrl(configUrl)).read(-1)
configFile = open(configPath, "w")
configFile.write(configString)
configFile.close()
scramTempDir = tempfile.mkdtemp()
wmcoreBase = getWMBASE()
if not envPath:
envPath = os.path.normpath(
os.path.join(
wmcoreBase,
"../../../../../../../../apps/wmagent/etc/profile.d/init.sh"))
scram = Scram(version=frameworkVersion,
architecture=scramArch,
directory=scramTempDir,
initialise=initCommand,
envCmd="source %s" % envPath)
scram.project()
scram.runtime()
if not binPath:
scram(
"python2.6 %s/../../../bin/inject-to-config-cache %s %s PromptSkimmer cmsdataops %s %s None"
% (wmcoreBase, couchUrl, couchDBName, configPath, configLabel))
else:
scram(
"python2.6 %s/inject-to-config-cache %s %s PromptSkimmer cmsdataops %s %s None"
% (binPath, couchUrl, couchDBName, configPath, configLabel))
shutil.rmtree(configTempDir)
shutil.rmtree(scramTempDir)
return
def __init__(self, config):
self.config = config
BasePlugin.__init__(self, config)
self.packageDir = None
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = config.Agent.agentName
self.sandbox = None
self.scriptFile = None
self.queue = None
self.resourceReq = None
self.jobGroup = None
return
def __init__(self, config):
self.config = config
BasePlugin.__init__(self, config)
self.packageDir = None
self.unpacker = os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')
self.agent = config.Agent.agentName
self.sandbox = None
self.scriptFile = None
self.queue = None
self.resourceReq = None
self.jobGroup = None
return
def submit(self, jobs, info = None):
jobscript = self.config.JobSubmitter.submitScript
unpacker = os.path.join(getWMBASE(),'src/python/WMCore/WMRuntime/Unpacker.py')
xrsl = '+\n'
wrapper = gen_wrapper_script()
for j in jobs:
logging.debug("ARCplugin.submit: got job %s" % str(j))
xrsl += '(' + gen_xrsl(j, wrapper.name, jobscript, unpacker) + ')\n'
xrsl_file = tempfile.NamedTemporaryFile(prefix="xrslcode.")
xrsl_file.write(xrsl)
xrsl_file.flush()
cmd = "ngsub %s -c %s" % (xrsl_file.name, jobs[0]['location'])
s, output = executeCommand(cmd)
logging.debug("executed %s" % cmd)
logging.debug("with exit stat %s" % s)
logging.debug("and output %s" % output)
xrsl_file.close()
wrapper.close()
subRe = re.compile("Job submitted with jobid: +(\w+://([a-zA-Z0-9.-]+)(:\d+)?(/.*)?/\d+)")
lines = output.split('\n')
n = 0
failure = []
success = []
for j in jobs:
if n < len(lines):
m = re.match(subRe, lines[n])
else:
m = None
if m:
j['gridid'] = m.group(1)
success.append(j)
logging.debug("Successful job %s" % j['gridid'])
else:
failure.append(j)
logging.debug("Failed job")
n += 1
logging.info("%i successful job submissions, %i failures" % (len(success), len(failure)))
return success, failure
def __init__(self, config):
self.config = config
BasePlugin.__init__(self, config)
self.packageDir = None
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = config.Agent.agentName
self.sandbox = None
self.scriptFile = None
self.queue = None
self.resourceReq = None
self.jobGroup = None
self.basePrio = getattr(config.BossAir, 'LsfBasePrio', 50)
return
def testRoundTrip(self):
"""
Test PFN, LFN matching upon an existing TFC XML file
"""
tfc_file = os.path.join(getWMBASE(),
"test/python/WMCore_t/Storage_t",
"T1_US_FNAL_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
# Check that an lfn goes to an srmv2 pfn and comes back as the same lfn
in_lfn = '/store/data/my/data'
in_pfn = "srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/data/my/data"
out_pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, in_pfn)
out_lfn = tfc.matchPFN('srmv2', in_pfn)
self.assertEqual(in_lfn, out_lfn)
out_pfn = tfc.matchLFN('srmv2', out_lfn)
self.assertEqual(in_pfn, out_pfn)
def thrashCouch():
"""
_thrashCouch_
"""
jobs = {
"new": set(),
"created": set(),
"executing": set(),
"complete": set(),
"success": set(),
"cleanout": set()
}
config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
changeState = ChangeState(config)
myReport = Report()
myReport.unpersist(
os.path.join(
getWMBASE(),
"test/python/WMComponent_t/JobAccountant_t/fwjrs/LoadTest00.pkl"))
for i in range(500):
jobs = createJobs()
changeState.recordInCouch(jobs, "created", "new")
changeState.recordInCouch(jobs, "executing", "created")
changeState.recordInCouch(jobs, "complete", "executing")
for job in jobs:
job["fwjr"] = myReport
changeState.recordInCouch(jobs, "success", "complete")
for job in jobs:
job["fwjr"] = None
changeState.recordInCouch(jobs, "cleanout", "success")
#time.sleep(10)
return
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testDir = self.testInit.generateWorkDir()
sys.path.insert(0, os.path.join(getWMBASE(), "test/python/WMCore_t/WMRuntime_t/Scripts_t"))
def __init__(self, config):
self.config = config
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger,
dbinterface = myThread.dbi)
self.locationAction = daoFactory(classname = "Locations.GetSiteInfo")
self.packageDir = None
if os.path.exists(os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')):
self.unpacker = os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = None
self.submitDir = None
self.removeTime = getattr(config.BossAir, 'removeTime', 60)
self.multiTasks = getattr(config.BossAir, 'multicoreTaskTypes', [])
self.useGSite = getattr(config.BossAir, 'useGLIDEINSites', False)
self.submitWMSMode = getattr(config.BossAir, 'submitWMSMode', False)
self.errorThreshold= getattr(config.BossAir, 'submitErrorThreshold', 10)
self.errorCount = 0
self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
# Build ourselves a pool
self.pool = []
self.input = None
self.result = None
self.nProcess = getattr(self.config.BossAir, 'nCondorProcesses', 4)
# Set up my proxy and glexec stuff
self.setupScript = getattr(config.BossAir, 'UISetupScript', None)
self.proxy = None
self.serverCert = getattr(config.BossAir, 'delegatedServerCert', None)
self.serverKey = getattr(config.BossAir, 'delegatedServerKey', None)
self.myproxySrv = getattr(config.BossAir, 'myproxyServer', None)
self.proxyDir = getattr(config.BossAir, 'proxyDir', '/tmp/')
self.serverHash = getattr(config.BossAir, 'delegatedServerHash', None)
self.glexecPath = getattr(config.BossAir, 'glexecPath', None)
self.glexecWrapScript = getattr(config.BossAir, 'glexecWrapScript', None)
self.glexecUnwrapScript = getattr(config.BossAir, 'glexecUnwrapScript', None)
self.jdlProxyFile = None # Proxy name to put in JDL (owned by submit user)
self.glexecProxyFile = None # Copy of same file owned by submit user
if self.glexecPath:
if not (self.myproxySrv and self.proxyDir):
raise WMException('glexec requires myproxyServer and proxyDir to be set.')
if self.myproxySrv:
if not (self.serverCert and self.serverKey):
raise WMException('MyProxy server requires serverCert and serverKey to be set.')
# Make the directory for the proxies
if self.proxyDir and not os.path.exists(self.proxyDir):
logging.debug("proxyDir not found: creating it.")
try:
os.makedirs(self.proxyDir, 01777)
except Exception, ex:
msg = "Error: problem when creating proxyDir directory - '%s'" % str(ex)
raise BossAirPluginException(msg)
__all__ = []
import thread
import socket
import re
import os.path
import WMCore.HTTPFrontEnd.RequestManager.ReqMgrConfiguration as ReqMgrConfig
from WMCore.WMInit import getWMBASE
thread.stack_size(128 * 1024)
INSTALL = getWMBASE()
HOST = socket.getfqdn().lower()
COUCH = "https://%s/couchdb" % HOST
if re.match(r"^vocms0(?:13[689]|14[0123]|16[135]|30[67]|318)\.cern\.ch$",
HOST):
COUCH = "https://cmsweb.cern.ch/couchdb"
elif re.match(r"^vocms0(?:13[12])\.cern\.ch$", HOST):
COUCH = "https://cmsweb-testbed.cern.ch/couchdb"
elif re.match(r"^vocms0127\.cern\.ch$", HOST):
COUCH = "https://cmsweb-dev.cern.ch/couchdb"
elif re.match(r"^vocms0126\.cern\.ch$", HOST):
COUCH = "https://cmsweb-sec.cern.ch/couchdb"
config = ReqMgrConfig.reqMgrConfig(installation=INSTALL, couchurl=COUCH)
TEMPLATES = os.path.normpath(
os.path.join(INSTALL, '../../../data/templates/WMCore/WebTools'))
JAVASCRIPT_PATH = os.path.normpath(
def testA(self):
try:
getWMBASE()
except:
self.fail("Error calling WMInit.getWMBASE")
def testDashboardTransitions(self):
"""
_testDashboardTransitions_
Verify that the dashboard transitions code works correctly.
"""
change = ChangeState(self.config, "changestate_t")
locationAction = self.daoFactory(classname = "Locations.New")
locationAction.execute("site1", seName = "somese.cern.ch")
testWorkflow = Workflow(spec = "spec.xml", owner = "Steve",
name = "wf001", task = "Test", wfType = "ReDigi")
testWorkflow.create()
testFileset = Fileset(name = "TestFileset")
testFileset.create()
testSubscription = Subscription(fileset = testFileset,
workflow = testWorkflow,
split_algo = "FileBased")
testSubscription.create()
testFileA = File(lfn = "SomeLFNA", events = 1024, size = 2048,
locations = set(["somese.cern.ch"]))
testFileB = File(lfn = "SomeLFNB", events = 1025, size = 2049,
locations = set(["somese.cern.ch"]))
testFileA.create()
testFileB.create()
testFileset.addFile(testFileA)
testFileset.addFile(testFileB)
testFileset.commit()
splitter = SplitterFactory()
jobFactory = splitter(package = "WMCore.WMBS",
subscription = testSubscription)
jobGroup = jobFactory(files_per_job = 1)[0]
assert len(jobGroup.jobs) == 2, \
"Error: Splitting should have created two jobs."
testJobA = jobGroup.jobs[0]
testJobA["user"] = "******"
testJobA["group"] = "DMWM"
testJobB = jobGroup.jobs[1]
testJobB["user"] = "******"
testJobB["group"] = "DMWM"
change.propagate([testJobA, testJobB], "new", "none")
change.propagate([testJobA, testJobB], "created", "new")
change.propagate([testJobA], "executing", "created")
change.propagate([testJobB], "submitfailed", "created")
change.propagate([testJobB], "submitcooloff", "submitfailed")
transitions = change.listTransitionsForDashboard()
self.assertEqual(len(transitions), 1,
"Error: Wrong number of transitions: %s" % transitions)
self.assertEqual(transitions[0]["name"], testJobA["name"],
"Error: Wrong job name.")
self.assertEqual(transitions[0]["retryCount"], 0,
"Error: Wrong retry count.")
self.assertEqual(transitions[0]["newState"], "executing",
"Error: Wrong new state.")
self.assertEqual(transitions[0]["oldState"], "created",
"Error: Wrong old state.")
self.assertEqual(transitions[0]["requestName"], "wf001",
"Error: Wrong request name.")
transitions = change.listTransitionsForDashboard()
self.assertEqual(len(transitions), 0,
"Error: Wrong number of transitions")
change.propagate([testJobB], "created", "submitcooloff")
change.propagate([testJobB], "executing", "created")
change.propagate([testJobA, testJobB], "complete", "executing")
change.propagate([testJobB], "success", "complete")
change.propagate([testJobA], "jobfailed", "complete")
timestamp = int(time.time())
transitions = change.listTransitionsForDashboard()
for transition in transitions:
self.assertTrue(timestamp - 10 < transition["timestamp"] and
timestamp + 10 > transition["timestamp"],
"Error: Timestamp is wrong.")
del transition["timestamp"]
goldenTransitions = [{"name": testJobA["name"], "retryCount": 0, "newState": "jobfailed",
"oldState": "complete", "requestName": "wf001", "user": "******",
"group": "DMWM", "jobType": "Processing", "taskType": "ReDigi",
"performance": {}, "exitCode": 0},
{"name": testJobB["name"], "retryCount": 1, "newState": "executing",
"oldState": "created", "requestName": "wf001", "user": "******",
"group": "DMWM", "jobType": "Processing", "taskType": "ReDigi",
"performance": {}, "exitCode": 0},
{"name": testJobB["name"], "retryCount": 1, "newState": "success",
"oldState": "complete", "requestName": "wf001", "user": "******",
"group": "DMWM", "jobType": "Processing", "taskType": "ReDigi",
"performance": {}, "exitCode": 0}]
self.assertEqual(transitions, goldenTransitions,
"Error: Wrong transitions: %s %s" % (transitions, goldenTransitions))
xmlPath = os.path.join(getWMBASE(),
"test/python/WMCore_t/FwkJobReport_t/PerformanceReport.xml")
myReport = Report("cmsRun1")
myReport.parse(xmlPath)
testJobA["fwjr"] = myReport
change.propagate([testJobA], "complete", "executing")
change.propagate([testJobA], "success", "complete")
transitions = change.listTransitionsForDashboard()
self.assertEqual(len(transitions), 1,
"Error: Wrong number of transitions.")
perfSection = transitions[0]["performance"]["cmsRun1"]
self.assertTrue(perfSection.has_key("storage"),
"Error: Storage section is missing.")
self.assertTrue(perfSection.has_key("memory"),
"Error: Memory section is missing.")
self.assertTrue(perfSection.has_key("cpu"),
"Error: CPU section is missing.")
self.assertEqual(perfSection["cpu"]["AvgEventCPU"], "0.626105",
"Error: AvgEventCPU is wrong.")
self.assertEqual(perfSection["cpu"]["TotalJobTime"], "23.5703",
"Error: TotalJobTime is wrong.")
self.assertEqual(perfSection["storage"]["readTotalMB"], 39.6166,
"Error: readTotalMB is wrong.")
self.assertEqual(perfSection["storage"]["readMaxMSec"], 320.653,
"Error: readMaxMSec is wrong")
self.assertEqual(perfSection["memory"]["PeakValueRss"], "492.293",
"Error: PeakValueRss is wrong.")
self.assertEqual(perfSection["memory"]["PeakValueVsize"], "643.281",
"Error: PeakValueVsize is wrong.")
failedReport = Report()
failedReport.unpersist(os.path.join(getWMBASE(),
"test/python/WMCore_t/JobStateMachine_t/FailedReport.pkl"))
change.propagate([testJobB], "complete", "executing")
testJobB["fwjr"] = failedReport
testJobB["retry_count"] += 1
change.propagate([testJobB], "success", "complete")
transitions = change.listTransitionsForDashboard()
self.assertEqual(len(transitions), 1,
"Error: Wrong number of transitions.")
self.assertEqual(transitions[0]["exitCode"], "8020",
"Error: Wrong exit code.")
return
config.component_("TaskArchiver")
config.TaskArchiver.namespace = "WMComponent.TaskArchiver.TaskArchiver"
config.TaskArchiver.componentDir = config.General.workDir + "/TaskArchiver"
config.TaskArchiver.logLevel = globalLogLevel
config.TaskArchiver.pollInterval = 240
config.TaskArchiver.timeOut = workflowArchiveTimeout
config.TaskArchiver.useWorkQueue = True
config.TaskArchiver.workloadSummaryCouchURL = workloadSummaryURL
config.webapp_("WMBSService")
config.WMBSService.default_expires = 0
config.WMBSService.componentDir = os.path.join(config.General.workDir, "WMBSService")
config.WMBSService.Webtools.port = wmbsServicePort
config.WMBSService.Webtools.host = serverHostName
config.WMBSService.Webtools.environment = "devel"
config.WMBSService.templates = os.path.join(getWMBASE(), "src/templates/WMCore/WebTools")
config.WMBSService.admin = config.Agent.contact
config.WMBSService.title = "WMBS Data Service"
config.WMBSService.description = "Provide WMBS related service call"
config.WMBSService.section_("security")
config.WMBSService.security.dangerously_insecure = True
config.WMBSService.section_("views")
active = config.WMBSService.views.section_("active")
wmbs = active.section_("wmbs")
wmbs.object = "WMCore.WebTools.RESTApi"
wmbs.templates = os.path.join(getWMBASE(), "src/templates/WMCore/WebTools/")
wmbs.section_("model")
wmbs.model.object = "WMCore.HTTPFrontEnd.WMBS.WMBSRESTModel"
wmbs.section_("formatter")
import thread
thread.stack_size(128*1024)
import socket
import re
import WMCore.HTTPFrontEnd.RequestManager.ReqMgrConfiguration as ReqMgrConfig
__all__ = []
from WMCore.WMInit import getWMBASE
import os.path
INSTALL = getWMBASE()
HOST = socket.getfqdn().lower()
COUCH = "https://%s/couchdb" % HOST
ADD_MONITOR_FLAG = False
if re.match(r"^vocms0(?:13[689]|140|16[135]|30[67]|318)\.cern\.ch$", HOST):
COUCH = "https://cmsweb.cern.ch/couchdb"
elif re.match(r"^vocms0(?:13[12])\.cern\.ch$", HOST):
COUCH = "https://cmsweb-testbed.cern.ch/couchdb"
elif re.match(r"^vocms0127\.cern\.ch$", HOST):
COUCH = "https://cmsweb-dev.cern.ch/couchdb"
config = ReqMgrConfig.reqMgrConfig(installation=INSTALL,
couchurl = COUCH, addMonitor = ADD_MONITOR_FLAG)
TEMPLATES = os.path.normpath(os.path.join(INSTALL, '../../../data/templates/WMCore/WebTools'))
JAVASCRIPT_PATH = os.path.normpath(os.path.join(INSTALL, '../../../data/javascript'))
HTML_PATH = os.path.normpath(os.path.join(INSTALL, '../../../data/html'))
config.webapp_("reqmgr")
config.reqmgr.html = os.path.join(HTML_PATH, 'RequestManager')
def testA(self):
try:
getWMBASE()
except:
self.fail("Error calling WMInit.getWMBASE")
def __init__(self, config):
self.config = config
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger,
dbinterface = myThread.dbi)
self.locationAction = daoFactory(classname = "Locations.GetSiteInfo")
self.packageDir = None
if os.path.exists(os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')):
self.unpacker = os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = None
self.submitDir = None
self.removeTime = getattr(config.BossAir, 'removeTime', 60)
self.multiTasks = getattr(config.BossAir, 'multicoreTaskTypes', [])
self.useGSite = getattr(config.BossAir, 'useGLIDEINSites', False)
self.submitWMSMode = getattr(config.BossAir, 'submitWMSMode', False)
self.errorThreshold= getattr(config.BossAir, 'submitErrorThreshold', 10)
self.errorCount = 0
self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
# Build ourselves a pool
self.pool = []
self.input = None
self.result = None
self.nProcess = getattr(self.config.BossAir, 'nCondorProcesses', 4)
# Set up my proxy and glexec stuff
self.setupScript = getattr(config.BossAir, 'UISetupScript', None)
self.proxy = None
self.serverCert = getattr(config.BossAir, 'delegatedServerCert', None)
self.serverKey = getattr(config.BossAir, 'delegatedServerKey', None)
self.myproxySrv = getattr(config.BossAir, 'myproxyServer', None)
self.proxyDir = getattr(config.BossAir, 'proxyDir', '/tmp/')
self.serverHash = getattr(config.BossAir, 'delegatedServerHash', None)
self.glexecPath = getattr(config.BossAir, 'glexecPath', None)
self.glexecWrapScript = getattr(config.BossAir, 'glexecWrapScript', None)
self.glexecUnwrapScript = getattr(config.BossAir, 'glexecUnwrapScript', None)
self.jdlProxyFile = None # Proxy name to put in JDL (owned by submit user)
self.glexecProxyFile = None # Copy of same file owned by submit user
if self.glexecPath:
if not (self.myproxySrv and self.proxyDir):
raise WMException('glexec requires myproxyServer and proxyDir to be set.')
if self.myproxySrv:
if not (self.serverCert and self.serverKey):
raise WMException('MyProxy server requires serverCert and serverKey to be set.')
# Make the directory for the proxies
if self.proxyDir and not os.path.exists(self.proxyDir):
logging.debug("proxyDir not found: creating it.")
try:
os.makedirs(self.proxyDir, 01777)
except Exception, ex:
msg = "Error: problem when creating proxyDir directory - '%s'" % str(ex)
raise BossAirPluginException(msg)
def __init__(self, config):
self.config = config
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger,
dbinterface = myThread.dbi)
self.locationAction = daoFactory(classname = "Locations.GetSiteInfo")
self.packageDir = None
if os.path.exists(os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')):
self.unpacker = os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = None
self.submitDir = None
self.removeTime = getattr(config.BossAir, 'removeTime', 60)
self.useGSite = getattr(config.BossAir, 'useGLIDEINSites', False)
self.submitWMSMode = getattr(config.BossAir, 'submitWMSMode', False)
self.errorThreshold= getattr(config.BossAir, 'submitErrorThreshold', 10)
self.errorCount = 0
self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
# Required for global pool accounting
self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops")
# Build ourselves a pool
self.pool = []
self.input = None
self.result = None
self.nProcess = getattr(self.config.BossAir, 'nCondorProcesses', 4)
# Set up my proxy and glexec stuff
self.setupScript = getattr(config.BossAir, 'UISetupScript', None)
self.proxy = None
self.serverCert = getattr(config.BossAir, 'delegatedServerCert', None)
self.serverKey = getattr(config.BossAir, 'delegatedServerKey', None)
self.myproxySrv = getattr(config.BossAir, 'myproxyServer', None)
self.proxyDir = getattr(config.BossAir, 'proxyDir', '/tmp/')
self.serverHash = getattr(config.BossAir, 'delegatedServerHash', None)
self.glexecPath = getattr(config.BossAir, 'glexecPath', None)
self.glexecWrapScript = getattr(config.BossAir, 'glexecWrapScript', None)
self.glexecUnwrapScript = getattr(config.BossAir, 'glexecUnwrapScript', None)
self.jdlProxyFile = None # Proxy name to put in JDL (owned by submit user)
self.glexecProxyFile = None # Copy of same file owned by submit user
if self.glexecPath:
if not (self.myproxySrv and self.proxyDir):
raise WMException('glexec requires myproxyServer and proxyDir to be set.')
if self.myproxySrv:
if not (self.serverCert and self.serverKey):
raise WMException('MyProxy server requires serverCert and serverKey to be set.')
# Make the directory for the proxies
if self.proxyDir and not os.path.exists(self.proxyDir):
logging.debug("proxyDir not found: creating it.")
try:
os.makedirs(self.proxyDir, 0o1777)
except Exception as ex:
msg = "Error: problem when creating proxyDir directory - '%s'" % str(ex)
raise BossAirPluginException(msg)
elif not os.path.isdir(self.proxyDir):
msg = "Error: proxyDir '%s' is not a directory" % self.proxyDir
raise BossAirPluginException(msg)
if self.serverCert and self.serverKey and self.myproxySrv:
self.proxy = self.setupMyProxy()
# Build a request string
self.reqStr = "(Memory >= 1 && OpSys == \"LINUX\" ) && (Arch == \"INTEL\" || Arch == \"X86_64\") && stringListMember(GLIDEIN_CMSSite, DESIRED_Sites) && ((REQUIRED_OS==\"any\") || (GLIDEIN_REQUIRED_OS==REQUIRED_OS))"
if hasattr(config.BossAir, 'condorRequirementsString'):
self.reqStr = config.BossAir.condorRequirementsString
return
def __init__(self, config):
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.locationAction = daoFactory(classname="Locations.GetSiteInfo")
self.packageDir = None
# if agent is running in a container, Unpacker.py must come from a directory
# on the host so the condor schedd can see it
# config.General.workDir should always be bind mounted to the container
if getattr(config.Agent, "isDocker", False):
unpackerPath = os.path.join(config.General.workDir +
"/Docker/WMRuntime/Unpacker.py")
else:
unpackerPath = os.path.join(
getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py')
if os.path.exists(unpackerPath):
self.unpacker = unpackerPath
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = config.JobSubmitter.submitScript
self.defaultTaskPriority = getattr(config.BossAir,
'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200)
self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500)
# Required for global pool accounting
self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser',
"cmsdataops")
# Build a requirement string. All CMS resources match DESIRED_Sites on the START
# expression side; however, there are currently some resources (T2_CH_CERN_HLT)
# that are missing the REQUIRED_OS logic. Hence, we duplicate it here.
# TODO(bbockelm): Remove reqStr once HLT has upgraded.
self.reqStr = (
'((REQUIRED_OS=?="any") || '
'(GLIDEIN_REQUIRED_OS =?= "any") || '
'stringListMember(GLIDEIN_REQUIRED_OS, REQUIRED_OS)) && '
'(AuthenticatedIdentity =!= "*****@*****.**")')
if hasattr(config.BossAir, 'condorRequirementsString'):
self.reqStr = config.BossAir.condorRequirementsString
# x509 proxy handling
proxy = Proxy({'logger': myThread.logger})
self.x509userproxy = proxy.getProxyFilename()
# These are added now by the condor client
#self.x509userproxysubject = proxy.getSubject()
#self.x509userproxyfqan = proxy.getAttributeFromProxy(self.x509userproxy)
return
config.Agent.useHeartbeat = False
config.section_("General")
config.General.workDir = workDirectory
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = databaseUrl
config.CoreDatabase.socket = databaseSocket
config.webapp_("ReqMgr")
reqMgrUrl = "http://%s:%s" % (serverHostName, reqMgrPort)
config.ReqMgr.componentDir = os.path.join(config.General.workDir, "ReqMgr")
config.ReqMgr.Webtools.host = serverHostName
config.ReqMgr.Webtools.port = reqMgrPort
config.ReqMgr.Webtools.environment = "devel"
config.ReqMgr.templates = os.path.join(getWMBASE(),
"src/templates/WMCore/WebTools/RequestManager")
config.ReqMgr.requestor = userName
config.ReqMgr.admin = userEmail
config.ReqMgr.title = "CMS Request Manager"
config.ReqMgr.description = "CMS Request Manager"
config.ReqMgr.couchURL = couchURL
config.ReqMgr.default_expires = 0
config.ReqMgr.yuiroot = yuiRoot
config.ReqMgr.couchUrl = couchURL
config.ReqMgr.configDBName = configCacheDBName
config.ReqMgr.section_("security")
config.ReqMgr.security.dangerously_insecure = True
views = config.ReqMgr.section_('views')
def testDuplicateJobReports(self):
"""
_testDuplicateJobReports_
Verify that everything works correctly if a job report is added to the
database more than once.
"""
change = ChangeState(self.config, "changestate_t")
locationAction = self.daoFactory(classname = "Locations.New")
locationAction.execute("site1", seName = "somese.cern.ch")
testWorkflow = Workflow(spec = "spec.xml", owner = "Steve",
name = "wf001", task = "Test")
testWorkflow.create()
testFileset = Fileset(name = "TestFileset")
testFileset.create()
testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"]))
testFile.create()
testFileset.addFile(testFile)
testFileset.commit()
testSubscription = Subscription(fileset = testFileset,
workflow = testWorkflow)
testSubscription.create()
splitter = SplitterFactory()
jobFactory = splitter(package = "WMCore.WMBS",
subscription = testSubscription)
jobGroup = jobFactory(files_per_job = 1)[0]
assert len(jobGroup.jobs) == 1, \
"Error: Splitting should have created one job."
testJobA = jobGroup.jobs[0]
testJobA["user"] = "******"
testJobA["group"] = "DMWM"
testJobA["taskType"] = "Processing"
change.propagate([testJobA], 'created', 'new')
myReport = Report()
reportPath = os.path.join(getWMBASE(),
"test/python/WMCore_t/JobStateMachine_t/Report.pkl")
myReport.unpersist(reportPath)
testJobA["fwjr"] = myReport
change.propagate([testJobA], 'executing', 'created')
change.propagate([testJobA], 'executing', 'created')
changeStateDB = self.couchServer.connectDatabase(dbname = "changestate_t/fwjrs")
allDocs = changeStateDB.document("_all_docs")
self.assertEqual(len(allDocs["rows"]), 2,
"Error: Wrong number of documents")
for resultRow in allDocs["rows"]:
if resultRow["id"] != "_design/FWJRDump":
fwjrDoc = changeStateDB.document(resultRow["id"])
break
return
def __init__(self, config):
self.config = config
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger,
dbinterface = myThread.dbi)
self.locationAction = daoFactory(classname = "Locations.GetSiteInfo")
self.packageDir = None
if os.path.exists(os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')):
self.unpacker = os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = None
self.submitDir = None
self.removeTime = getattr(config.BossAir, 'removeTime', 60)
self.useGSite = getattr(config.BossAir, 'useGLIDEINSites', False)
self.submitWMSMode = getattr(config.BossAir, 'submitWMSMode', False)
self.errorThreshold= getattr(config.BossAir, 'submitErrorThreshold', 10)
self.errorCount = 0
self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
# Required for global pool accounting
self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops")
# Build ourselves a pool
self.pool = []
self.input = None
self.result = None
self.nProcess = getattr(self.config.BossAir, 'nCondorProcesses', 4)
# Set up my proxy and glexec stuff
self.setupScript = getattr(config.BossAir, 'UISetupScript', None)
self.proxy = None
self.serverCert = getattr(config.BossAir, 'delegatedServerCert', None)
self.serverKey = getattr(config.BossAir, 'delegatedServerKey', None)
self.myproxySrv = getattr(config.BossAir, 'myproxyServer', None)
self.proxyDir = getattr(config.BossAir, 'proxyDir', '/tmp/')
self.serverHash = getattr(config.BossAir, 'delegatedServerHash', None)
self.glexecPath = getattr(config.BossAir, 'glexecPath', None)
self.glexecWrapScript = getattr(config.BossAir, 'glexecWrapScript', None)
self.glexecUnwrapScript = getattr(config.BossAir, 'glexecUnwrapScript', None)
self.jdlProxyFile = None # Proxy name to put in JDL (owned by submit user)
self.glexecProxyFile = None # Copy of same file owned by submit user
if self.glexecPath:
if not (self.myproxySrv and self.proxyDir):
raise WMException('glexec requires myproxyServer and proxyDir to be set.')
if self.myproxySrv:
if not (self.serverCert and self.serverKey):
raise WMException('MyProxy server requires serverCert and serverKey to be set.')
# Make the directory for the proxies
if self.proxyDir and not os.path.exists(self.proxyDir):
logging.debug("proxyDir not found: creating it.")
try:
os.makedirs(self.proxyDir, 0o1777)
except Exception as ex:
msg = "Error: problem when creating proxyDir directory - '%s'" % str(ex)
raise BossAirPluginException(msg)
elif not os.path.isdir(self.proxyDir):
msg = "Error: proxyDir '%s' is not a directory" % self.proxyDir
raise BossAirPluginException(msg)
if self.serverCert and self.serverKey and self.myproxySrv:
self.proxy = self.setupMyProxy()
# Build a request string
self.reqStr = "(Memory >= 1 && OpSys == \"LINUX\" ) && (Arch == \"INTEL\" || Arch == \"X86_64\") && stringListMember(GLIDEIN_CMSSite, DESIRED_Sites) && ((REQUIRED_OS==\"any\") || (GLIDEIN_REQUIRED_OS==REQUIRED_OS))"
if hasattr(config.BossAir, 'condorRequirementsString'):
self.reqStr = config.BossAir.condorRequirementsString
return
def getConfig(self):
"""
_getConfig_
This is the global test configuration object
"""
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
# Now the CoreDatabase information
# This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobCreator
config.component_("JobCreator")
config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
config.JobCreator.logLevel = 'DEBUG'
config.JobCreator.maxThreads = 1
config.JobCreator.UpdateFromResourceControl = True
config.JobCreator.pollInterval = 10
config.JobCreator.jobCacheDir = self.testDir
config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control
config.JobCreator.workerThreads = 2
config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components')
# JobSubmitter
config.component_("JobSubmitter")
config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter'
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'CondorGlobusPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'DEBUG'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
# JobAccountant
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobAccountant.logLevel = 'INFO'
# JobArchiver
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobArchiver.numberOfJobsToCluster = 1000
# Task Archiver
config.component_("TaskArchiver")
config.TaskArchiver.componentDir = self.testInit.generateWorkDir()
config.TaskArchiver.WorkQueueParams = {}
config.TaskArchiver.pollInterval = 60
config.TaskArchiver.logLevel = 'INFO'
config.TaskArchiver.timeOut = 0
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL',
'mnorman:[email protected]:5984')
config.JobStateMachine.couchDBName = "mnorman_test"
# Needed, because this is a test
os.makedirs(config.JobSubmitter.submitDir)
return config
def tearDown(self):
sys.path.remove(os.path.join(getWMBASE(), "test/python/WMCore_t/WMRuntime_t/Scripts_t"))
self.testInit.delWorkDir()
os.unsetenv("WMAGENT_SITE_CONFIG_OVERRIDE")