def __init__(self, config): self.config = config BasePlugin.__init__(self, config) self.locationDict = {} myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.locationAction = daoFactory(classname="Locations.GetSiteInfo") self.packageDir = None if os.path.exists(os.path.join(getWMBASE(), "src/python/WMCore/WMRuntime/Unpacker.py")): self.unpacker = os.path.join(getWMBASE(), "src/python/WMCore/WMRuntime/Unpacker.py") else: self.unpacker = os.path.join(getWMBASE(), "WMCore/WMRuntime/Unpacker.py") self.agent = getattr(config.Agent, "agentName", "WMAgent") self.sandbox = None self.scriptFile = None self.submitDir = None self.removeTime = getattr(config.BossAir, "removeTime", 60) self.multiTasks = getattr(config.BossAir, "multicoreTaskTypes", []) # Build ourselves a pool self.pool = [] self.input = multiprocessing.Queue() self.result = multiprocessing.Queue() self.nProcess = getattr(self.config.BossAir, "nCondorProcesses", 4) return
def __init__(self, config): BasePlugin.__init__(self, config) self.locationDict = {} myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.locationAction = daoFactory(classname="Locations.GetSiteInfo") self.packageDir = None if os.path.exists( os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py')): self.unpacker = os.path.join( getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py') else: self.unpacker = os.path.join(getWMBASE(), 'WMCore/WMRuntime/Unpacker.py') self.agent = getattr(config.Agent, 'agentName', 'WMAgent') self.sandbox = None self.scriptFile = config.JobSubmitter.submitScript self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0) self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7) self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200) self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500) # Required for global pool accounting self.acctGroup = getattr(config.BossAir, 'acctGroup', "production") self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops") # Build a requirement string. All CMS resources match DESIRED_Sites on the START # expression side; however, there are currently some resources (T2_CH_CERN_HLT) # that are missing the REQUIRED_OS logic. Hence, we duplicate it here. # TODO(bbockelm): Remove reqStr once HLT has upgraded. self.reqStr = ( '((REQUIRED_OS=?="any") || ' '(GLIDEIN_REQUIRED_OS =?= "any") || ' 'stringListMember(GLIDEIN_REQUIRED_OS, REQUIRED_OS)) && ' '(AuthenticatedIdentity =!= "*****@*****.**")') if hasattr(config.BossAir, 'condorRequirementsString'): self.reqStr = config.BossAir.condorRequirementsString # x509 proxy handling proxy = Proxy({'logger': myThread.logger}) self.x509userproxy = proxy.getProxyFilename() self.x509userproxysubject = proxy.getSubject() self.x509userproxyfqan = proxy.getAttributeFromProxy( self.x509userproxy) # Remove the x509 ads if the job is matching a volunteer resource self.x509Expr = 'ifThenElse("$$(GLIDEIN_CMSSite)" =?= "T3_CH_Volunteer",undefined,"%s")' return
def injectIntoConfigCache(self, frameworkVersion, scramArch, initCommand, configUrl, configLabel, couchUrl, couchDBName): """ _injectIntoConfigCache_ """ logging.error("Injecting to config cache.\n"); configTempDir = tempfile.mkdtemp() configPath = os.path.join(configTempDir, "cmsswConfig.py") configString = urllib.urlopen(configUrl).read(-1) configFile = open(configPath, "w") configFile.write(configString) configFile.close() scramTempDir = tempfile.mkdtemp() wmcoreBase = getWMBASE() envPath = os.path.normpath(os.path.join(getWMBASE(), "../../../../../../../../apps/wmagent/etc/profile.d/init.sh")) scram = Scram(version = frameworkVersion, architecture = scramArch, directory = scramTempDir, initialise = initCommand, envCmd = "source %s" % envPath) scram.project() scram.runtime() scram("python2.6 %s/../../../bin/inject-to-config-cache %s %s PromptSkimmer cmsdataops %s %s None" % (wmcoreBase, couchUrl, couchDBName, configPath, configLabel)) shutil.rmtree(configTempDir) shutil.rmtree(scramTempDir) return
def __init__(self, config): BasePlugin.__init__(self, config) self.locationDict = {} myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.locationAction = daoFactory(classname="Locations.GetSiteInfo") self.packageDir = None # if agent is running in a container, Unpacker.py must come from a directory # on the host so the condor schedd can see it # config.General.workDir should always be bind mounted to the container if getattr(config.Agent, "isDocker", False): unpackerPath = os.path.join(config.General.workDir + "/Docker/WMRuntime/Unpacker.py") else: unpackerPath = os.path.join( getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py') if os.path.exists(unpackerPath): self.unpacker = unpackerPath else: self.unpacker = os.path.join(getWMBASE(), 'WMCore/WMRuntime/Unpacker.py') self.agent = getattr(config.Agent, 'agentName', 'WMAgent') self.sandbox = None self.scriptFile = config.JobSubmitter.submitScript self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0) self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7) self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200) self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500) # Required for global pool accounting self.acctGroup = getattr(config.BossAir, 'acctGroup', "production") self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops") if hasattr(config.BossAir, 'condorRequirementsString'): self.reqStr = config.BossAir.condorRequirementsString else: self.reqStr = None # x509 proxy handling proxy = Proxy({'logger': myThread.logger}) self.x509userproxy = proxy.getProxyFilename() # These are added now by the condor client #self.x509userproxysubject = proxy.getSubject() #self.x509userproxyfqan = proxy.getAttributeFromProxy(self.x509userproxy) return
def __init__(self, config): BasePlugin.__init__(self, config) self.locationDict = {} myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.locationAction = daoFactory(classname="Locations.GetSiteInfo") self.packageDir = None if os.path.exists(os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py')): self.unpacker = os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py') else: self.unpacker = os.path.join(getWMBASE(), 'WMCore/WMRuntime/Unpacker.py') self.agent = getattr(config.Agent, 'agentName', 'WMAgent') self.sandbox = None self.scriptFile = config.JobSubmitter.submitScript self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0) self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7) self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200) self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500) # Required for global pool accounting self.acctGroup = getattr(config.BossAir, 'acctGroup', "production") self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops") # Build a requirement string. All CMS resources match DESIRED_Sites on the START # expression side; however, there are currently some resources (T2_CH_CERN_HLT) # that are missing the REQUIRED_OS logic. Hence, we duplicate it here. # TODO(bbockelm): Remove reqStr once HLT has upgraded. self.reqStr = ('((REQUIRED_OS=?="any") || ' '(GLIDEIN_REQUIRED_OS =?= "any") || ' 'stringListMember(GLIDEIN_REQUIRED_OS, REQUIRED_OS)) && ' '(AuthenticatedIdentity =!= "*****@*****.**")') if hasattr(config.BossAir, 'condorRequirementsString'): self.reqStr = config.BossAir.condorRequirementsString # x509 proxy handling proxy = Proxy({'logger': myThread.logger}) self.x509userproxy = proxy.getProxyFilename() self.x509userproxysubject = proxy.getSubject() self.x509userproxyfqan = proxy.getAttributeFromProxy(self.x509userproxy) # Remove the x509 ads if the job is matching a volunteer resource self.x509Expr = 'ifThenElse("$$(GLIDEIN_CMSSite)" =?= "T3_CH_Volunteer",undefined,"%s")' return
def __init__(self, config): BasePlugin.__init__(self, config) self.locationDict = {} myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.locationAction = daoFactory(classname="Locations.GetSiteInfo") self.packageDir = None if os.path.exists( os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py')): self.unpacker = os.path.join( getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py') else: self.unpacker = os.path.join(getWMBASE(), 'WMCore/WMRuntime/Unpacker.py') self.agent = getattr(config.Agent, 'agentName', 'WMAgent') self.sandbox = None self.scriptFile = config.JobSubmitter.submitScript self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0) self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7) self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200) # Required for global pool accounting self.acctGroup = getattr(config.BossAir, 'acctGroup', "production") self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops") # Build a requirement string self.reqStr = "stringListMember(GLIDEIN_CMSSite, DESIRED_Sites) && ((REQUIRED_OS=?=\"any\") || (GLIDEIN_REQUIRED_OS=?=REQUIRED_OS)) && (TARGET.Cpus >= RequestCpus)" if hasattr(config.BossAir, 'condorRequirementsString'): self.reqStr = config.BossAir.condorRequirementsString # x509 proxy handling proxy = Proxy({'logger': myThread.logger}) self.x509userproxy = proxy.getProxyFilename() self.x509userproxysubject = proxy.getSubject() return
def testB_addingConfigsAndTweaks(self): """ _addingConfigsAndTweaks_ Test adding config files and tweak files """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." attach = "Hello, I am an attachment" configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCache.createUserGroup(groupname = "testGroup", username = '******') configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.attachments['attach1'] = attach psetPath = os.path.join(getWMBASE(), "test/python/WMCore_t/Cache_t/PSet.txt") configCache.addConfig(newConfig = psetPath, psetHash = None) configCache.setLabel("sample-label") configCache.setDescription("describe this config here") configCache.save() configString1 = configCache.getConfig() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test', id = configCache.getCouchID(), rev = configCache.getCouchRev()) configCache2.loadByID(configCache.getCouchID()) configString2 = configCache2.getConfig() self.assertEqual(configString1, configString2) self.assertEqual(configCache2.attachments.get('attach1', None), attach) configCache.delete() return
def testC_testViews(self): """ _testViews_ Prototype test for what should be a lot of other tests. """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." attach = "Hello, I am an attachment" configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCache.createUserGroup(groupname = "testGroup", username = '******') configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.attachments['attach1'] = attach configCache.document['md5_hash'] = "somemd5" psetPath = os.path.join(getWMBASE(), "test/python/WMCore_t/Cache_t/PSet.txt") configCache.addConfig(newConfig = psetPath, psetHash = None) configCache.save() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCache2.document['md5_hash'] = configCache.document['md5_hash'] configCache2.load() self.assertEqual(configCache2.attachments.get('attach1', None), attach) configCache2.delete() return
def testTrivialFileCatalog(self): """ Run some simple tests on reading a trivialFileCatalog """ tfcFilename = os.path.join(getWMBASE(), "test/python/WMCore_t/Storage_t", "T1_US_FNAL_TrivialFileCatalog.xml") if not os.path.exists(tfcFilename): raise Exception("No TrivialFileCatalog found!") tfcInstance = readTFC(tfcFilename) self.assertEqual(type(tfcInstance), type(TrivialFileCatalog())) # Look for similarities in each node of the TFC file for mapping in ['lfn-to-pfn', 'pfn-to-lfn']: for x in tfcInstance[mapping]: self.assertEqual(x.has_key('path-match-expr'), True) self.assertEqual(x.has_key('path-match'), True) self.assertEqual(x.has_key('protocol'), True) self.assertEqual(x.has_key('result'), True) self.assertEqual(x.has_key('chain'), True) self.assertEqual(x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'], True, 'Could not find protocol %s' % (x['protocol'])) self.assertEqual(x['chain'], None, 'Invalid chain %s' % (x['chain']))
def injectIntoConfigCache(self, frameworkVersion, scramArch, initCommand, configUrl, configLabel, couchUrl, couchDBName): """ _injectIntoConfigCache_ """ configTempDir = tempfile.mkdtemp() configPath = os.path.join(configTempDir, "cmsswConfig.py") configString = urllib.urlopen(configUrl).read(-1) configFile = open(configPath, "w") configFile.write(configString) configFile.close() scramTempDir = tempfile.mkdtemp() scram = Scram(version = frameworkVersion, architecture = scramArch, directory = scramTempDir, initialise = initCommand) scram.project() scram.runtime() wmcoreBase = getWMBASE() scram("python2.6 %s/bin/inject-to-config-cache %s %s PromptSkimmer cmsdataops %s %s None" % (wmcoreBase, couchUrl, couchDBName, configPath, configLabel)) shutil.rmtree(configTempDir) shutil.rmtree(scramTempDir) return
def initialize(self): self.config = DefaultConfig() self.config.UnitTests.templates = getWMBASE() + '/src/templates/WMCore/WebTools' self.config.Webtools.section_('server') self.config.Webtools.server.socket_timeout = 1 self.urlbase = self.config.getServerUrl() self.cache_path = tempfile.mkdtemp()
def thrashCouch(): """ _thrashCouch_ """ jobs = {"new": set(), "created": set(), "executing": set(), "complete": set(), "success": set(), "cleanout": set()} config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"]) changeState = ChangeState(config) myReport = Report() myReport.unpersist(os.path.join(getWMBASE(), "test/python/WMComponent_t/JobAccountant_t/fwjrs/LoadTest00.pkl")) for i in range(500): jobs = createJobs() changeState.recordInCouch(jobs, "created", "new") changeState.recordInCouch(jobs, "executing", "created") changeState.recordInCouch(jobs, "complete", "executing") for job in jobs: job["fwjr"] = myReport changeState.recordInCouch(jobs, "success", "complete") for job in jobs: job["fwjr"] = None changeState.recordInCouch(jobs, "cleanout", "success") #time.sleep(10) return
def __init__(self, config): BasePlugin.__init__(self, config) self.locationDict = {} myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.locationAction = daoFactory(classname="Locations.GetSiteInfo") self.packageDir = None if os.path.exists(os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py')): self.unpacker = os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py') else: self.unpacker = os.path.join(getWMBASE(), 'WMCore/WMRuntime/Unpacker.py') self.agent = getattr(config.Agent, 'agentName', 'WMAgent') self.sandbox = None self.scriptFile = config.JobSubmitter.submitScript self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0) self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7) self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200) self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500) # Required for global pool accounting self.acctGroup = getattr(config.BossAir, 'acctGroup', "production") self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops") # Build a requirement string self.reqStr = ('stringListMember(GLIDEIN_CMSSite, DESIRED_Sites) ' '&& ((REQUIRED_OS=?="any") || stringListMember(GLIDEIN_REQUIRED_OS, REQUIRED_OS))' '&& (TARGET.Cpus >= RequestCpus)') if hasattr(config.BossAir, 'condorRequirementsString'): self.reqStr = config.BossAir.condorRequirementsString # x509 proxy handling proxy = Proxy({'logger': myThread.logger}) self.x509userproxy = proxy.getProxyFilename() self.x509userproxysubject = proxy.getSubject() return
def testVanderbiltSiteLocalConfig(self): """ _testFNALSiteLocalConfig_ Verify that the FNAL site config file is parsed correctly. """ vandyConfigFileName = os.path.join(getWMBASE(), "test/python/WMCore_t/Storage_t", "T3_US_Vanderbilt_SiteLocalConfig.xml") mySiteConfig = SiteLocalConfig(vandyConfigFileName) assert mySiteConfig.siteName == "T3_US_Vanderbilt", \ "Error: Wrong site name." assert len(mySiteConfig.eventData.keys()) == 1, \ "Error: Wrong number of event data keys." assert mySiteConfig.eventData["catalog"] == "trivialcatalog_file://gpfs1/grid/grid-app/cmssoft/cms/SITECONF/local/PhEDEx/storage.xml?protocol=direct", \ "Eroror: Event data catalog is wrong." goldenServers = ["http://cmsfrontier.cern.ch:8000/FrontierInt", "http://cmsfrontier1.cern.ch:8000/FrontierInt", "http://cmsfrontier2.cern.ch:8000/FrontierInt", "http://cmsfrontier3.cern.ch:8000/FrontierInt"] for frontierServer in mySiteConfig.frontierServers: assert frontierServer in goldenServers, \ "Error: Unknown server: %s" % frontierServer goldenServers.remove(frontierServer) assert len(goldenServers) == 0, \ "Error: Missing frontier servers." goldenProxies = ["http://se1.accre.vanderbilt.edu:3128"] for frontierProxy in mySiteConfig.frontierProxies: assert frontierProxy in goldenProxies, \ "Error: Unknown proxy: %s" % frontierProxy goldenProxies.remove(frontierProxy) assert len(goldenProxies) == 0, \ "Error: Missing proxy servers." assert mySiteConfig.localStageOut["se-name"] == "se1.accre.vanderbilt.edu", \ "Error: Wrong se name from local stageout." assert mySiteConfig.localStageOut["command"] == "srmv2", \ "Error: Wrong stage out command." assert mySiteConfig.localStageOut["catalog"] == "trivialcatalog_file://gpfs1/grid/grid-app/cmssoft/cms/SITECONF/local/PhEDEx/storage.xml?protocol=srmv2", \ "Error: TFC catalog is not correct." assert len(mySiteConfig.fallbackStageOut) == 1, \ "Error: Incorrect number of fallback stageout methods" assert mySiteConfig.fallbackStageOut[0]["command"] == "srmv2-lcg", \ "Error: Incorrect fallback command." assert mySiteConfig.fallbackStageOut[0]["se-name"] == "se1.accre.vanderbilt.edu", \ "Error: Incorrect fallback SE." assert mySiteConfig.fallbackStageOut[0]["lfn-prefix"] == "srm://se1.accre.vanderbilt.edu:6288/srm/v2/server?SFN=", \ "Error: Incorrect fallback LFN prefix." return
def testFNALSiteLocalConfig(self): """ _testFNALSiteLocalConfig_ Verify that the FNAL site config file is parsed correctly. """ fnalConfigFileName = os.path.join(getWMBASE(), "test/python/WMCore_t/Storage_t", "T1_US_FNAL_SiteLocalConfig.xml") mySiteConfig = SiteLocalConfig(fnalConfigFileName) assert mySiteConfig.siteName == "T1_US_FNAL", \ "Error: Wrong site name." assert len(mySiteConfig.eventData.keys()) == 1, \ "Error: Wrong number of event data keys." assert mySiteConfig.eventData["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \ "Eroror: Event data catalog is wrong." goldenServers = ["http://cmsfrontier.cern.ch:8000/FrontierInt", "http://cmsfrontier.cern.ch:8000/FrontierInt", "http://cmsfrontier1.cern.ch:8000/FrontierInt", "http://cmsfrontier2.cern.ch:8000/FrontierInt", "http://cmsfrontier3.cern.ch:8000/FrontierInt", "http://cmsfrontier4.cern.ch:8000/FrontierInt"] for frontierServer in mySiteConfig.frontierServers: assert frontierServer in goldenServers, \ "Error: Unknown server: %s" % frontierServer goldenServers.remove(frontierServer) assert len(goldenServers) == 0, \ "Error: Missing frontier servers." goldenProxies = ["http://cmsfrontier1.fnal.gov:3128", "http://cmsfrontier2.fnal.gov:3128", "http://cmsfrontier3.fnal.gov:3128", "http://cmsfrontier4.fnal.gov:3128"] for frontierProxy in mySiteConfig.frontierProxies: assert frontierProxy in goldenProxies, \ "Error: Unknown proxy: %s" % frontierProxy goldenProxies.remove(frontierProxy) assert len(goldenProxies) == 0, \ "Error: Missing proxy servers." assert mySiteConfig.localStageOut["se-name"] == "cmssrm.fnal.gov", \ "Error: Wrong se name from local stageout." assert mySiteConfig.localStageOut["command"] == "dccp-fnal", \ "Error: Wrong stage out command." assert mySiteConfig.localStageOut["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=srmv2", \ "Error: TFC catalog is not correct." assert mySiteConfig.fallbackStageOut == [], \ "Error: Fallback config is incorrect." return
def submit(self, jobs, info=None): jobscript = self.config.JobSubmitter.submitScript unpacker = os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py') xrsl = '+\n' wrapper = gen_wrapper_script() for j in jobs: logging.debug("ARCplugin.submit: got job %s" % str(j)) xrsl += '(' + gen_xrsl(j, wrapper.name, jobscript, unpacker) + ')\n' xrsl_file = tempfile.NamedTemporaryFile(prefix="xrslcode.") xrsl_file.write(xrsl) xrsl_file.flush() cmd = "ngsub %s -c %s" % (xrsl_file.name, jobs[0]['location']) s, output = executeCommand(cmd) logging.debug("executed %s" % cmd) logging.debug("with exit stat %s" % s) logging.debug("and output %s" % output) xrsl_file.close() wrapper.close() subRe = re.compile( "Job submitted with jobid: +(\w+://([a-zA-Z0-9.-]+)(:\d+)?(/.*)?/\d+)" ) lines = output.split('\n') n = 0 failure = [] success = [] for j in jobs: if n < len(lines): m = re.match(subRe, lines[n]) else: m = None if m: j['gridid'] = m.group(1) success.append(j) logging.debug("Successful job %s" % j['gridid']) else: failure.append(j) logging.debug("Failed job") n += 1 logging.info("%i successful job submissions, %i failures" % (len(success), len(failure))) return success, failure
def testLoadingConfigFromOverridenEnvVarriable(self): """ test SiteLocalConfig module method loadSiteLocalConfig when loading site config from location defined by WMAGENT_SITE_CONFIG_OVERRIDE env. variable """ vandyConfigFileName = os.path.join(getWMBASE(), "test/python/WMCore_t/Storage_t", "T3_US_Vanderbilt_SiteLocalConfig.xml") os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = vandyConfigFileName mySiteConfig = loadSiteLocalConfig() self.assertEqual(mySiteConfig.siteName, "T3_US_Vanderbilt", "Error: Wrong site name.")
def injectIntoConfigCache(frameworkVersion, scramArch, initCommand, configUrl, configLabel, couchUrl, couchDBName, envPath=None, binPath=None): """ _injectIntoConfigCache_ """ logging.info("Injecting to config cache.\n") configTempDir = tempfile.mkdtemp() configPath = os.path.join(configTempDir, "cmsswConfig.py") configString = urllib.urlopen(fixCVSUrl(configUrl)).read(-1) configFile = open(configPath, "w") configFile.write(configString) configFile.close() scramTempDir = tempfile.mkdtemp() wmcoreBase = getWMBASE() if not envPath: envPath = os.path.normpath( os.path.join( wmcoreBase, "../../../../../../../../apps/wmagent/etc/profile.d/init.sh")) scram = Scram(version=frameworkVersion, architecture=scramArch, directory=scramTempDir, initialise=initCommand, envCmd="source %s" % envPath) scram.project() scram.runtime() if not binPath: scram( "python2.6 %s/../../../bin/inject-to-config-cache %s %s PromptSkimmer cmsdataops %s %s None" % (wmcoreBase, couchUrl, couchDBName, configPath, configLabel)) else: scram( "python2.6 %s/inject-to-config-cache %s %s PromptSkimmer cmsdataops %s %s None" % (binPath, couchUrl, couchDBName, configPath, configLabel)) shutil.rmtree(configTempDir) shutil.rmtree(scramTempDir) return
def __init__(self, config): self.config = config BasePlugin.__init__(self, config) self.packageDir = None self.unpacker = os.path.join(getWMBASE(), 'WMCore/WMRuntime/Unpacker.py') self.agent = config.Agent.agentName self.sandbox = None self.scriptFile = None self.queue = None self.resourceReq = None self.jobGroup = None return
def __init__(self, config): self.config = config BasePlugin.__init__(self, config) self.packageDir = None self.unpacker = os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py') self.agent = config.Agent.agentName self.sandbox = None self.scriptFile = None self.queue = None self.resourceReq = None self.jobGroup = None return
def submit(self, jobs, info = None): jobscript = self.config.JobSubmitter.submitScript unpacker = os.path.join(getWMBASE(),'src/python/WMCore/WMRuntime/Unpacker.py') xrsl = '+\n' wrapper = gen_wrapper_script() for j in jobs: logging.debug("ARCplugin.submit: got job %s" % str(j)) xrsl += '(' + gen_xrsl(j, wrapper.name, jobscript, unpacker) + ')\n' xrsl_file = tempfile.NamedTemporaryFile(prefix="xrslcode.") xrsl_file.write(xrsl) xrsl_file.flush() cmd = "ngsub %s -c %s" % (xrsl_file.name, jobs[0]['location']) s, output = executeCommand(cmd) logging.debug("executed %s" % cmd) logging.debug("with exit stat %s" % s) logging.debug("and output %s" % output) xrsl_file.close() wrapper.close() subRe = re.compile("Job submitted with jobid: +(\w+://([a-zA-Z0-9.-]+)(:\d+)?(/.*)?/\d+)") lines = output.split('\n') n = 0 failure = [] success = [] for j in jobs: if n < len(lines): m = re.match(subRe, lines[n]) else: m = None if m: j['gridid'] = m.group(1) success.append(j) logging.debug("Successful job %s" % j['gridid']) else: failure.append(j) logging.debug("Failed job") n += 1 logging.info("%i successful job submissions, %i failures" % (len(success), len(failure))) return success, failure
def __init__(self, config): self.config = config BasePlugin.__init__(self, config) self.packageDir = None self.unpacker = os.path.join(getWMBASE(), 'WMCore/WMRuntime/Unpacker.py') self.agent = config.Agent.agentName self.sandbox = None self.scriptFile = None self.queue = None self.resourceReq = None self.jobGroup = None self.basePrio = getattr(config.BossAir, 'LsfBasePrio', 50) return
def testRoundTrip(self): """ Test PFN, LFN matching upon an existing TFC XML file """ tfc_file = os.path.join(getWMBASE(), "test/python/WMCore_t/Storage_t", "T1_US_FNAL_TrivialFileCatalog.xml") tfc = readTFC(tfc_file) # Check that an lfn goes to an srmv2 pfn and comes back as the same lfn in_lfn = '/store/data/my/data' in_pfn = "srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/data/my/data" out_pfn = tfc.matchLFN('srmv2', in_lfn) self.assertEqual(out_pfn, in_pfn) out_lfn = tfc.matchPFN('srmv2', in_pfn) self.assertEqual(in_lfn, out_lfn) out_pfn = tfc.matchLFN('srmv2', out_lfn) self.assertEqual(in_pfn, out_pfn)
def thrashCouch(): """ _thrashCouch_ """ jobs = { "new": set(), "created": set(), "executing": set(), "complete": set(), "success": set(), "cleanout": set() } config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"]) changeState = ChangeState(config) myReport = Report() myReport.unpersist( os.path.join( getWMBASE(), "test/python/WMComponent_t/JobAccountant_t/fwjrs/LoadTest00.pkl")) for i in range(500): jobs = createJobs() changeState.recordInCouch(jobs, "created", "new") changeState.recordInCouch(jobs, "executing", "created") changeState.recordInCouch(jobs, "complete", "executing") for job in jobs: job["fwjr"] = myReport changeState.recordInCouch(jobs, "success", "complete") for job in jobs: job["fwjr"] = None changeState.recordInCouch(jobs, "cleanout", "success") #time.sleep(10) return
def setUp(self): self.testInit = TestInit(__file__) self.testInit.setLogging() self.testDir = self.testInit.generateWorkDir() sys.path.insert(0, os.path.join(getWMBASE(), "test/python/WMCore_t/WMRuntime_t/Scripts_t"))
def __init__(self, config): self.config = config BasePlugin.__init__(self, config) self.locationDict = {} myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.locationAction = daoFactory(classname = "Locations.GetSiteInfo") self.packageDir = None if os.path.exists(os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py')): self.unpacker = os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py') else: self.unpacker = os.path.join(getWMBASE(), 'WMCore/WMRuntime/Unpacker.py') self.agent = getattr(config.Agent, 'agentName', 'WMAgent') self.sandbox = None self.scriptFile = None self.submitDir = None self.removeTime = getattr(config.BossAir, 'removeTime', 60) self.multiTasks = getattr(config.BossAir, 'multicoreTaskTypes', []) self.useGSite = getattr(config.BossAir, 'useGLIDEINSites', False) self.submitWMSMode = getattr(config.BossAir, 'submitWMSMode', False) self.errorThreshold= getattr(config.BossAir, 'submitErrorThreshold', 10) self.errorCount = 0 self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0) self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7) # Build ourselves a pool self.pool = [] self.input = None self.result = None self.nProcess = getattr(self.config.BossAir, 'nCondorProcesses', 4) # Set up my proxy and glexec stuff self.setupScript = getattr(config.BossAir, 'UISetupScript', None) self.proxy = None self.serverCert = getattr(config.BossAir, 'delegatedServerCert', None) self.serverKey = getattr(config.BossAir, 'delegatedServerKey', None) self.myproxySrv = getattr(config.BossAir, 'myproxyServer', None) self.proxyDir = getattr(config.BossAir, 'proxyDir', '/tmp/') self.serverHash = getattr(config.BossAir, 'delegatedServerHash', None) self.glexecPath = getattr(config.BossAir, 'glexecPath', None) self.glexecWrapScript = getattr(config.BossAir, 'glexecWrapScript', None) self.glexecUnwrapScript = getattr(config.BossAir, 'glexecUnwrapScript', None) self.jdlProxyFile = None # Proxy name to put in JDL (owned by submit user) self.glexecProxyFile = None # Copy of same file owned by submit user if self.glexecPath: if not (self.myproxySrv and self.proxyDir): raise WMException('glexec requires myproxyServer and proxyDir to be set.') if self.myproxySrv: if not (self.serverCert and self.serverKey): raise WMException('MyProxy server requires serverCert and serverKey to be set.') # Make the directory for the proxies if self.proxyDir and not os.path.exists(self.proxyDir): logging.debug("proxyDir not found: creating it.") try: os.makedirs(self.proxyDir, 01777) except Exception, ex: msg = "Error: problem when creating proxyDir directory - '%s'" % str(ex) raise BossAirPluginException(msg)
__all__ = [] import thread import socket import re import os.path import WMCore.HTTPFrontEnd.RequestManager.ReqMgrConfiguration as ReqMgrConfig from WMCore.WMInit import getWMBASE thread.stack_size(128 * 1024) INSTALL = getWMBASE() HOST = socket.getfqdn().lower() COUCH = "https://%s/couchdb" % HOST if re.match(r"^vocms0(?:13[689]|14[0123]|16[135]|30[67]|318)\.cern\.ch$", HOST): COUCH = "https://cmsweb.cern.ch/couchdb" elif re.match(r"^vocms0(?:13[12])\.cern\.ch$", HOST): COUCH = "https://cmsweb-testbed.cern.ch/couchdb" elif re.match(r"^vocms0127\.cern\.ch$", HOST): COUCH = "https://cmsweb-dev.cern.ch/couchdb" elif re.match(r"^vocms0126\.cern\.ch$", HOST): COUCH = "https://cmsweb-sec.cern.ch/couchdb" config = ReqMgrConfig.reqMgrConfig(installation=INSTALL, couchurl=COUCH) TEMPLATES = os.path.normpath( os.path.join(INSTALL, '../../../data/templates/WMCore/WebTools')) JAVASCRIPT_PATH = os.path.normpath(
def testA(self): try: getWMBASE() except: self.fail("Error calling WMInit.getWMBASE")
def testDashboardTransitions(self): """ _testDashboardTransitions_ Verify that the dashboard transitions code works correctly. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = "Test", wfType = "ReDigi") testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased") testSubscription.create() testFileA = File(lfn = "SomeLFNA", events = 1024, size = 2048, locations = set(["somese.cern.ch"])) testFileB = File(lfn = "SomeLFNB", events = 1025, size = 2049, locations = set(["somese.cern.ch"])) testFileA.create() testFileB.create() testFileset.addFile(testFileA) testFileset.addFile(testFileB) testFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 2, \ "Error: Splitting should have created two jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" change.propagate([testJobA, testJobB], "new", "none") change.propagate([testJobA, testJobB], "created", "new") change.propagate([testJobA], "executing", "created") change.propagate([testJobB], "submitfailed", "created") change.propagate([testJobB], "submitcooloff", "submitfailed") transitions = change.listTransitionsForDashboard() self.assertEqual(len(transitions), 1, "Error: Wrong number of transitions: %s" % transitions) self.assertEqual(transitions[0]["name"], testJobA["name"], "Error: Wrong job name.") self.assertEqual(transitions[0]["retryCount"], 0, "Error: Wrong retry count.") self.assertEqual(transitions[0]["newState"], "executing", "Error: Wrong new state.") self.assertEqual(transitions[0]["oldState"], "created", "Error: Wrong old state.") self.assertEqual(transitions[0]["requestName"], "wf001", "Error: Wrong request name.") transitions = change.listTransitionsForDashboard() self.assertEqual(len(transitions), 0, "Error: Wrong number of transitions") change.propagate([testJobB], "created", "submitcooloff") change.propagate([testJobB], "executing", "created") change.propagate([testJobA, testJobB], "complete", "executing") change.propagate([testJobB], "success", "complete") change.propagate([testJobA], "jobfailed", "complete") timestamp = int(time.time()) transitions = change.listTransitionsForDashboard() for transition in transitions: self.assertTrue(timestamp - 10 < transition["timestamp"] and timestamp + 10 > transition["timestamp"], "Error: Timestamp is wrong.") del transition["timestamp"] goldenTransitions = [{"name": testJobA["name"], "retryCount": 0, "newState": "jobfailed", "oldState": "complete", "requestName": "wf001", "user": "******", "group": "DMWM", "jobType": "Processing", "taskType": "ReDigi", "performance": {}, "exitCode": 0}, {"name": testJobB["name"], "retryCount": 1, "newState": "executing", "oldState": "created", "requestName": "wf001", "user": "******", "group": "DMWM", "jobType": "Processing", "taskType": "ReDigi", "performance": {}, "exitCode": 0}, {"name": testJobB["name"], "retryCount": 1, "newState": "success", "oldState": "complete", "requestName": "wf001", "user": "******", "group": "DMWM", "jobType": "Processing", "taskType": "ReDigi", "performance": {}, "exitCode": 0}] self.assertEqual(transitions, goldenTransitions, "Error: Wrong transitions: %s %s" % (transitions, goldenTransitions)) xmlPath = os.path.join(getWMBASE(), "test/python/WMCore_t/FwkJobReport_t/PerformanceReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) testJobA["fwjr"] = myReport change.propagate([testJobA], "complete", "executing") change.propagate([testJobA], "success", "complete") transitions = change.listTransitionsForDashboard() self.assertEqual(len(transitions), 1, "Error: Wrong number of transitions.") perfSection = transitions[0]["performance"]["cmsRun1"] self.assertTrue(perfSection.has_key("storage"), "Error: Storage section is missing.") self.assertTrue(perfSection.has_key("memory"), "Error: Memory section is missing.") self.assertTrue(perfSection.has_key("cpu"), "Error: CPU section is missing.") self.assertEqual(perfSection["cpu"]["AvgEventCPU"], "0.626105", "Error: AvgEventCPU is wrong.") self.assertEqual(perfSection["cpu"]["TotalJobTime"], "23.5703", "Error: TotalJobTime is wrong.") self.assertEqual(perfSection["storage"]["readTotalMB"], 39.6166, "Error: readTotalMB is wrong.") self.assertEqual(perfSection["storage"]["readMaxMSec"], 320.653, "Error: readMaxMSec is wrong") self.assertEqual(perfSection["memory"]["PeakValueRss"], "492.293", "Error: PeakValueRss is wrong.") self.assertEqual(perfSection["memory"]["PeakValueVsize"], "643.281", "Error: PeakValueVsize is wrong.") failedReport = Report() failedReport.unpersist(os.path.join(getWMBASE(), "test/python/WMCore_t/JobStateMachine_t/FailedReport.pkl")) change.propagate([testJobB], "complete", "executing") testJobB["fwjr"] = failedReport testJobB["retry_count"] += 1 change.propagate([testJobB], "success", "complete") transitions = change.listTransitionsForDashboard() self.assertEqual(len(transitions), 1, "Error: Wrong number of transitions.") self.assertEqual(transitions[0]["exitCode"], "8020", "Error: Wrong exit code.") return
config.component_("TaskArchiver") config.TaskArchiver.namespace = "WMComponent.TaskArchiver.TaskArchiver" config.TaskArchiver.componentDir = config.General.workDir + "/TaskArchiver" config.TaskArchiver.logLevel = globalLogLevel config.TaskArchiver.pollInterval = 240 config.TaskArchiver.timeOut = workflowArchiveTimeout config.TaskArchiver.useWorkQueue = True config.TaskArchiver.workloadSummaryCouchURL = workloadSummaryURL config.webapp_("WMBSService") config.WMBSService.default_expires = 0 config.WMBSService.componentDir = os.path.join(config.General.workDir, "WMBSService") config.WMBSService.Webtools.port = wmbsServicePort config.WMBSService.Webtools.host = serverHostName config.WMBSService.Webtools.environment = "devel" config.WMBSService.templates = os.path.join(getWMBASE(), "src/templates/WMCore/WebTools") config.WMBSService.admin = config.Agent.contact config.WMBSService.title = "WMBS Data Service" config.WMBSService.description = "Provide WMBS related service call" config.WMBSService.section_("security") config.WMBSService.security.dangerously_insecure = True config.WMBSService.section_("views") active = config.WMBSService.views.section_("active") wmbs = active.section_("wmbs") wmbs.object = "WMCore.WebTools.RESTApi" wmbs.templates = os.path.join(getWMBASE(), "src/templates/WMCore/WebTools/") wmbs.section_("model") wmbs.model.object = "WMCore.HTTPFrontEnd.WMBS.WMBSRESTModel" wmbs.section_("formatter")
import thread thread.stack_size(128*1024) import socket import re import WMCore.HTTPFrontEnd.RequestManager.ReqMgrConfiguration as ReqMgrConfig __all__ = [] from WMCore.WMInit import getWMBASE import os.path INSTALL = getWMBASE() HOST = socket.getfqdn().lower() COUCH = "https://%s/couchdb" % HOST ADD_MONITOR_FLAG = False if re.match(r"^vocms0(?:13[689]|140|16[135]|30[67]|318)\.cern\.ch$", HOST): COUCH = "https://cmsweb.cern.ch/couchdb" elif re.match(r"^vocms0(?:13[12])\.cern\.ch$", HOST): COUCH = "https://cmsweb-testbed.cern.ch/couchdb" elif re.match(r"^vocms0127\.cern\.ch$", HOST): COUCH = "https://cmsweb-dev.cern.ch/couchdb" config = ReqMgrConfig.reqMgrConfig(installation=INSTALL, couchurl = COUCH, addMonitor = ADD_MONITOR_FLAG) TEMPLATES = os.path.normpath(os.path.join(INSTALL, '../../../data/templates/WMCore/WebTools')) JAVASCRIPT_PATH = os.path.normpath(os.path.join(INSTALL, '../../../data/javascript')) HTML_PATH = os.path.normpath(os.path.join(INSTALL, '../../../data/html')) config.webapp_("reqmgr") config.reqmgr.html = os.path.join(HTML_PATH, 'RequestManager')
def __init__(self, config): self.config = config BasePlugin.__init__(self, config) self.locationDict = {} myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.locationAction = daoFactory(classname = "Locations.GetSiteInfo") self.packageDir = None if os.path.exists(os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py')): self.unpacker = os.path.join(getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py') else: self.unpacker = os.path.join(getWMBASE(), 'WMCore/WMRuntime/Unpacker.py') self.agent = getattr(config.Agent, 'agentName', 'WMAgent') self.sandbox = None self.scriptFile = None self.submitDir = None self.removeTime = getattr(config.BossAir, 'removeTime', 60) self.useGSite = getattr(config.BossAir, 'useGLIDEINSites', False) self.submitWMSMode = getattr(config.BossAir, 'submitWMSMode', False) self.errorThreshold= getattr(config.BossAir, 'submitErrorThreshold', 10) self.errorCount = 0 self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0) self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7) # Required for global pool accounting self.acctGroup = getattr(config.BossAir, 'acctGroup', "production") self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops") # Build ourselves a pool self.pool = [] self.input = None self.result = None self.nProcess = getattr(self.config.BossAir, 'nCondorProcesses', 4) # Set up my proxy and glexec stuff self.setupScript = getattr(config.BossAir, 'UISetupScript', None) self.proxy = None self.serverCert = getattr(config.BossAir, 'delegatedServerCert', None) self.serverKey = getattr(config.BossAir, 'delegatedServerKey', None) self.myproxySrv = getattr(config.BossAir, 'myproxyServer', None) self.proxyDir = getattr(config.BossAir, 'proxyDir', '/tmp/') self.serverHash = getattr(config.BossAir, 'delegatedServerHash', None) self.glexecPath = getattr(config.BossAir, 'glexecPath', None) self.glexecWrapScript = getattr(config.BossAir, 'glexecWrapScript', None) self.glexecUnwrapScript = getattr(config.BossAir, 'glexecUnwrapScript', None) self.jdlProxyFile = None # Proxy name to put in JDL (owned by submit user) self.glexecProxyFile = None # Copy of same file owned by submit user if self.glexecPath: if not (self.myproxySrv and self.proxyDir): raise WMException('glexec requires myproxyServer and proxyDir to be set.') if self.myproxySrv: if not (self.serverCert and self.serverKey): raise WMException('MyProxy server requires serverCert and serverKey to be set.') # Make the directory for the proxies if self.proxyDir and not os.path.exists(self.proxyDir): logging.debug("proxyDir not found: creating it.") try: os.makedirs(self.proxyDir, 0o1777) except Exception as ex: msg = "Error: problem when creating proxyDir directory - '%s'" % str(ex) raise BossAirPluginException(msg) elif not os.path.isdir(self.proxyDir): msg = "Error: proxyDir '%s' is not a directory" % self.proxyDir raise BossAirPluginException(msg) if self.serverCert and self.serverKey and self.myproxySrv: self.proxy = self.setupMyProxy() # Build a request string self.reqStr = "(Memory >= 1 && OpSys == \"LINUX\" ) && (Arch == \"INTEL\" || Arch == \"X86_64\") && stringListMember(GLIDEIN_CMSSite, DESIRED_Sites) && ((REQUIRED_OS==\"any\") || (GLIDEIN_REQUIRED_OS==REQUIRED_OS))" if hasattr(config.BossAir, 'condorRequirementsString'): self.reqStr = config.BossAir.condorRequirementsString return
def __init__(self, config): BasePlugin.__init__(self, config) self.locationDict = {} myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.locationAction = daoFactory(classname="Locations.GetSiteInfo") self.packageDir = None # if agent is running in a container, Unpacker.py must come from a directory # on the host so the condor schedd can see it # config.General.workDir should always be bind mounted to the container if getattr(config.Agent, "isDocker", False): unpackerPath = os.path.join(config.General.workDir + "/Docker/WMRuntime/Unpacker.py") else: unpackerPath = os.path.join( getWMBASE(), 'src/python/WMCore/WMRuntime/Unpacker.py') if os.path.exists(unpackerPath): self.unpacker = unpackerPath else: self.unpacker = os.path.join(getWMBASE(), 'WMCore/WMRuntime/Unpacker.py') self.agent = getattr(config.Agent, 'agentName', 'WMAgent') self.sandbox = None self.scriptFile = config.JobSubmitter.submitScript self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0) self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7) self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200) self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500) # Required for global pool accounting self.acctGroup = getattr(config.BossAir, 'acctGroup', "production") self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops") # Build a requirement string. All CMS resources match DESIRED_Sites on the START # expression side; however, there are currently some resources (T2_CH_CERN_HLT) # that are missing the REQUIRED_OS logic. Hence, we duplicate it here. # TODO(bbockelm): Remove reqStr once HLT has upgraded. self.reqStr = ( '((REQUIRED_OS=?="any") || ' '(GLIDEIN_REQUIRED_OS =?= "any") || ' 'stringListMember(GLIDEIN_REQUIRED_OS, REQUIRED_OS)) && ' '(AuthenticatedIdentity =!= "*****@*****.**")') if hasattr(config.BossAir, 'condorRequirementsString'): self.reqStr = config.BossAir.condorRequirementsString # x509 proxy handling proxy = Proxy({'logger': myThread.logger}) self.x509userproxy = proxy.getProxyFilename() # These are added now by the condor client #self.x509userproxysubject = proxy.getSubject() #self.x509userproxyfqan = proxy.getAttributeFromProxy(self.x509userproxy) return
config.Agent.useHeartbeat = False config.section_("General") config.General.workDir = workDirectory config.section_("CoreDatabase") config.CoreDatabase.connectUrl = databaseUrl config.CoreDatabase.socket = databaseSocket config.webapp_("ReqMgr") reqMgrUrl = "http://%s:%s" % (serverHostName, reqMgrPort) config.ReqMgr.componentDir = os.path.join(config.General.workDir, "ReqMgr") config.ReqMgr.Webtools.host = serverHostName config.ReqMgr.Webtools.port = reqMgrPort config.ReqMgr.Webtools.environment = "devel" config.ReqMgr.templates = os.path.join(getWMBASE(), "src/templates/WMCore/WebTools/RequestManager") config.ReqMgr.requestor = userName config.ReqMgr.admin = userEmail config.ReqMgr.title = "CMS Request Manager" config.ReqMgr.description = "CMS Request Manager" config.ReqMgr.couchURL = couchURL config.ReqMgr.default_expires = 0 config.ReqMgr.yuiroot = yuiRoot config.ReqMgr.couchUrl = couchURL config.ReqMgr.configDBName = configCacheDBName config.ReqMgr.section_("security") config.ReqMgr.security.dangerously_insecure = True views = config.ReqMgr.section_('views')
def testDuplicateJobReports(self): """ _testDuplicateJobReports_ Verify that everything works correctly if a job report is added to the database more than once. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute("site1", seName = "somese.cern.ch") testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task = "Test") testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testFile = File(lfn = "SomeLFNC", locations = set(["somese.cern.ch"])) testFile.create() testFileset.addFile(testFile) testFileset.commit() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow) testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = testSubscription) jobGroup = jobFactory(files_per_job = 1)[0] assert len(jobGroup.jobs) == 1, \ "Error: Splitting should have created one job." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Processing" change.propagate([testJobA], 'created', 'new') myReport = Report() reportPath = os.path.join(getWMBASE(), "test/python/WMCore_t/JobStateMachine_t/Report.pkl") myReport.unpersist(reportPath) testJobA["fwjr"] = myReport change.propagate([testJobA], 'executing', 'created') change.propagate([testJobA], 'executing', 'created') changeStateDB = self.couchServer.connectDatabase(dbname = "changestate_t/fwjrs") allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 2, "Error: Wrong number of documents") for resultRow in allDocs["rows"]: if resultRow["id"] != "_design/FWJRDump": fwjrDoc = changeStateDB.document(resultRow["id"]) break return
def getConfig(self): """ _getConfig_ This is the global test configuration object """ config = Configuration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = 'test' # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) # Now the CoreDatabase information # This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobCreator config.component_("JobCreator") config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator' config.JobCreator.logLevel = 'DEBUG' config.JobCreator.maxThreads = 1 config.JobCreator.UpdateFromResourceControl = True config.JobCreator.pollInterval = 10 config.JobCreator.jobCacheDir = self.testDir config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control config.JobCreator.workerThreads = 2 config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components') # JobSubmitter config.component_("JobSubmitter") config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter' config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'CondorGlobusPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') config.JobSubmitter.submitScript = os.path.join(getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'DEBUG' config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'CondorTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components') config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 # JobAccountant config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components') config.JobAccountant.logLevel = 'INFO' # JobArchiver config.component_("JobArchiver") config.JobArchiver.pollInterval = 60 config.JobArchiver.logLevel = 'INFO' config.JobArchiver.logDir = os.path.join(self.testDir, 'logs') config.JobArchiver.componentDir = os.path.join(os.getcwd(), 'Components') config.JobArchiver.numberOfJobsToCluster = 1000 # Task Archiver config.component_("TaskArchiver") config.TaskArchiver.componentDir = self.testInit.generateWorkDir() config.TaskArchiver.WorkQueueParams = {} config.TaskArchiver.pollInterval = 60 config.TaskArchiver.logLevel = 'INFO' config.TaskArchiver.timeOut = 0 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'mnorman:[email protected]:5984') config.JobStateMachine.couchDBName = "mnorman_test" # Needed, because this is a test os.makedirs(config.JobSubmitter.submitDir) return config
def tearDown(self): sys.path.remove(os.path.join(getWMBASE(), "test/python/WMCore_t/WMRuntime_t/Scripts_t")) self.testInit.delWorkDir() os.unsetenv("WMAGENT_SITE_CONFIG_OVERRIDE")