def test_SimpleParametricJob(): job = Job() job.setExecutable('myExec') job.setLogLevel('DEBUG') parList = [1, 2, 3] job.setParameterSequence('JOB_ID', parList, addToWorkflow=True) inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'], ['/lhcb/data/data3', '/lhcb/data/data4'], ['/lhcb/data/data5', '/lhcb/data/data6']] job.setParameterSequence('InputData', inputDataList, addToWorkflow=True) jdl = job._toJDL() try: with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWF.jdl') as fd: expected = fd.read() assert jdl == expected clad = ClassAd('[' + jdl + ']') arguments = clad.getAttributeString('Arguments') job_id = clad.getAttributeString('JOB_ID') inputData = clad.getAttributeString('InputData') assert job_id == '%(JOB_ID)s' assert inputData == '%(InputData)s' assert 'jobDescription.xml' in arguments assert '-o LogLevel=DEBUG' in arguments assert '-p JOB_ID=%(JOB_ID)s' in arguments assert '-p InputData=%(InputData)s' in arguments
def test_fixed(self): """this tests executes a job that requires exactly 4 processors""" j = Job() j.setName("MP-test") j.setExecutable(self.mpExe) j.setInputSandbox( find_all("mpTest.py", rootPath, "DIRAC/tests/Utilities")[0]) j.setNumberOfProcessors( 4) # This requires a fixed number of processors j.setLogLevel("DEBUG") try: # This is the standard location in Jenkins j.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): j.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) j.setConfigArgs("pilot.cfg") res = j.runLocal(self.d) if multiprocessing.cpu_count() > 1: self.assertTrue(res["OK"]) else: self.assertFalse(res["OK"])
def test_min2(self): """this tests executes a job that requires at least 2 processors""" j = Job() j.setName("MP-test-min2") # FIXME: the number of processors should be discovered at runtime using JobParameters.getNumberOfJobProcessors() # here, and later j.setExecutable(self.mpExeFlex, arguments="2") j.setInputSandbox( find_all("mpTest-flexible.py", rootPath, "DIRAC/tests/Utilities")[0]) j.setNumberOfProcessors( minNumberOfProcessors=2) # This requires at least 2 processors j.setLogLevel("DEBUG") try: # This is the standard location in Jenkins j.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): j.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) j.setConfigArgs("pilot.cfg") res = j.runLocal(self.d) self.assertTrue(res["OK"])
def test_min2max4(self): """ this tests executes a job that requires 2 to 4 processors """ j = Job() j.setName("MP-test-min2max4") j.setExecutable(self.mpExeFlex, arguments='2') j.setInputSandbox( find_all('mpTest-flexible.py', rootPath, 'DIRAC/tests/Utilities')[0]) j.setNumberOfProcessors( minNumberOfProcessors=2, maxNumberOfProcessors=4) # This requires 2 to 4 processors j.setLogLevel('DEBUG') try: # This is the standard location in Jenkins j.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): j.setInputSandbox(find_all('pilot.cfg', rootPath)[0]) j.setConfigArgs('pilot.cfg') res = j.runLocal(self.d) self.assertTrue(res['OK'])
def test_SimpleParametricJob(): job = Job() job.setExecutable("myExec") job.setLogLevel("DEBUG") parList = [1, 2, 3] job.setParameterSequence("JOB_ID", parList, addToWorkflow=True) inputDataList = [ ["/lhcb/data/data1", "/lhcb/data/data2"], ["/lhcb/data/data3", "/lhcb/data/data4"], ["/lhcb/data/data5", "/lhcb/data/data6"], ] job.setParameterSequence("InputData", inputDataList, addToWorkflow=True) jdl = job._toJDL() with open(join(dirname(__file__), "testWF.jdl")) as fd: expected = fd.read() assert jdl == expected clad = ClassAd("[" + jdl + "]") arguments = clad.getAttributeString("Arguments") job_id = clad.getAttributeString("JOB_ID") inputData = clad.getAttributeString("InputData") assert job_id == "%(JOB_ID)s" assert inputData == "%(InputData)s" assert "jobDescription.xml" in arguments assert "-o LogLevel=DEBUG" in arguments assert "-p JOB_ID=%(JOB_ID)s" in arguments assert "-p InputData=%(InputData)s" in arguments
def test_execute(self): job = Job() job.setName("helloWorld-test") job.setExecutable(find_all("helloWorld.py", '.', 'Integration')[0], arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ('applicationLog', 'string', '', "Log file name"), ('someCustomOne', 'string', '', "boh")], paramValues=[('someCustomOne', 'aCustomValue')]) job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org']) job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setType('jobType') job.setDestination('DIRAC.someSite.ch') job.setCPUTime(12345) job.setLogLevel('DEBUG') res = job.runLocal(self.d) self.assertTrue(res['OK'])
def test_execute( self ): job = Job() job.setName( "helloWorld-test" ) job.setExecutable( find_all( "helloWorld.py", '.', 'Integration' )[0], arguments = "This is an argument", logFile = "aLogFileForTest.txt" , parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ( 'applicationLog', 'string', '', "Log file name" ), ( 'someCustomOne', 'string', '', "boh" )], paramValues = [( 'someCustomOne', 'aCustomValue' )] ) job.setBannedSites( ['LCG.SiteA.com', 'DIRAC.SiteB.org'] ) job.setOwner( 'ownerName' ) job.setOwnerGroup( 'ownerGroup' ) job.setName( 'jobName' ) job.setJobGroup( 'jobGroup' ) job.setType( 'jobType' ) job.setDestination( 'DIRAC.someSite.ch' ) job.setCPUTime( 12345 ) job.setLogLevel( 'DEBUG' ) res = job.runLocal( self.d ) self.assertTrue( res['OK'] )
def test_SimpleParametricJob(self): job = Job() job.setExecutable('myExec') job.setLogLevel('DEBUG') parList = [1, 2, 3] job.setParameterSequence('JOB_ID', parList, addToWorkflow=True) inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'], ['/lhcb/data/data3', '/lhcb/data/data4'], ['/lhcb/data/data5', '/lhcb/data/data6']] job.setParameterSequence('InputData', inputDataList, addToWorkflow=True) jdl = job._toJDL() print jdl clad = ClassAd('[' + jdl + ']') arguments = clad.getAttributeString('Arguments') job_id = clad.getAttributeString('JOB_ID') inputData = clad.getAttributeString('InputData') print "arguments", arguments self.assertEqual(job_id, '%(JOB_ID)s') self.assertEqual(inputData, '%(InputData)s') self.assertIn('jobDescription.xml', arguments) self.assertIn('-o LogLevel=DEBUG', arguments) self.assertIn('-p JOB_ID=%(JOB_ID)s', arguments) self.assertIn('-p InputData=%(InputData)s', arguments)
def test_execute(self): j = Job() j.setName("helloWorld-test") j.setExecutable(self.exeScriptLocation) j.setLogLevel('DEBUG') res = j.runLocal(self.d) self.assertTrue(res['OK'])
def base(): job = Job() job.setName("helloWorld-TEST-TO-Jenkins") job.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]]) job.setExecutable("exe-script.py", "", "helloWorld.log") job.setCPUTime(1780) job.setDestination('DIRAC.Jenkins.ch') job.setLogLevel('DEBUG') return job
def test_execute(self): """ just testing unix "ls" """ job = Job() job.setName("ls-test") job.setExecutable("/bin/ls", '-l') job.setLogLevel('DEBUG') res = job.runLocal(self.d) self.assertTrue(res['OK'])
def base(): job = Job() job.setName("helloWorld-TEST-TO-Jenkins") executablePath = find_all("exe-script.py", "..", "/DIRAC/tests/Workflow/")[0] job.setInputSandbox([executablePath]) job.setExecutable(executablePath, "", "helloWorld.log") job.setCPUTime(1780) job.setDestination("DIRAC.Jenkins.ch") job.setLogLevel("DEBUG") return job
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc): job = Job() job.setExecutable("myExec") job.setLogLevel("DEBUG") job.setNumberOfProcessors(proc, minProc, maxProc) jdl = job._toJDL() clad = ClassAd("[" + jdl + "]") processors = clad.getAttributeInt("NumberOfProcessors") minProcessors = clad.getAttributeInt("MinNumberOfProcessors") maxProcessors = clad.getAttributeInt("MaxNumberOfProcessors") assert processors == expectedProc assert minProcessors == expectedMinProc assert maxProcessors == expectedMaxProc
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc): job = Job() job.setExecutable('myExec') job.setLogLevel('DEBUG') job.setNumberOfProcessors(proc, minProc, maxProc) jdl = job._toJDL() clad = ClassAd('[' + jdl + ']') processors = clad.getAttributeInt('NumberOfProcessors') minProcessors = clad.getAttributeInt('MinNumberOfProcessors') maxProcessors = clad.getAttributeInt('MaxNumberOfProcessors') assert processors == expectedProc assert minProcessors == expectedMinProc assert maxProcessors == expectedMaxProc
def test_execute(self): """ this one tests that I can execute a job that requires multi-processing """ j = Job() j.setName("MP-test") j.setExecutable(self.mpExe) j.setInputSandbox(find_all('mpTest.py', rootPath, 'DIRAC/tests/Utilities')[0]) j.setTag('MultiProcessor') j.setLogLevel('DEBUG') res = j.runLocal(self.d) if multiprocessing.cpu_count() > 1: self.assertTrue(res['OK']) else: self.assertFalse(res['OK'])
def test_execute(self): j = Job() j.setName("helloWorld-test") j.setExecutable(self.exeScriptLocation) j.setLogLevel("DEBUG") try: # This is the standard location in Jenkins j.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): j.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) j.setConfigArgs("pilot.cfg") res = j.runLocal(self.d) self.assertTrue(res["OK"])
def test_execute(self): j = Job() j.setName("helloWorld-test") j.setExecutable(self.exeScriptLocation) j.setLogLevel('DEBUG') try: # This is the standard location in Jenkins j.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): j.setInputSandbox(find_all('pilot.cfg', rootPath)[0]) j.setConfigArgs('pilot.cfg') res = j.runLocal(self.d) self.assertTrue(res['OK'])
def test_SimpleParametricJob(): job = Job() job.setExecutable('myExec') job.setLogLevel('DEBUG') parList = [1, 2, 3] job.setParameterSequence('JOB_ID', parList, addToWorkflow=True) inputDataList = [ [ '/lhcb/data/data1', '/lhcb/data/data2' ], [ '/lhcb/data/data3', '/lhcb/data/data4' ], [ '/lhcb/data/data5', '/lhcb/data/data6' ] ] job.setParameterSequence('InputData', inputDataList, addToWorkflow=True) jdl = job._toJDL() try: with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWF.jdl') as fd: expected = fd.read() assert jdl == expected clad = ClassAd('[' + jdl + ']') arguments = clad.getAttributeString('Arguments') job_id = clad.getAttributeString('JOB_ID') inputData = clad.getAttributeString('InputData') assert job_id == '%(JOB_ID)s' assert inputData == '%(InputData)s' assert 'jobDescription.xml' in arguments assert '-o LogLevel=DEBUG' in arguments assert'-p JOB_ID=%(JOB_ID)s' in arguments assert'-p InputData=%(InputData)s' in arguments
def test_execute(self): """just testing unix "ls" """ job = Job() job.setName("ls-test") job.setExecutable("/bin/ls", "-l") job.setLogLevel("DEBUG") try: # This is the standard location in Jenkins job.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): job.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) job.setConfigArgs("pilot.cfg") res = job.runLocal(self.d) self.assertTrue(res["OK"])
def test_execute(self): """ just testing unix "ls" """ job = Job() job.setName("ls-test") job.setExecutable("/bin/ls", '-l') job.setLogLevel('DEBUG') try: # This is the standard location in Jenkins job.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): job.setInputSandbox(find_all('pilot.cfg', rootPath)[0]) job.setConfigArgs('pilot.cfg') res = job.runLocal(self.d) self.assertTrue(res['OK'])
def test_SimpleParametricJob( self ): job = Job() job.setExecutable( 'myExec' ) job.setLogLevel( 'DEBUG' ) parList = [1,2,3] job.setParameterSequence( 'JOB_ID', parList, addToWorkflow=True ) inputDataList = [ [ '/lhcb/data/data1', '/lhcb/data/data2' ], [ '/lhcb/data/data3', '/lhcb/data/data4' ], [ '/lhcb/data/data5', '/lhcb/data/data6' ] ] job.setParameterSequence( 'InputData', inputDataList, addToWorkflow=True ) jdl = job._toJDL() print jdl clad = ClassAd( '[' + jdl + ']' ) arguments = clad.getAttributeString( 'Arguments' ) job_id = clad.getAttributeString( 'JOB_ID' ) inputData = clad.getAttributeString( 'InputData' ) print "arguments", arguments self.assertEqual( job_id, '%(JOB_ID)s' ) self.assertEqual( inputData, '%(InputData)s' ) self.assertIn( 'jobDescription.xml', arguments ) self.assertIn( '-o LogLevel=DEBUG', arguments ) self.assertIn( '-p JOB_ID=%(JOB_ID)s', arguments ) self.assertIn( '-p InputData=%(InputData)s', arguments )
def test_execute(self): job = Job() job._siteSet = {"DIRAC.someSite.ch"} job.setName("helloWorld-test") job.setExecutable( self.helloWorld, arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[ ("executable", "string", "", "Executable Script"), ("arguments", "string", "", "Arguments for executable Script"), ("applicationLog", "string", "", "Log file name"), ("someCustomOne", "string", "", "boh"), ], paramValues=[("someCustomOne", "aCustomValue")], ) job.setBannedSites(["LCG.SiteA.com", "DIRAC.SiteB.org"]) job.setOwner("ownerName") job.setOwnerGroup("ownerGroup") job.setName("jobName") job.setJobGroup("jobGroup") job.setType("jobType") job.setDestination("DIRAC.someSite.ch") job.setCPUTime(12345) job.setLogLevel("DEBUG") try: # This is the standard location in Jenkins job.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): job.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) job.setConfigArgs("pilot.cfg") res = job.runLocal(self.d) self.assertTrue(res["OK"])
def test_min1(self): """this tests executes a job that requires at least 1 processor""" j = Job() j.setName("MP-test-min1") j.setExecutable(self.mpExeFlex, arguments="2") j.setInputSandbox( find_all("mpTest-flexible.py", rootPath, "DIRAC/tests/Utilities")[0]) j.setNumberOfProcessors( minNumberOfProcessors=1) # This requires 1 to infinite processors j.setLogLevel("DEBUG") try: # This is the standard location in Jenkins j.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): j.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) j.setConfigArgs("pilot.cfg") res = j.runLocal(self.d) self.assertTrue(res["OK"])
def test_execute(self): job = Job() job._siteSet = {'DIRAC.someSite.ch'} job.setName("helloWorld-test") job.setExecutable(self.helloWorld, arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ('applicationLog', 'string', '', "Log file name"), ('someCustomOne', 'string', '', "boh")], paramValues=[('someCustomOne', 'aCustomValue')]) job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org']) job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setType('jobType') job.setDestination('DIRAC.someSite.ch') job.setCPUTime(12345) job.setLogLevel('DEBUG') try: # This is the standard location in Jenkins job.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): job.setInputSandbox(find_all('pilot.cfg', rootPath)[0]) job.setConfigArgs('pilot.cfg') res = job.runLocal(self.d) self.assertTrue(res['OK'])
gLogger.setLevel('DEBUG') cwd = os.path.realpath('.') dirac = Dirac() # Simple Hello Word job to DIRAC.Jenkins.ch gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch") helloJ = Job() helloJ.setName("helloWorld-TEST-TO-Jenkins") helloJ.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]]) helloJ.setExecutable("exe-script.py", "", "helloWorld.log") helloJ.setCPUTime(1780) helloJ.setDestination('DIRAC.Jenkins.ch') helloJ.setLogLevel('DEBUG') result = dirac.submitJob(helloJ) gLogger.info("Hello world job: ", result) if not result['OK']: gLogger.error("Problem submitting job", result['Message']) exit(1) # Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN") helloJMP = Job() helloJMP.setName("helloWorld-TEST-TO-Jenkins-MP") helloJMP.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]]) helloJMP.setExecutable("exe-script.py", "", "helloWorld.log") helloJMP.setCPUTime(1780) helloJMP.setDestination('DIRAC.Jenkins.ch') helloJMP.setLogLevel('DEBUG')
j = Job(script=None, stdout='submission.out', stderr='submission.err') j.setInputSandbox(input_file) input_file_base = os.path.basename(input_file) ## prepare the list of output files run_log = input_file_base + ".log" dat = input_file_base.replace('aug', 'DAT') datlong = dat + ".long" output_files= [run_log, 'fluka11.out', 'fluka15.err', dat, datlong ] ## prepare the output location in GRID storage; the input path will be the used also for GRID storage outdir = grid_basedir_output + path + "/" + str(runnr) ## ALWAYS, INFO, VERBOSE, WARN, DEBUG j.setLogLevel('debug') j.setName('AUGER test simulation') j.setDestinationCE(ce1) ## j.setDestination(site_dirac) j.setCPUTime(345600) ## 4 days ## download the script for preparing corsika input file for usage with cvmfs j.setExecutable( 'curl', arguments = ' -fsSLkO http://issaf.spacescience.ro/adrian/AUGER/make_run4cvmfs',logFile='cmd_logs.log') j.setExecutable( 'chmod', arguments = ' +x make_run4cvmfs',logFile='cmd_logs.log') ## create the simulation script configured for use with cvmfs ## set the make_run4cvmfs arguments to include the corsika_version and corsika_bin make_run4cvmfs_arg = input_file_base + " " + corsika_version + " " + corsika_bin