def helloWorldJob(): job = Job() job.setName("helloWorld") exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0] job.setInputSandbox(exeScriptLocation) job.setExecutable(exeScriptLocation, "", "helloWorld.log") return job
def helloWorldJob(): job = Job() job.setName( "helloWorld" ) exeScriptLocation = find_all( 'exe-script.py', '.', 'WorkloadManagementSystem' )[0] job.setInputSandbox( exeScriptLocation ) job.setExecutable( exeScriptLocation, "", "helloWorld.log" ) return job
def __submit( self, site, CE, vo ): """ set the job and submit. """ job = Job() job.setName( self.testType ) job.setJobGroup( 'CE-Test' ) job.setExecutable( self.executable ) job.setInputSandbox( '%s/%s' % ( self.__scriptPath, self.executable ) ) if site and not CE: job.setDestination( site ) if CE: job.setDestinationCE( CE ) LOCK.acquire() proxyPath = BESUtils.getProxyByVO( 'zhangxm', vo ) if not proxyPath[ 'OK' ]: LOCK.release() return proxyPath proxyPath = proxyPath[ 'Value' ] oldProxy = os.environ.get( 'X509_USER_PROXY' ) os.environ[ 'X509_USER_PROXY' ] = proxyPath result = self.dirac.submit( job ) if oldProxy is None: del os.environ[ 'X509_USER_PROXY' ] else: os.environ[ 'X509_USER_PROXY' ] = oldProxy LOCK.release() return result
def job(): from DIRAC.Interfaces.API.Job import Job job = Job(stdout="printer", stderr="/dev/null") job.setInputSandbox([ "LFN:/vo/user/i/initial/important.tar.gz", "/abspath/absfile.xml", "file_in_pwd.xml" ]) return job
def job(): from DIRAC.Interfaces.API.Job import Job job = Job(stdout='printer', stderr='/dev/null') job.setInputSandbox([ 'LFN:/vo/user/i/initial/important.tar.gz', '/abspath/absfile.xml', 'file_in_pwd.xml', ]) return job
def parametricJob(): job = Job() job.setName("parametric_helloWorld_%n") exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0] job.setInputSandbox(exeScriptLocation) job.setParameterSequence("args", ['one', 'two', 'three']) job.setParameterSequence("iargs", [1, 2, 3]) job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log') return job
def base(): job = Job() job.setName("helloWorld-TEST-TO-Jenkins") job.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]]) job.setExecutable("exe-script.py", "", "helloWorld.log") job.setCPUTime(1780) job.setDestination('DIRAC.Jenkins.ch') job.setLogLevel('DEBUG') return job
def parametricJob(): job = Job() job.setName("parametric_helloWorld_%n") exeScriptLocation = find_all("exe-script.py", "..", "/DIRAC/tests/Integration")[0] job.setInputSandbox(exeScriptLocation) job.setParameterSequence("args", ["one", "two", "three"]) job.setParameterSequence("iargs", [1, 2, 3]) job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile="helloWorld_%n.log") return job
def submit(name, job_group, task_id, input_sandbox, output_sandbox, executable, site=None, banned_site=None, sub_ids=[]): dirac = Dirac() submit_result = {'backend_job_ids': {}} jobInfos = {} for run in range(int((len(sub_ids) + 99) / 100)): ids_this_run = [x for x in sub_ids[run * 100:(run + 1) * 100]] job_names = ['%s.%s' % (name, sub_id) for sub_id in ids_this_run] j = Job() j.setName(name) j.setExecutable(executable) j.setParameterSequence('JobName', job_names, addToWorkflow=True) j.setParameterSequence('arguments', ids_this_run, addToWorkflow=True) if input_sandbox: j.setInputSandbox(input_sandbox) if output_sandbox: j.setOutputSandbox(output_sandbox) if job_group: j.setJobGroup(job_group) if site: # set destination to a certain site; list not allowed j.setDestination(site) if banned_site: j.setBannedSites(banned_site) result = dirac.submitJob(j) if not result['OK']: sys.stdout.write('DIRAC job submit error: %s\n' % result['Message']) sys.exit(1) for sub_id, dirac_id in zip(ids_this_run, result['Value']): submit_result['backend_job_ids'][sub_id] = dirac_id jobInfos[dirac_id] = {'SubID': sub_id} #Register on Task-manager Webapp of IHEPDIRAC task = RPCClient('WorkloadManagement/TaskManager') taskInfo = {'TaskName': name, 'JobGroup': job_group, 'JSUB-ID': task_id} task_result = task.createTask(name, taskInfo, jobInfos) task_web_id = task_result['Value'] submit_result['backend_task_id'] = task_web_id return submit_result
def base(): job = Job() job.setName("helloWorld-TEST-TO-Jenkins") executablePath = find_all("exe-script.py", "..", "/DIRAC/tests/Workflow/")[0] job.setInputSandbox([executablePath]) job.setExecutable(executablePath, "", "helloWorld.log") job.setCPUTime(1780) job.setDestination("DIRAC.Jenkins.ch") job.setLogLevel("DEBUG") return job
def test_execute( self ): """ this one tests that I can execute a job that requires multi-processing """ j = Job() j.setName( "MP-test" ) j.setExecutable( self.mpExe ) j.setInputSandbox( find_all( 'mpTest.py', '.', 'Utilities' )[0] ) j.setTag( 'MultiProcessor' ) res = j.runLocal( self.d ) self.assertTrue( res['OK'] )
def read_hessjob(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) != 1): Script.showHelp() version = args[0] user_script = './read_hess2dst.sh' sim_file = 'simtel_file.list' infileLFNList = [ '/vo.cta.in2p3.fr/MC/PROD2/Config_310113/prod-2_21122012_corsika/gamma/prod-2_06052013_simtel_STD/Data/002xxx/gamma_20.0_180.0_alt2662.0_run002997.simtel.gz', '/vo.cta.in2p3.fr/MC/PROD2/Config_310113/prod-2_21122012_corsika/gamma/prod-2_06052013_simtel_STD/Data/002xxx/gamma_20.0_180.0_alt2662.0_run002998.simtel.gz' ] f = open(sim_file, 'w') for infileLFN in infileLFNList: filein = os.path.basename(infileLFN) f.write(filein) f.write('\n') f.close() j = Job() j.setInputData(infileLFNList) options = [] options = [sim_file] executablestr = "%s %s %s" % (version, user_script, ' '.join(options)) j.setExecutable('./cta-read_hess.py', executablestr) j.setInputSandbox(['cta-read_hess.py', user_script, sim_file]) j.setOutputSandbox(['read_hess.log']) j.setOutputData(['*dst.gz']) j.setName(user_script) j.setCPUTime(100000) Script.gLogger.info(j._toJDL()) Dirac().submit(j)
def softinstall(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) < 2): Script.showHelp() version = args[0] site = args[1] if version not in ['18122013', 'v0r7p0']: Script.gLogger.error('Version not valid') Script.showHelp() j = Job() j.setInputSandbox(['cta-ctools-install.py', 'SoftwareInstallation.py']) j.setExecutable('./cta-ctools-install.py', version) j.setDestination([site]) j.setName('ctoolsInstall') j.setCPUTime(100000) Script.gLogger.info(j._toJDL()) if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']: if site == 'LCG.GRIF.fr': ceList = [ 'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr', 'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr', 'node74.datagrid.cea.fr' ] elif site == 'LCG.M3PEC.fr': # ceList = ['ce0.bordeaux.inra.fr','ce0.m3pec.u-bordeaux1.fr'] ceList = ['ce0.bordeaux.inra.fr'] for ce in ceList: j.setDestinationCE(ce) name = 'ctoolsInstall' + '_' + ce j.setName(name) res = Dirac().submit(j) print res DIRAC.exit() else: name = 'ctoolsInstall' j.setName(name) res = Dirac().submit(j) print res
def _submitJob(self, result_id, executable, test_name, site_name): executable = executable.split('&') j = Job() j.setExecutable('python', arguments=executable[0] + " " + str(result_id)) sandBox = [] for file_name in executable: sandBox.append(SAM_TEST_DIR + file_name) j.setInputSandbox(sandBox) j.setName(test_name) j.setJobGroup('sam_test') j.setDestination(site_name) result = self.dirac.submit(j) return result
def test_execute( self ): """ this one tests that I can execute a job that requires multi-processing """ j = Job() j.setName( "MP-test" ) j.setExecutable( self.mpExe ) j.setInputSandbox( find_all( 'mpTest.py', '.', 'Utilities' )[0] ) j.setTag( 'MultiProcessor' ) res = j.runLocal( self.d ) if multiprocessing.cpu_count() > 1: self.assertTrue( res['OK'] ) else: self.assertFalse( res['OK'] )
def test_execute(self): """ this one tests that I can execute a job that requires multi-processing """ j = Job() j.setName("MP-test") j.setExecutable(self.mpExe) j.setInputSandbox(find_all('mpTest.py', rootPath, 'DIRAC/tests/Utilities')[0]) j.setTag('MultiProcessor') j.setLogLevel('DEBUG') res = j.runLocal(self.d) if multiprocessing.cpu_count() > 1: self.assertTrue(res['OK']) else: self.assertFalse(res['OK'])
def submitJob(jobPara): dirac = Dirac() j = Job() j.setName(jobPara['jobName']) j.setJobGroup(jobPara['jobGroup']) j.setExecutable(jobPara['jobScript'], logFile = jobPara['jobScriptLog']) j.setInputSandbox(jobPara['inputSandbox']) j.setOutputSandbox(jobPara['outputSandbox']) j.setOutputData(jobPara['outputData'], jobPara['SE']) j.setDestination(jobPara['sites']) j.setCPUTime(jobPara['CPUTime']) result = dirac.submit(j) if result['OK']: print 'Job %s submitted successfully. ID = %d' %(jobPara['jobName'],result['Value']) else: print 'Job %s submitted failed' %jobPara['jobName'] return result
def CorsikaSimtelInstall(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) != 2): Script.showHelp() version = args[0] site = args[1] if version not in ['prod-2_13112014']: Script.gLogger.error('Version not valid') Script.showHelp() j = Job() CorsikaSimtelPack = os.path.join('corsika_simhessarray', version, 'corsika_simhessarray') CorsikaSimtelLFN = 'LFN:' + os.path.join('/vo.cta.in2p3.fr/software', CorsikaSimtelPack) + '.tar.gz' j.setInputSandbox(['cta-corsikasimtel-install.py', CorsikaSimtelLFN]) j.setExecutable('./cta-corsikasimtel-install.py', version) j.setDestination([site]) j.setJobGroup('SoftInstall') j.setCPUTime(100000) if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']: if site == 'LCG.GRIF.fr': ceList = [ 'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr', 'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr', 'node74.datagrid.cea.fr' ] if site == 'LCG.M3PEC.fr': ceList = ['ce0.bordeaux.inra.fr', 'ce0.m3pec.u-bordeaux1.fr'] for ce in ceList: j.setDestinationCE(ce) name = 'corsikasimtelInstall' + '_' + ce j.setName(name) Dirac().submit(j) DIRAC.exit() j.setName('corsikasimtelInstall') Script.gLogger.info(j._toJDL()) Dirac().submit(j)
def test_execute(self): j = Job() j.setName("helloWorld-test") j.setExecutable(self.exeScriptLocation) j.setLogLevel('DEBUG') try: # This is the standard location in Jenkins j.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): j.setInputSandbox(find_all('pilot.cfg', rootPath)[0]) j.setConfigArgs('pilot.cfg') res = j.runLocal(self.d) self.assertTrue(res['OK'])
def test_execute(self): j = Job() j.setName("helloWorld-test") j.setExecutable(self.exeScriptLocation) j.setLogLevel("DEBUG") try: # This is the standard location in Jenkins j.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): j.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) j.setConfigArgs("pilot.cfg") res = j.runLocal(self.d) self.assertTrue(res["OK"])
def test_execute(self): """ this one tests that I can execute a job that requires multi-processing """ j = Job() j.setName("MP-test") j.setExecutable(self.mpExe) j.setInputSandbox( find_all('mpTest.py', rootPath, 'DIRAC/tests/Utilities')[0]) j.setNumberOfProcessors(4) j.setLogLevel('DEBUG') res = j.runLocal(self.d) if multiprocessing.cpu_count() > 1: self.assertTrue(res['OK']) else: self.assertFalse(res['OK'])
def submitJob(jobPara): dirac = Dirac() j = Job() j.setName(jobPara['jobName']) j.setJobGroup(jobPara['jobGroup']) j.setExecutable(jobPara['jobScript'], logFile=jobPara['jobScriptLog']) j.setInputSandbox(jobPara['inputSandbox']) j.setOutputSandbox(jobPara['outputSandbox']) j.setOutputData(jobPara['outputData'], jobPara['SE']) j.setDestination(jobPara['sites']) j.setCPUTime(jobPara['CPUTime']) result = dirac.submit(j) if result['OK']: print 'Job %s submitted successfully. ID = %d' % (jobPara['jobName'], result['Value']) else: print 'Job %s submitted failed' % jobPara['jobName'] return result
def test_execute(self): """just testing unix "ls" """ job = Job() job.setName("ls-test") job.setExecutable("/bin/ls", "-l") job.setLogLevel("DEBUG") try: # This is the standard location in Jenkins job.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): job.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) job.setConfigArgs("pilot.cfg") res = job.runLocal(self.d) self.assertTrue(res["OK"])
def test_submit(self): print "**********************************************************************************************************" gLogger.info("\n Submitting hello world job") helloJ = Job() helloJ.setName("helloWorld-test-T2s") helloJ.setInputSandbox( [find_all('exe-script.py', '.', 'GridTestSubmission')[0]]) helloJ.setExecutable("exe-script.py", "", "helloWorld.log") helloJ.setCPUTime(17800) result = self.dirac.submit(helloJ) gLogger.info("Hello world job: ", result) jobID = int(result['Value']) jobsSubmittedList.append(jobID) self.assert_(result['OK']) print "**********************************************************************************************************" gLogger.info("\n Submitting a job that uploads an output") helloJ = Job() helloJ.setName("upload-Output-test") helloJ.setInputSandbox( [find_all('testFileUpload.txt', '.', 'GridTestSubmission')[0]]) helloJ.setExecutable("exe-script.py", "", "helloWorld.log") helloJ.setCPUTime(17800) helloJ.setOutputData(['testFileUpload.txt']) result = self.dirac.submit(helloJ) gLogger.info("Hello world with output: ", result) jobID = int(result['Value']) jobsSubmittedList.append(jobID) self.assert_(result['OK'])
def test_execute(self): """ just testing unix "ls" """ job = Job() job.setName("ls-test") job.setExecutable("/bin/ls", '-l') job.setLogLevel('DEBUG') try: # This is the standard location in Jenkins job.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): job.setInputSandbox(find_all('pilot.cfg', rootPath)[0]) job.setConfigArgs('pilot.cfg') res = job.runLocal(self.d) self.assertTrue(res['OK'])
def basicTest(self): j = Job() j.setCPUTime(50000) j.setExecutable( '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py' ) # j.setExecutable('/bin/echo hello') j.setOwner('paterson') j.setType('test') j.setName('MyJobName') #j.setAncestorDepth(1) j.setInputSandbox([ '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts', '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts' ]) j.setOutputSandbox(['firstfile.txt', 'anotherfile.root']) j.setInputData([ '/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst', '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst' ]) j.setOutputData(['my.dst', 'myfile.log']) j.setDestination('LCG.CERN.ch') j.setPlatform('LCG') j.setSystemConfig('x86_64-slc5-gcc43-opt') j.setSoftwareTags(['VO-lhcb-Brunel-v30r17', 'VO-lhcb-Boole-v12r10']) #print j._toJDL() #print j.printObj() xml = j._toXML() testFile = 'jobDescription.xml' if os.path.exists(testFile): os.remove(testFile) xmlfile = open(testFile, 'w') xmlfile.write(xml) xmlfile.close() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow' print j.createCode() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow' j.execute() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file' workflow = fromXMLFile(testFile) code = workflow.createCode() print code workflow.execute()
def simteljob(args = None ): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args)!=1): Script.showHelp() version = args[0] user_script = './run_simtel.sh' infileLFNList = ['/vo.cta.in2p3.fr/MC/PROD2/Config_120213/prod-2_21122012_corsika/proton/Data/044xxx/proton_20.0_180.0_alt2662.0_run044019.corsika.gz', '/vo.cta.in2p3.fr/MC/PROD2/Config_120213/prod-2_21122012_corsika/proton/Data/044xxx/proton_20.0_180.0_alt2662.0_run044085.corsika.gz'] for infileLFN in infileLFNList: filein = os.path.basename(infileLFN) j = Job() j.setInputSandbox( ['cta-simtel.py', user_script] ) j.setInputData(infileLFN) user_args = [] user_args = [filein] executablestr = "%s %s %s" % ( version, user_script, ' '.join( user_args ) ) j.setExecutable('./cta-simtel.py', executablestr) sim_out = 'Data/sim_telarray/cta-ultra5/0.0deg/Data/*.simtel.gz' log_out = 'Data/sim_telarray/cta-ultra5/0.0deg/Log/*.log.gz' hist_out = 'Data/sim_telarray/cta-ultra5/0.0deg/Histograms/*.hdata.gz' j.setOutputData([sim_out,log_out,hist_out]) j.setOutputSandbox('simtel.log') j.setName(user_script) j.setCPUTime(100000) Script.gLogger.info( j._toJDL() ) Dirac().submit( j )
def launch_batch_pict( pitch_start, step, n_pict ): j = Job() j.setCPUTime(500) j.setName('%s_%f' % (EXEC, pitch_start)) j.setJobGroup(JOBGROUP) j.setInputSandbox([EXEC]) out_bmp_list=[] pitch=pitch_start for i in range(n_pict): out_bmp='out_%f.bmp' % pitch out_bmp_list.append(out_bmp) j.setExecutable(EXEC,arguments="-W 600 -H 600 -X -0.77568377 -Y -0.13646737 -P %f -M 500 %s" % (pitch, out_bmp)) pitch+=step j.setOutputSandbox(out_bmp_list + ["StdOut"] + ["StdErr"]) result = dirac.submit(j) print 'Submission Result: ',result return result
def submit(self, param): j = Job() j.setName(param['jobName']) j.setExecutable(param['jobScript'],logFile = param['jobScriptLog']) if self.site: j.setDestination(self.site) if self.jobGroup: j.setJobGroup(self.jobGroup) j.setInputSandbox(param['inputSandbox']) j.setOutputSandbox(param['outputSandbox']) j.setOutputData(param['outputData'], outputSE = self.outputSE, outputPath = self.outputPath) dirac = GridDirac() result = dirac.submit(j) status = {} status['submit'] = result['OK'] if status['submit']: status['job_id'] = result['Value'] return status
def test_min2max4(self): """ this tests executes a job that requires 2 to 4 processors """ j = Job() j.setName("MP-test-min2max4") j.setExecutable(self.mpExeFlex, arguments='2') j.setInputSandbox( find_all('mpTest-flexible.py', rootPath, 'DIRAC/tests/Utilities')[0]) j.setNumberOfProcessors( minNumberOfProcessors=2, maxNumberOfProcessors=4) # This requires 2 to 4 processors j.setLogLevel('DEBUG') try: # This is the standard location in Jenkins j.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): j.setInputSandbox(find_all('pilot.cfg', rootPath)[0]) j.setConfigArgs('pilot.cfg') res = j.runLocal(self.d) self.assertTrue(res['OK'])
def test_min2(self): """this tests executes a job that requires at least 2 processors""" j = Job() j.setName("MP-test-min2") # FIXME: the number of processors should be discovered at runtime using JobParameters.getNumberOfJobProcessors() # here, and later j.setExecutable(self.mpExeFlex, arguments="2") j.setInputSandbox( find_all("mpTest-flexible.py", rootPath, "DIRAC/tests/Utilities")[0]) j.setNumberOfProcessors( minNumberOfProcessors=2) # This requires at least 2 processors j.setLogLevel("DEBUG") try: # This is the standard location in Jenkins j.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): j.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) j.setConfigArgs("pilot.cfg") res = j.runLocal(self.d) self.assertTrue(res["OK"])
def test_fixed(self): """this tests executes a job that requires exactly 4 processors""" j = Job() j.setName("MP-test") j.setExecutable(self.mpExe) j.setInputSandbox( find_all("mpTest.py", rootPath, "DIRAC/tests/Utilities")[0]) j.setNumberOfProcessors( 4) # This requires a fixed number of processors j.setLogLevel("DEBUG") try: # This is the standard location in Jenkins j.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): j.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) j.setConfigArgs("pilot.cfg") res = j.runLocal(self.d) if multiprocessing.cpu_count() > 1: self.assertTrue(res["OK"]) else: self.assertFalse(res["OK"])
def test_execute(self): job = Job() job._siteSet = {"DIRAC.someSite.ch"} job.setName("helloWorld-test") job.setExecutable( self.helloWorld, arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[ ("executable", "string", "", "Executable Script"), ("arguments", "string", "", "Arguments for executable Script"), ("applicationLog", "string", "", "Log file name"), ("someCustomOne", "string", "", "boh"), ], paramValues=[("someCustomOne", "aCustomValue")], ) job.setBannedSites(["LCG.SiteA.com", "DIRAC.SiteB.org"]) job.setOwner("ownerName") job.setOwnerGroup("ownerGroup") job.setName("jobName") job.setJobGroup("jobGroup") job.setType("jobType") job.setDestination("DIRAC.someSite.ch") job.setCPUTime(12345) job.setLogLevel("DEBUG") try: # This is the standard location in Jenkins job.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): job.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) job.setConfigArgs("pilot.cfg") res = job.runLocal(self.d) self.assertTrue(res["OK"])
def submit_job(self, request_id, executable, macro, starting_seed=8000000, njobs=10, platform='ANY', output_data_site='UKI-LT2-IC-HEP-disk', output_log='lzproduction_output.log'): """ Submit LZProduction job to DIRAC. Args: request_id (int): The id number of the associated request executable (str): The full path to the executable job script macro (str): The full path to the macro for this job starting_seed (int): The random seed for the first of the parametric jobs njobs (int): The number of parametric jobs to create platform (str): The required platform output_data_site (str): The name of the grid site to store the output data at output_log (str): The file name for the output log file Returns: list: The list of created parametric job DIRAC ids """ j = Job() j.setName(os.path.splitext(os.path.basename(macro))[0] + '%(args)s') j.setExecutable(os.path.basename(executable), os.path.basename(macro) + ' %(args)s', output_log) j.setInputSandbox([executable, macro]) j.setOutputData('*.root', output_data_site, str(request_id)) j.setParameterSequence( "args", [str(i) for i in xrange(starting_seed, starting_seed + njobs)], addToWorkflow=True) j.setPlatform(platform) return self.status(self._dirac_api.submit(j).get("Value", []))
def submit(self, param): j = Job() j.setName(param['jobName']) j.setExecutable(param['jobScript'], logFile=param['jobScriptLog']) if self.site: j.setDestination(self.site) if self.jobGroup: j.setJobGroup(self.jobGroup) j.setInputSandbox(param['inputSandbox']) j.setOutputSandbox(param['outputSandbox']) j.setOutputData(param['outputData'], outputSE=self.outputSE, outputPath=self.outputPath) dirac = GridDirac() result = dirac.submit(j) status = {} status['submit'] = result['OK'] if status['submit']: status['job_id'] = result['Value'] return status
def test_execute(self): job = Job() job._siteSet = {'DIRAC.someSite.ch'} job.setName("helloWorld-test") job.setExecutable(self.helloWorld, arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ('applicationLog', 'string', '', "Log file name"), ('someCustomOne', 'string', '', "boh")], paramValues=[('someCustomOne', 'aCustomValue')]) job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org']) job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setType('jobType') job.setDestination('DIRAC.someSite.ch') job.setCPUTime(12345) job.setLogLevel('DEBUG') try: # This is the standard location in Jenkins job.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): job.setInputSandbox(find_all('pilot.cfg', rootPath)[0]) job.setConfigArgs('pilot.cfg') res = job.runLocal(self.d) self.assertTrue(res['OK'])
def basicTest(self): j = Job() j.setCPUTime(50000) j.setExecutable('/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py') # j.setExecutable('/bin/echo hello') j.setOwner('paterson') j.setType('test') j.setName('MyJobName') #j.setAncestorDepth(1) j.setInputSandbox(['/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts','/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts']) j.setOutputSandbox(['firstfile.txt','anotherfile.root']) j.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst', '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst']) j.setOutputData(['my.dst','myfile.log']) j.setDestination('LCG.CERN.ch') j.setPlatform('LCG') j.setSystemConfig('x86_64-slc5-gcc43-opt') j.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10']) #print j._toJDL() #print j.printObj() xml = j._toXML() testFile = 'jobDescription.xml' if os.path.exists(testFile): os.remove(testFile) xmlfile = open(testFile,'w') xmlfile.write(xml) xmlfile.close() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow' print j.createCode() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow' j.execute() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file' workflow = fromXMLFile(testFile) code = workflow.createCode() print code workflow.execute()
def test_min1(self): """this tests executes a job that requires at least 1 processor""" j = Job() j.setName("MP-test-min1") j.setExecutable(self.mpExeFlex, arguments="2") j.setInputSandbox( find_all("mpTest-flexible.py", rootPath, "DIRAC/tests/Utilities")[0]) j.setNumberOfProcessors( minNumberOfProcessors=1) # This requires 1 to infinite processors j.setLogLevel("DEBUG") try: # This is the standard location in Jenkins j.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): j.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) j.setConfigArgs("pilot.cfg") res = j.runLocal(self.d) self.assertTrue(res["OK"])
# Setup the job. ## The DIRAC job to submit. j = Job(stdout='StdOut', stderr='StdErr') # Set the name of the job (viewable in the web portal). j.setName("CERNatschool_test_%03d" % (jobnum)) # As we're just copying the input sandbox to the storage element # via OutputData, we'll just list the files as a check for the # output written to StdOut. j.setExecutable('/bin/ls -l') # Here we add the list of LFNs we have obtained from the metadata # query. j.setInputSandbox(lfns) #...and added to the desried storage element with the corresponding # LFN via the job's OutputData. You may wish to change: # * The Storage Element - by changing the outputSE parameter; # * The LFN base name - by changing the outputPath parameter. #j.setOutputData(file_dict.keys(), \ # outputSE='GLASGOW-disk', \ # outputPath='/diractest%03d/' % (jobnum)\ # ) # These are the files retrieved with the local job output. j.setOutputSandbox(['StdOut', 'StdErr']) # You can set your preferred site here. #j.setDestination('LCG.Liverpool.uk')
if '/'.join([output_path.strip('/'), output_filename_wave]) in GRID_filelist: print("\n{} already on GRID SE\n".format(job_name)) continue if NJobs == 0: print("maximum number of jobs to submit reached") print("breaking loop now") break else: NJobs -= 1 j = Job() # runtime in seconds times 8 (CPU normalisation factor) j.setCPUTime(6 * 3600 * 8) j.setName(job_name) j.setInputSandbox(input_sandbox) if banned_sites: j.setBannedSites(banned_sites) # j.setDestination( 'LCG.IN2P3-CC.fr' ) # mr_filter loses its executable property by uploading it to the GRID SE; reset j.setExecutable('chmod', '+x mr_filter') j.setExecutable('ls -lah') for run_file in run_filelist: file_token = re.split('_', run_file)[3] # wait for a random number of seconds (up to five minutes) before starting # to add a bit more entropy in the starting times of the dirac querries.
for fn in retrieved_frames: #print("* Found: '%s'." % (fn)) inputfiles.append(fn) print("*") ## The DIRAC job to submit. j = Job(stdout='StdOut', stderr='StdErr') # Set the name of the job (viewable in the web portal). j.setName(jobname) # j.setExecutable('/bin/sh', arguments='%s %s %s' % ('run.sh', '/cvmfs/cernatschool.gridpp.ac.uk/grid-klustering-001-00-07/', 'process-frames.py')) # j.setInputSandbox(inputfiles) #...and added to the desired storage element with the corresponding # LFN via the job's OutputData. You may wish to change: # * The Storage Element - by changing the outputSE parameter; # * The LFN base name - by changing the outputPath parameter. j.setOutputData(kluster_file_names, \ outputSE='%s' % (se), \ outputPath='/%s/' % (gridoutdir)\ ) # These are the files retrieved with the local job output. j.setOutputSandbox(['StdOut', 'StdErr', 'klusters.json', 'log_process_frames.log']) # You can set your preferred site here. j.setDestination(sitename)
executable = args[0].replace("bash ","").replace("./","") if not os.path.isfile(executable): gLogger.error("file %s not found."%executable) dexit(1) os.chmod(executable,0755) # make file executable input_sandbox_files.append(executable) j.setExecutable(str(executable)) else: gLogger.error("No executable defined.") dexit(1) j.setName("MC job") if not opts.name is None: j.setName(opts.name) j.setInputSandbox(input_sandbox_files) # all input files in the sandbox j.setOutputSandbox(output_sandbox_files) j.setCPUTime(opts.cpu) if not opts.site is None: j.setDestination(opts.site.split(","))#can also be a list if not opts.bannedSites is None: j.setBannedSites(opts.bannedSites.split(",")) if not opts.release is None: tag = opts.release cl = SoftwareTagClient() result = cl.getSitesForTag(tag,'Valid') # keyword doesn't work there. if not result['OK']: gLogger.error("*ERROR* Could not get sites for Tag %s"%tag,result['Message'])
from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac #from tests.Workflow.Integration.Test_UserJobs import createJob gLogger.setLevel( 'DEBUG' ) cwd = os.path.realpath( '.' ) dirac = Dirac() # Simple Hello Word job to DIRAC.Jenkins.ch gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch" ) helloJ = Job() helloJ.setName( "helloWorld-TEST-TO-Jenkins" ) helloJ.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJ.setExecutable( "exe-script.py", "", "helloWorld.log" ) helloJ.setCPUTime( 17800 ) helloJ.setDestination( 'DIRAC.Jenkins.ch' ) result = dirac.submit( helloJ ) gLogger.info( "Hello world job: ", result ) if not result['OK']: gLogger.error( "Problem submitting job", result['Message'] ) exit( 1 ) # Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN" ) helloJMP = Job() helloJMP.setName( "helloWorld-TEST-TO-Jenkins-MP" ) helloJMP.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJMP.setExecutable( "exe-script.py", "", "helloWorld.log" )
from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job dirac = Dirac() j = Job() j.setName("compile_mrfilter") j.setCPUTime(80) j.setInputSandbox(["dirac_compile_mrfilter_pilot.sh"]) j.setExecutable("dirac_compile_mrfilter_pilot.sh", "") j.setOutputData(["mr_filter"], outputSE=None, outputPath="cta/bin/mr_filter/v3_1/") Dirac().submit(j)
f.close() if runnr == -1 : print "Something is wrong with determination of run number from input file" break print "runnr = '{0}'".format(runnr) ###################################### ## BEGIN JOB DESCRIPTION ###################################### dirac = Dirac(withRepo=True, repoLocation='jobid.list', useCertificates=False) j = Job(script=None, stdout='submission.out', stderr='submission.err') j.setInputSandbox(input_file) input_file_base = os.path.basename(input_file) ## prepare the list of output files run_log = input_file_base + ".log" dat = input_file_base.replace('aug', 'DAT') datlong = dat + ".long" output_files= [run_log, 'fluka11.out', 'fluka15.err', dat, datlong ] ## prepare the output location in GRID storage; the input path will be the used also for GRID storage outdir = grid_basedir_output + path + "/" + str(runnr) ## ALWAYS, INFO, VERBOSE, WARN, DEBUG j.setLogLevel('debug') j.setName('AUGER test simulation')
# dirac job created by ganga from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac j = Job() dirac = Dirac() # default commands added by ganga j.setName( "helloWorld-test" ) j.setInputSandbox( ['/afs/cern.ch/user/f/fstagni/userJobs/_inputHello.tar.bz2', '/afs/cern.ch/user/f/fstagni/userJobs/hello-script.py'] ) j.setExecutable( "exe-script.py", "", "Ganga_Executable.log" ) # <-- user settings j.setCPUTime( 172800 ) tier1s = DMSHelpers().getTiers( tier = ( 0, 1 ) ) j.setBannedSites( tier1s ) # user settings --> # print j.workflow # submit the job to dirac result = dirac.submit( j ) print result
from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac #from tests.Workflow.Integration.Test_UserJobs import createJob gLogger.setLevel('DEBUG') cwd = os.path.realpath('.') dirac = Dirac() # Simple Hello Word job to DIRAC.Jenkins.ch gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch") helloJ = Job() helloJ.setName("helloWorld-TEST-TO-Jenkins") helloJ.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]]) helloJ.setExecutable("exe-script.py", "", "helloWorld.log") helloJ.setCPUTime(1780) helloJ.setDestination('DIRAC.Jenkins.ch') helloJ.setLogLevel('DEBUG') result = dirac.submitJob(helloJ) gLogger.info("Hello world job: ", result) if not result['OK']: gLogger.error("Problem submitting job", result['Message']) exit(1) # Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN") helloJMP = Job() helloJMP.setName("helloWorld-TEST-TO-Jenkins-MP") helloJMP.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
print "runnr = '{0}'".format(runnr) print "prmpar = '{0}'".format(prmpar) print "Energy Min = '{0}'".format(e_min) print "Energy Max = '{0}'".format(e_max) print "Theta Min = '{0}'".format(theta_min) print "Theta Max = '{0}'".format(theta_max) ###################################### ## BEGIN JOB DESCRIPTION ###################################### dirac = Dirac(withRepo=True, repoLocation='jobid.list', useCertificates=False) j = Job(script=None, stdout='submission.out', stderr='submission.err') input_files_list = [] input_files_list.append(input_file) j.setInputSandbox(input_files_list) input_file_base = os.path.basename(input_file) # set the list of output files output_files = [ 'data.tar.gz', 'logs.tar.gz' ] ## prepare the output location in GRID storage; the input path will be the used also for GRID storage # outdir = grid_basedir_output + PROD_NAME + "/" + str(e_min) + "_" + str(e_max) + "/" + str(theta_min) + "_" + str(theta_max) + "/" + str(prmpar) + "/" + str(runnr) # outdir = "/" + PROD_NAME + "/" + str(e_min) + "_" + str(e_max) + "/" + str(theta_min) + "_" + str(theta_max) + "/" + str(prmpar) + "/" + str(runnr) outdir = "/" + PROD_NAME + "/" + str(e_min) + "/" + str(theta_min) + "/" + str(prmpar) + "/" + str(runnr) print 'SE = ',se lfns_list = [] if ( group == "auger_prod" ): base_output_path = prod_path
outputfiles.append("StdErr") ## The DIRAC job to submit. j = Job(stdout='StdOut', stderr='StdErr') # Set the name of the job (viewable in the web portal). j.setName(jobname) # As we're just copying the input sandbox to the storage element # via OutputData, we'll just list the files as a check for the # output written to StdOut. j.setExecutable('/bin/ls -l') # Here we add the names of the temporary copies of the frame data # files in the dataset to the input sandbox. These will be uploaded # to the grid with the job... j.setInputSandbox(retrieved_clusters) # These are the files retrieved with the local job output. j.setOutputSandbox(outputfiles) # You can set your preferred site here. j.setDestination(sitename) ## The DIRAC instance. dirac = Dirac() # # Submit the job and print the result. # result = dirac.submit(j) # print 'Submission result: ', result
## The DIRAC job to submit. j = Job(stdout='StdOut', stderr='StdErr') # Set the name of the job (viewable in the web portal). j.setName(jobname) # As we're just copying the input sandbox to the storage element # via OutputData, we'll just list the files as a check for the # output written to StdOut. j.setExecutable('/bin/ls -l') # Here we add the names of the temporary copies of the frame data # files in the dataset to the input sandbox. These will be uploaded # to the grid with the job... j.setInputSandbox(file_dict.keys()) #...and added to the desired storage element with the corresponding # LFN via the job's OutputData. You may wish to change: # * The Storage Element - by changing the outputSE parameter; # * The LFN base name - by changing the outputPath parameter. j.setOutputData(file_dict.keys(), \ outputSE='%s' % (se), \ outputPath='/%s/' % (gridoutdir)\ ) # These are the files retrieved with the local job output. j.setOutputSandbox(['StdOut', 'StdErr']) # You can set your preferred site here. j.setDestination(sitename)
# dirac job created by ganga from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac j = Job() dirac = Dirac() # default commands added by ganga j.setName("helloWorld-test") j.setInputSandbox( ['/afs/cern.ch/user/f/fstagni/userJobs/_inputHello.tar.bz2', '/afs/cern.ch/user/f/fstagni/userJobs/hello-script.py'] ) j.setExecutable("exe-script.py","","Ganga_Executable.log") # <-- user settings j.setCPUTime(172800) j.setBannedSites(['LCG.CERN.ch', 'LCG.CNAF.it', 'LCG.GRIDKA.de', 'LCG.IN2P3.fr', 'LCG.NIKHEF.nl', 'LCG.PIC.es', 'LCG.RAL.uk', 'LCG.SARA.nl']) # user settings --> #print j.workflow # submit the job to dirac result = dirac.submit(j) print result
def helloWorldJob(): job = Job() job.setName( "helloWorld" ) job.setInputSandbox( '../../Integration/exe-script.py' ) job.setExecutable( "exe-script.py", "", "helloWorld.log" ) return job