def test_execute( self ): job = Job() job.setName( "helloWorld-test" ) job.setExecutable( find_all( "helloWorld.py", '.', 'Integration' )[0], arguments = "This is an argument", logFile = "aLogFileForTest.txt" , parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ( 'applicationLog', 'string', '', "Log file name" ), ( 'someCustomOne', 'string', '', "boh" )], paramValues = [( 'someCustomOne', 'aCustomValue' )] ) job.setBannedSites( ['LCG.SiteA.com', 'DIRAC.SiteB.org'] ) job.setOwner( 'ownerName' ) job.setOwnerGroup( 'ownerGroup' ) job.setName( 'jobName' ) job.setJobGroup( 'jobGroup' ) job.setType( 'jobType' ) job.setDestination( 'DIRAC.someSite.ch' ) job.setCPUTime( 12345 ) job.setLogLevel( 'DEBUG' ) res = job.runLocal( self.d ) self.assertTrue( res['OK'] )
def SoftClean(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) != 3): Script.showHelp() package = args[0] version = args[1] site = args[2] j = Job() j.setInputSandbox(['cta-swclean.py']) arguments = package + ' ' + version j.setExecutable('./cta-swclean.py', arguments) j.setDestination([site]) name = 'SoftClean_' + package + '_' + version j.setName('SoftClean') j.setJobGroup('SoftClean') j.setCPUTime(100000) Script.gLogger.info(j._toJDL()) res = Dirac().submit(j) print res['Value']
def test_execute(self): job = Job() job.setName("helloWorld-test") job.setExecutable(find_all("helloWorld.py", '.', 'Integration')[0], arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ('applicationLog', 'string', '', "Log file name"), ('someCustomOne', 'string', '', "boh")], paramValues=[('someCustomOne', 'aCustomValue')]) job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org']) job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setType('jobType') job.setDestination('DIRAC.someSite.ch') job.setCPUTime(12345) job.setLogLevel('DEBUG') res = job.runLocal(self.d) self.assertTrue(res['OK'])
def submitProbeJobs(self, ce): """ Submit some jobs to the CEs """ #need credentials, should be there since the initialize from DIRAC.Interfaces.API.Dirac import Dirac d = Dirac() from DIRAC.Interfaces.API.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations import DIRAC ops = Operations() scriptname = ops.getValue("ResourceStatus/SofwareManagementScript", self.script) j = Job() j.setDestinationCE(ce) j.setCPUTime(1000) j.setName("Probe %s" % ce) j.setJobGroup("SoftwareProbe") j.setExecutable("%s/GlastDIRAC/ResourceStatusSystem/Client/%s" % (DIRAC.rootPath, scriptname), logFile='SoftwareProbe.log') j.setOutputSandbox('*.log') res = d.submit(j) if not res['OK']: return res return S_OK()
def submitProbeJobs(self, ce): """ Submit some jobs to the CEs """ #need credentials, should be there since the initialize from DIRAC.Interfaces.API.Dirac import Dirac d = Dirac() from DIRAC.Interfaces.API.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations import os ops = Operations("glast.org") scriptname = ops.getValue("ResourceStatus/SofwareManagementScript", self.script) j = Job() j.setDestinationCE(ce) j.setCPUTime(1000) j.setName("Probe %s" % ce) j.setJobGroup("SoftwareProbe") j.setExecutable("%s/GlastDIRAC/ResourceStatusSystem/Client/%s" % (os.environ['DIRAC'], scriptname), logFile='SoftwareProbe.log') j.setOutputSandbox('*.log') res = d.submit(j) if not res['OK']: return res return S_OK()
def softinstall( args = None ) : from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args)!=2): Script.showHelp() version = args[0] site = args[1] if version not in ['prod-2_21122012','prod-2_08032013']: Script.gLogger.error('Version not valid') Script.showHelp() j = Job() j.setInputSandbox( ['cta-swinstall.py','SoftwareInstallation.py','CODE'] ) j.setExecutable('./cta-swinstall.py', version) j.setDestination([site]) j.setName('SoftInstall') j.setCPUTime(100000) Script.gLogger.info( j._toJDL() ) Dirac().submit( j )
def base(): job = Job() job.setName("helloWorld-TEST-TO-Jenkins") job.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]]) job.setExecutable("exe-script.py", "", "helloWorld.log") job.setCPUTime(1780) job.setDestination('DIRAC.Jenkins.ch') job.setLogLevel('DEBUG') return job
def base(): job = Job() job.setName("helloWorld-TEST-TO-Jenkins") executablePath = find_all("exe-script.py", "..", "/DIRAC/tests/Workflow/")[0] job.setInputSandbox([executablePath]) job.setExecutable(executablePath, "", "helloWorld.log") job.setCPUTime(1780) job.setDestination("DIRAC.Jenkins.ch") job.setLogLevel("DEBUG") return job
def softinstall(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) < 2): Script.showHelp() version = args[0] site = args[1] if version not in ['18122013', 'v0r7p0']: Script.gLogger.error('Version not valid') Script.showHelp() j = Job() j.setInputSandbox(['cta-ctools-install.py', 'SoftwareInstallation.py']) j.setExecutable('./cta-ctools-install.py', version) j.setDestination([site]) j.setName('ctoolsInstall') j.setCPUTime(100000) Script.gLogger.info(j._toJDL()) if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']: if site == 'LCG.GRIF.fr': ceList = [ 'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr', 'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr', 'node74.datagrid.cea.fr' ] elif site == 'LCG.M3PEC.fr': # ceList = ['ce0.bordeaux.inra.fr','ce0.m3pec.u-bordeaux1.fr'] ceList = ['ce0.bordeaux.inra.fr'] for ce in ceList: j.setDestinationCE(ce) name = 'ctoolsInstall' + '_' + ce j.setName(name) res = Dirac().submit(j) print res DIRAC.exit() else: name = 'ctoolsInstall' j.setName(name) res = Dirac().submit(j) print res
def read_hessjob(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) != 1): Script.showHelp() version = args[0] user_script = './read_hess2dst.sh' sim_file = 'simtel_file.list' infileLFNList = [ '/vo.cta.in2p3.fr/MC/PROD2/Config_310113/prod-2_21122012_corsika/gamma/prod-2_06052013_simtel_STD/Data/002xxx/gamma_20.0_180.0_alt2662.0_run002997.simtel.gz', '/vo.cta.in2p3.fr/MC/PROD2/Config_310113/prod-2_21122012_corsika/gamma/prod-2_06052013_simtel_STD/Data/002xxx/gamma_20.0_180.0_alt2662.0_run002998.simtel.gz' ] f = open(sim_file, 'w') for infileLFN in infileLFNList: filein = os.path.basename(infileLFN) f.write(filein) f.write('\n') f.close() j = Job() j.setInputData(infileLFNList) options = [] options = [sim_file] executablestr = "%s %s %s" % (version, user_script, ' '.join(options)) j.setExecutable('./cta-read_hess.py', executablestr) j.setInputSandbox(['cta-read_hess.py', user_script, sim_file]) j.setOutputSandbox(['read_hess.log']) j.setOutputData(['*dst.gz']) j.setName(user_script) j.setCPUTime(100000) Script.gLogger.info(j._toJDL()) Dirac().submit(j)
def do_installonsite(self, argss): """ Install a release on a grid site : installonsite tag site """ args = argss.split() if len(args) < 2: print self.do_installonsite.__doc__ return tag = args[0] site = args[1] #print "Check if the software with the tag '"+tag+"' exists on the rsync server..." #res = self.client.getSitesForTag(tag) #if not res['OK']: #print res['Message'] #return #print "tag found !" from DIRAC.Interfaces.API.Dirac import Dirac d = Dirac() from DIRAC.Interfaces.API.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations import os ops = Operations() scriptname = "InstallSoftware.py" j = Job() j.setDestination(site) j.setCPUTime(1000) j.setName("Installation " + tag) j.setExecutable(os.environ['DIRAC'] + "/GlastDIRAC/ResourceStatusSystem/Client/externals/" + scriptname, logFile='SoftwareInstallation.log') j.setOutputSandbox('*.log') res = d.submit(j) if not res['OK']: print "Could not submit the installation at site %s, message %s" % ( site, res['Message']) return print "Job submitted, id = " + str(res['Value']) print "Add tag :" res = self.client.addTagAtSite(tag, site) if not res['OK']: print "Could not register tag %s at site %s, message %s" % ( tag, site, res['Message']) return print "Added %s to %i CEs" % (tag, len(res['Value'][tag]))
def run_test_job(args): simtel_files = load_files_from_list(args[0]) #simtel_files = ["/vo.cta.in2p3.fr/MC/PROD3/LaPalma/proton/simtel/1260/Data/071xxx/proton_40deg_180deg_run71001___cta-prod3-lapalma3-2147m-LaPalma.simtel.gz", #"/vo.cta.in2p3.fr/MC/PROD3/LaPalma/proton/simtel/1260/Data/070xxx/proton_40deg_180deg_run70502___cta-prod3-lapalma3-2147m-LaPalma.simtel.gz"] dirac = Dirac() j = Job() j.setCPUTime(500) j.setInputData(simtel_files[0]) j.setExecutable('echo', 'Hello World!') j.setName('Hello World') res = dirac.submit(j) print('Submission Result: {}'.format(res)) return res
def submitJob(jobPara): dirac = Dirac() j = Job() j.setName(jobPara['jobName']) j.setJobGroup(jobPara['jobGroup']) j.setExecutable(jobPara['jobScript'], logFile = jobPara['jobScriptLog']) j.setInputSandbox(jobPara['inputSandbox']) j.setOutputSandbox(jobPara['outputSandbox']) j.setOutputData(jobPara['outputData'], jobPara['SE']) j.setDestination(jobPara['sites']) j.setCPUTime(jobPara['CPUTime']) result = dirac.submit(j) if result['OK']: print 'Job %s submitted successfully. ID = %d' %(jobPara['jobName'],result['Value']) else: print 'Job %s submitted failed' %jobPara['jobName'] return result
def CorsikaSimtelInstall(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) != 2): Script.showHelp() version = args[0] site = args[1] if version not in ['prod-2_13112014']: Script.gLogger.error('Version not valid') Script.showHelp() j = Job() CorsikaSimtelPack = os.path.join('corsika_simhessarray', version, 'corsika_simhessarray') CorsikaSimtelLFN = 'LFN:' + os.path.join('/vo.cta.in2p3.fr/software', CorsikaSimtelPack) + '.tar.gz' j.setInputSandbox(['cta-corsikasimtel-install.py', CorsikaSimtelLFN]) j.setExecutable('./cta-corsikasimtel-install.py', version) j.setDestination([site]) j.setJobGroup('SoftInstall') j.setCPUTime(100000) if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']: if site == 'LCG.GRIF.fr': ceList = [ 'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr', 'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr', 'node74.datagrid.cea.fr' ] if site == 'LCG.M3PEC.fr': ceList = ['ce0.bordeaux.inra.fr', 'ce0.m3pec.u-bordeaux1.fr'] for ce in ceList: j.setDestinationCE(ce) name = 'corsikasimtelInstall' + '_' + ce j.setName(name) Dirac().submit(j) DIRAC.exit() j.setName('corsikasimtelInstall') Script.gLogger.info(j._toJDL()) Dirac().submit(j)
def do_installonsite(self,argss): """ Install a release on a grid site : installonsite tag site """ args = argss.split() if len(args)<2: print self.do_installonsite.__doc__ return tag = args[0] site = args[1] #print "Check if the software with the tag '"+tag+"' exists on the rsync server..." #res = self.client.getSitesForTag(tag) #if not res['OK']: #print res['Message'] #return #print "tag found !" from DIRAC.Interfaces.API.Dirac import Dirac d = Dirac() from DIRAC.Interfaces.API.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations import os ops = Operations() scriptname = "InstallSoftware.py" j = Job() j.setDestination(site) j.setCPUTime(1000) j.setName("Installation "+tag) j.setExecutable(os.environ['DIRAC']+"/GlastDIRAC/ResourceStatusSystem/Client/"+scriptname , logFile='SoftwareInstallation.log') j.setOutputSandbox('*.log') res = d.submit(j) if not res['OK']: print "Could not submit the installation at site %s, message %s"%(site,res['Message']) return print "Job submitted, id = "+str(res['Value']) print "Add tag :" res = self.client.addTagAtSite(tag,site) if not res['OK']: print "Could not register tag %s at site %s, message %s"%(tag,site,res['Message']) return print "Added %s to %i CEs"%(tag,len(res['Value'][tag]))
def submitJob(jobPara): dirac = Dirac() j = Job() j.setName(jobPara['jobName']) j.setJobGroup(jobPara['jobGroup']) j.setExecutable(jobPara['jobScript'], logFile=jobPara['jobScriptLog']) j.setInputSandbox(jobPara['inputSandbox']) j.setOutputSandbox(jobPara['outputSandbox']) j.setOutputData(jobPara['outputData'], jobPara['SE']) j.setDestination(jobPara['sites']) j.setCPUTime(jobPara['CPUTime']) result = dirac.submit(j) if result['OK']: print 'Job %s submitted successfully. ID = %d' % (jobPara['jobName'], result['Value']) else: print 'Job %s submitted failed' % jobPara['jobName'] return result
def test_submit(self): print "**********************************************************************************************************" gLogger.info("\n Submitting hello world job") helloJ = Job() helloJ.setName("helloWorld-test-T2s") helloJ.setInputSandbox( [find_all('exe-script.py', '.', 'GridTestSubmission')[0]]) helloJ.setExecutable("exe-script.py", "", "helloWorld.log") helloJ.setCPUTime(17800) result = self.dirac.submit(helloJ) gLogger.info("Hello world job: ", result) jobID = int(result['Value']) jobsSubmittedList.append(jobID) self.assert_(result['OK']) print "**********************************************************************************************************" gLogger.info("\n Submitting a job that uploads an output") helloJ = Job() helloJ.setName("upload-Output-test") helloJ.setInputSandbox( [find_all('testFileUpload.txt', '.', 'GridTestSubmission')[0]]) helloJ.setExecutable("exe-script.py", "", "helloWorld.log") helloJ.setCPUTime(17800) helloJ.setOutputData(['testFileUpload.txt']) result = self.dirac.submit(helloJ) gLogger.info("Hello world with output: ", result) jobID = int(result['Value']) jobsSubmittedList.append(jobID) self.assert_(result['OK'])
def basicTest(self): j = Job() j.setCPUTime(50000) j.setExecutable( '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py' ) # j.setExecutable('/bin/echo hello') j.setOwner('paterson') j.setType('test') j.setName('MyJobName') #j.setAncestorDepth(1) j.setInputSandbox([ '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts', '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts' ]) j.setOutputSandbox(['firstfile.txt', 'anotherfile.root']) j.setInputData([ '/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst', '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst' ]) j.setOutputData(['my.dst', 'myfile.log']) j.setDestination('LCG.CERN.ch') j.setPlatform('LCG') j.setSystemConfig('x86_64-slc5-gcc43-opt') j.setSoftwareTags(['VO-lhcb-Brunel-v30r17', 'VO-lhcb-Boole-v12r10']) #print j._toJDL() #print j.printObj() xml = j._toXML() testFile = 'jobDescription.xml' if os.path.exists(testFile): os.remove(testFile) xmlfile = open(testFile, 'w') xmlfile.write(xml) xmlfile.close() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow' print j.createCode() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow' j.execute() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file' workflow = fromXMLFile(testFile) code = workflow.createCode() print code workflow.execute()
def simteljob(args = None ): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args)!=1): Script.showHelp() version = args[0] user_script = './run_simtel.sh' infileLFNList = ['/vo.cta.in2p3.fr/MC/PROD2/Config_120213/prod-2_21122012_corsika/proton/Data/044xxx/proton_20.0_180.0_alt2662.0_run044019.corsika.gz', '/vo.cta.in2p3.fr/MC/PROD2/Config_120213/prod-2_21122012_corsika/proton/Data/044xxx/proton_20.0_180.0_alt2662.0_run044085.corsika.gz'] for infileLFN in infileLFNList: filein = os.path.basename(infileLFN) j = Job() j.setInputSandbox( ['cta-simtel.py', user_script] ) j.setInputData(infileLFN) user_args = [] user_args = [filein] executablestr = "%s %s %s" % ( version, user_script, ' '.join( user_args ) ) j.setExecutable('./cta-simtel.py', executablestr) sim_out = 'Data/sim_telarray/cta-ultra5/0.0deg/Data/*.simtel.gz' log_out = 'Data/sim_telarray/cta-ultra5/0.0deg/Log/*.log.gz' hist_out = 'Data/sim_telarray/cta-ultra5/0.0deg/Histograms/*.hdata.gz' j.setOutputData([sim_out,log_out,hist_out]) j.setOutputSandbox('simtel.log') j.setName(user_script) j.setCPUTime(100000) Script.gLogger.info( j._toJDL() ) Dirac().submit( j )
def launch_batch_pict( pitch_start, step, n_pict ): j = Job() j.setCPUTime(500) j.setName('%s_%f' % (EXEC, pitch_start)) j.setJobGroup(JOBGROUP) j.setInputSandbox([EXEC]) out_bmp_list=[] pitch=pitch_start for i in range(n_pict): out_bmp='out_%f.bmp' % pitch out_bmp_list.append(out_bmp) j.setExecutable(EXEC,arguments="-W 600 -H 600 -X -0.77568377 -Y -0.13646737 -P %f -M 500 %s" % (pitch, out_bmp)) pitch+=step j.setOutputSandbox(out_bmp_list + ["StdOut"] + ["StdErr"]) result = dirac.submit(j) print 'Submission Result: ',result return result
def test_execute(self): job = Job() job._siteSet = {"DIRAC.someSite.ch"} job.setName("helloWorld-test") job.setExecutable( self.helloWorld, arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[ ("executable", "string", "", "Executable Script"), ("arguments", "string", "", "Arguments for executable Script"), ("applicationLog", "string", "", "Log file name"), ("someCustomOne", "string", "", "boh"), ], paramValues=[("someCustomOne", "aCustomValue")], ) job.setBannedSites(["LCG.SiteA.com", "DIRAC.SiteB.org"]) job.setOwner("ownerName") job.setOwnerGroup("ownerGroup") job.setName("jobName") job.setJobGroup("jobGroup") job.setType("jobType") job.setDestination("DIRAC.someSite.ch") job.setCPUTime(12345) job.setLogLevel("DEBUG") try: # This is the standard location in Jenkins job.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): job.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) job.setConfigArgs("pilot.cfg") res = job.runLocal(self.d) self.assertTrue(res["OK"])
def test_execute(self): job = Job() job._siteSet = {'DIRAC.someSite.ch'} job.setName("helloWorld-test") job.setExecutable(self.helloWorld, arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ('applicationLog', 'string', '', "Log file name"), ('someCustomOne', 'string', '', "boh")], paramValues=[('someCustomOne', 'aCustomValue')]) job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org']) job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setType('jobType') job.setDestination('DIRAC.someSite.ch') job.setCPUTime(12345) job.setLogLevel('DEBUG') try: # This is the standard location in Jenkins job.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): job.setInputSandbox(find_all('pilot.cfg', rootPath)[0]) job.setConfigArgs('pilot.cfg') res = job.runLocal(self.d) self.assertTrue(res['OK'])
def basicTest(self): j = Job() j.setCPUTime(50000) j.setExecutable('/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py') # j.setExecutable('/bin/echo hello') j.setOwner('paterson') j.setType('test') j.setName('MyJobName') #j.setAncestorDepth(1) j.setInputSandbox(['/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts','/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts']) j.setOutputSandbox(['firstfile.txt','anotherfile.root']) j.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst', '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst']) j.setOutputData(['my.dst','myfile.log']) j.setDestination('LCG.CERN.ch') j.setPlatform('LCG') j.setSystemConfig('x86_64-slc5-gcc43-opt') j.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10']) #print j._toJDL() #print j.printObj() xml = j._toXML() testFile = 'jobDescription.xml' if os.path.exists(testFile): os.remove(testFile) xmlfile = open(testFile,'w') xmlfile.write(xml) xmlfile.close() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow' print j.createCode() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow' j.execute() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file' workflow = fromXMLFile(testFile) code = workflow.createCode() print code workflow.execute()
prodJobType = 'Merge' transName = 'testProduction_' + str(int(time.time())) desc = 'just test' prodJob = Job() prodJob._addParameter(prodJob.workflow, 'PRODUCTION_ID', 'string', '00012345', 'ProductionID') prodJob._addParameter(prodJob.workflow, 'JOB_ID', 'string', '00006789', 'ProductionJobID') prodJob._addParameter(prodJob.workflow, 'eventType', 'string', 'TestEventType', 'Event Type of the prodJobuction') prodJob._addParameter(prodJob.workflow, 'numberOfEvents', 'string', '-1', 'Number of events requested') prodJob._addParameter(prodJob.workflow, 'ProcessingType', 'JDL', str('Test'), 'ProductionGroupOrType') prodJob._addParameter(prodJob.workflow, 'Priority', 'JDL', str(9), 'UserPriority') prodJob.setType(prodJobType) prodJob.workflow.setName(transName) prodJob.workflow.setDescrShort(desc) prodJob.workflow.setDescription(desc) prodJob.setCPUTime(86400) prodJob.setInputDataPolicy('Download') prodJob.setExecutable('/bin/ls', '-l') # Let's submit the prodJobuction now #result = prodJob.create() name = prodJob.workflow.getName() name = name.replace('/', '').replace('\\', '') prodJob.workflow.toXMLFile(name) print 'Workflow XML file name is: %s' % name workflowBody = '' if os.path.exists(name): with open(name, 'r') as fopen:
print "Usage %s <scriptName> <jobName> <nbJobs>"%sys.argv[0] sys.exit(1) scriptName = sys.argv[1] jobName = sys.argv[2] nbJobs = int(sys.argv[3]) if not os.path.exists(jobName): os.makedirs(jobName) os.makedirs("%s/Done"%jobName) os.makedirs("%s/Failed"%jobName) else: print "Folder %s exists"%jobName sys.exit(1) f = open("%s/jobIdList.txt"%jobName, 'w') for i in xrange(nbJobs): j = Job() j.setCPUTime(10000) j.setExecutable(scriptName) j.addToOutputSandbox.append('myLog.txt') j.addToOutputSandbox.append('clock.txt') j.addToOutputSandbox.append('time.txt') dirac = Dirac() jobID = dirac.submit(j) realId = jobID.get('JobID') f.write("%s\n"%realId) f.close()
## PREPARE JOB ## ##################### if (DO_NOT_SUBMIT): sys.exit(os.EX_USAGE) ### ALWAYS, INFO, VERBOSE, WARN, DEBUG j.setLogLevel('debug') j.setDestination(site_dirac) JOB_IDX = first_job + 1 + idx JOB_NAME = PROD_NAME + " IDX_" + str(JOB_IDX) print '\nJOB NAME is : ', JOB_NAME j.setName(JOB_NAME) j.setCPUTime(JOB_CPUTIME) ## 4 days run_corsika_sim_args = input_file_base + " " + corsika_version + " " + corsika_bin j.setExecutable( './run_corsika_sim', arguments = run_corsika_sim_args, logFile='run_sim.log') if (TEST_JOB) : jobID = dirac.submit(j,mode='local') else : jobID = dirac.submit(j) id = str(jobID) + "\n" print 'Submission Result: ',jobID with open('jobids.list', 'a') as f_id_log: f_id_log.write(id + '\n')
dat = input_file_base.replace('aug', 'DAT') datlong = dat + ".long" output_files= [run_log, 'fluka11.out', 'fluka15.err', dat, datlong ] ## prepare the output location in GRID storage; the input path will be the used also for GRID storage outdir = grid_basedir_output + path + "/" + str(runnr) ## ALWAYS, INFO, VERBOSE, WARN, DEBUG j.setLogLevel('debug') j.setName('AUGER test simulation') j.setDestinationCE(ce1) ## j.setDestination(site_dirac) j.setCPUTime(345600) ## 4 days ## download the script for preparing corsika input file for usage with cvmfs j.setExecutable( 'curl', arguments = ' -fsSLkO http://issaf.spacescience.ro/adrian/AUGER/make_run4cvmfs',logFile='cmd_logs.log') j.setExecutable( 'chmod', arguments = ' +x make_run4cvmfs',logFile='cmd_logs.log') ## create the simulation script configured for use with cvmfs ## set the make_run4cvmfs arguments to include the corsika_version and corsika_bin make_run4cvmfs_arg = input_file_base + " " + corsika_version + " " + corsika_bin j.setExecutable( './make_run4cvmfs', arguments = make_run4cvmfs_arg, logFile='cmd_logs.log') ## run simulation j.setExecutable( './execsim',logFile='cmd_logs.log') j.setOutputSandbox(output_files) j.setOutputData(output_files, outputSE=se, outputPath=outdir)
""" simple hello world job """ from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers j = Job() j.setName( "helloWorld-test" ) j.setExecutable( "exe-script.py", "", "Executable.log" ) # <-- user settings j.setCPUTime( 172800 ) tier1s = DMSHelpers().getTiers( tier = ( 0, 1 ) ) j.setBannedSites( tier1s ) # user settings --> # print j.workflow # submit the job to dirac result = Dirac().submitJob(j) print result
def main(dataset, chunksize, test): ''' The DATASET argument is a list of paths to MC files on the grid. Like the output of cta-prod3-dump-dataset for example. See also https://forge.in2p3.fr/projects/cta_dirac/wiki/CTA-DIRAC_MC_PROD3_Status Keep in mind that for some effing reason this needs to be executed within this weird 'dirac' environment which comes with its own glibc, python and pip. I guess the real Mr. Dirac would turn in his grave. ''' dirac = Dirac() with open(dataset) as f: simtel_files = f.readlines() print('Analysing {}'.format(len(simtel_files))) server_list = [ "TORINO-USER", "CYF-STORM-USER", "CYF-STORM-Disk", "M3PEC-Disk", "OBSPM-Disk", "POLGRID-Disk", "FRASCATI-USER", "LAL-Disk", "CIEMAT-Disk", "CIEMAT-USER", "CPPM-Disk", "LAL-USER", "CYFRONET-Disk", "DESY-ZN-USER", "M3PEC-USER", "LPNHE-Disk", "LPNHE-USER", "LAPP-USER", "LAPP-Disk" ] desy_server = 'DESY-ZN-USER' servers_with_miniconda = [ 'LCG.IN2P3-CC.fr', 'LCG.DESY-ZEUTHEN.de', 'LCG.CNAF.it', 'LCG.GRIF.fr', 'LCG.CYFRONET.pl', 'LCG.Prague.cz', 'LCG.CIEMAT.es' ] chunks = np.array_split(sorted(simtel_files), int(len(simtel_files) / chunksize)) print('Got a total of {} chunks'.format(len(chunks))) for c, simtel_filenames in tqdm(enumerate( chunks[0:2])): # send just 2 jobs for now. # convert chunk to a list of strings. becasue this dirac thing cant take numpy arrays simtel_filenames = [ str(s).strip() for s in simtel_filenames if 'SCT' not in s ] print('Starting processing for chunk {}'.format(c)) print(simtel_filenames) j = Job() # set runtime to 0.5h j.setCPUTime(30 * 60) j.setName('cta_preprocessing_{}'.format(c)) j.setInputData(simtel_filenames) j.setOutputData(['./processing_output/*.hdf5'], outputSE=None, outputPath='cta_preprocessing/') j.setInputSandbox( ['../process_simtel.py', './install_dependencies.py']) j.setOutputSandbox(['cta_preprocessing.log']) j.setExecutable('./job_script.sh') # These servers seem to have mini conda installed # destination = np.random.choice(servers_with_miniconda) j.setDestination(servers_with_miniconda) value = dirac.submit(j) print('Number {} Submission Result: {}'.format(c, value))
def main(): """ Launch job on the GRID """ # this thing pilots everything related to the GRID dirac = Dirac() if switches["output_type"] in "TRAINING": print("Preparing submission for TRAINING data") elif switches["output_type"] in "DL2": print("Preparing submission for DL2 data") else: print("You have to choose either TRAINING or DL2 as output type!") sys.exit() # Read configuration file cfg = load_config(switches["config_file"]) # Analysis config_path = cfg["General"]["config_path"] config_file = cfg["General"]["config_file"] mode = cfg["General"]["mode"] # One mode naw particle = cfg["General"]["particle"] estimate_energy = cfg["General"]["estimate_energy"] force_tailcut_for_extended_cleaning = cfg["General"][ "force_tailcut_for_extended_cleaning"] # Take parameters from the analysis configuration file ana_cfg = load_config(os.path.join(config_path, config_file)) config_name = ana_cfg["General"]["config_name"] cam_id_list = ana_cfg["General"]["cam_id_list"] # Regressor and classifier methods regressor_method = ana_cfg["EnergyRegressor"]["method_name"] classifier_method = ana_cfg["GammaHadronClassifier"]["method_name"] # Someone might want to create DL2 without score or energy estimation if regressor_method in ["None", "none", None]: use_regressor = False else: use_regressor = True if classifier_method in ["None", "none", None]: use_classifier = False else: use_classifier = True # GRID outdir = os.path.join(cfg["GRID"]["outdir"], config_name) n_file_per_job = cfg["GRID"]["n_file_per_job"] n_jobs_max = cfg["GRID"]["n_jobs_max"] model_dir = cfg["GRID"]["model_dir"] training_dir_energy = cfg["GRID"]["training_dir_energy"] training_dir_classification = cfg["GRID"]["training_dir_classification"] dl2_dir = cfg["GRID"]["dl2_dir"] home_grid = cfg["GRID"]["home_grid"] user_name = cfg["GRID"]["user_name"] banned_sites = cfg["GRID"]["banned_sites"] # HACK if force_tailcut_for_extended_cleaning is True: print("Force tail cuts for extended cleaning!!!") # Prepare command to launch script source_ctapipe = "source /cvmfs/cta.in2p3.fr/software/conda/dev/setupConda.sh" source_ctapipe += " && conda activate ctapipe_v0.11.0" if switches["output_type"] in "TRAINING": execute = "data_training.py" script_args = [ "--config_file={}".format(config_file), "--estimate_energy={}".format(str(estimate_energy)), "--regressor_config={}.yaml".format(regressor_method), "--regressor_dir=./", "--outfile {outfile}", "--indir ./ --infile_list={infile_name}", "--max_events={}".format(switches["max_events"]), "--{mode}", "--cam_ids {}".format(cam_id_list), ] output_filename_template = "TRAINING" elif switches["output_type"] in "DL2": execute = "write_dl2.py" script_args = [ "--config_file={}".format(config_file), "--regressor_config={}.yaml".format(regressor_method), "--regressor_dir=./", "--classifier_config={}.yaml".format(classifier_method), "--classifier_dir=./", "--outfile {outfile}", "--indir ./ --infile_list={infile_name}", "--max_events={}".format(switches["max_events"]), "--{mode}", "--force_tailcut_for_extended_cleaning={}".format( force_tailcut_for_extended_cleaning), "--cam_ids {}".format(cam_id_list), ] output_filename_template = "DL2" # Make the script save also the full calibrated images if required if switches["save_images"] is True: script_args.append("--save_images") # Make the script print debug information if required if switches["debug_script"] is True: script_args.append("--debug") cmd = [source_ctapipe, "&&", "./" + execute] cmd += script_args pilot_args_write = " ".join(cmd) # For table merging if multiple runs pilot_args_merge = " ".join([ source_ctapipe, "&&", "./merge_tables.py", "--template_file_name", "{in_name}", "--outfile", "{out_name}", ]) prod3b_filelist = dict() if estimate_energy is False and switches["output_type"] in "TRAINING": prod3b_filelist["gamma"] = cfg["EnergyRegressor"]["gamma_list"] elif estimate_energy is True and switches["output_type"] in "TRAINING": prod3b_filelist["gamma"] = cfg["GammaHadronClassifier"]["gamma_list"] prod3b_filelist["proton"] = cfg["GammaHadronClassifier"]["proton_list"] elif switches["output_type"] in "DL2": prod3b_filelist["gamma"] = cfg["Performance"]["gamma_list"] prod3b_filelist["proton"] = cfg["Performance"]["proton_list"] prod3b_filelist["electron"] = cfg["Performance"]["electron_list"] # from IPython import embed # embed() # Split list of files according to stoprage elements with open(prod3b_filelist[particle]) as f: filelist = f.readlines() filelist = ["{}".format(_.replace("\n", "")) for _ in filelist] res = dirac.splitInputData(filelist, n_file_per_job) list_run_to_loop_on = res["Value"] # define a template name for the file that's going to be written out. # the placeholder braces are going to get set during the file-loop output_filename = output_filename_template output_path = outdir if estimate_energy is False and switches["output_type"] in "TRAINING": output_path += "/{}/".format(training_dir_energy) step = "energy" if estimate_energy is True and switches["output_type"] in "TRAINING": output_path += "/{}/".format(training_dir_classification) step = "classification" if switches["output_type"] in "DL2": if force_tailcut_for_extended_cleaning is False: output_path += "/{}/".format(dl2_dir) else: output_path += "/{}_force_tc_extended_cleaning/".format(dl2_dir) step = "" output_filename += "_{}.h5" # sets all the local files that are going to be uploaded with the job # plus the pickled classifier # if file name starts with `LFN:`, it will be copied from the GRID input_sandbox = [ # Utility to assign one job to one command... os.path.expandvars("$GRID/pilot.sh"), os.path.expandvars("$PROTOPIPE/protopipe/"), os.path.expandvars("$GRID/merge_tables.py"), # python wrapper for the mr_filter wavelet cleaning # os.path.expandvars("$PYWI/pywi/"), # os.path.expandvars("$PYWICTA/pywicta/"), # script that is being run os.path.expandvars("$PROTOPIPE/protopipe/scripts/" + execute), # Configuration file os.path.expandvars(os.path.join(config_path, config_file)), ] models_to_upload = [] configs_to_upload = [] if estimate_energy is True and switches["output_type"] in "TRAINING": config_path_template = "LFN:" + os.path.join(home_grid, outdir, model_dir, "{}.yaml") config_to_upload = config_path_template.format(regressor_method) model_path_template = "LFN:" + os.path.join( home_grid, outdir, model_dir, "regressor_{}_{}.pkl.gz") for cam_id in cam_id_list: model_to_upload = model_path_template.format( cam_id, regressor_method) # TBC print("The following model(s) will be uploaded to the GRID:") print(model_to_upload) models_to_upload.append(model_to_upload) print( "The following configs(s) for such models will be uploaded to the GRID:" ) print(config_to_upload) configs_to_upload.append(config_to_upload) # input_sandbox.append(model_to_upload) elif estimate_energy is False and switches["output_type"] in "TRAINING": pass else: # Charge also classifer for DL2 model_type_list = ["regressor", "classifier"] model_method_list = [regressor_method, classifier_method] config_path_template = "LFN:" + os.path.join(home_grid, outdir, model_dir, "{}.yaml") model_path_template = "LFN:" + os.path.join( home_grid, outdir, model_dir, "{}_{}_{}.pkl.gz") if force_tailcut_for_extended_cleaning is True: force_mode = mode.replace("wave", "tail") print("################") print(force_mode) else: force_mode = mode for idx, model_type in enumerate(model_type_list): print( "The following configuration file will be uploaded to the GRID:" ) config_to_upload = config_path_template.format( model_method_list[idx]) print(config_to_upload) configs_to_upload.append(config_to_upload) # upload only 1 copy print( "The following model(s) related to such configuration file will be uploaded to the GRID:" ) for cam_id in cam_id_list: if model_type in "regressor" and use_regressor is False: print("Do not upload regressor model on GRID!!!") continue if model_type in "classifier" and use_classifier is False: print("Do not upload classifier model on GRID!!!") continue model_to_upload = model_path_template.format( model_type_list[idx], cam_id, model_method_list[idx]) print(model_to_upload) models_to_upload.append(model_to_upload) # input_sandbox.append(model_to_upload) # summary before submitting print("\nDEBUG> running as:") print(pilot_args_write) print("\nDEBUG> with input_sandbox:") print(input_sandbox) print("\nDEBUG> with output file:") print(output_filename.format("{job_name}")) print("\nDEBUG> Particles:") print(particle) print("\nDEBUG> Energy estimation:") print(estimate_energy) # ######## ## ## ## ## ## ## #### ## ## ###### # ## ## ## ## ### ## ### ## ## ### ## ## ## # ## ## ## ## #### ## #### ## ## #### ## ## # ######## ## ## ## ## ## ## ## ## ## ## ## ## ## #### # ## ## ## ## ## #### ## #### ## ## #### ## ## # ## ## ## ## ## ### ## ### ## ## ### ## ## # ## ## ####### ## ## ## ## #### ## ## ###### # list of files on the GRID SE space # not submitting jobs where we already have the output batcmd = "dirac-dms-user-lfns --BaseDir {}".format( os.path.join(home_grid, output_path)) result = subprocess.check_output(batcmd, shell=True) try: grid_filelist = open(result.split()[-1]).read() except IOError: raise IOError("ERROR> cannot read GRID filelist...") # get jobs from today and yesterday... days = [] for i in range(2): # how many days do you want to look back? days.append( (datetime.date.today() - datetime.timedelta(days=i)).isoformat()) # get list of run_tokens that are currently running / waiting running_ids = set() running_names = [] for status in ["Waiting", "Running", "Checking"]: for day in days: try: [ running_ids.add(id) for id in dirac.selectJobs( status=status, date=day, owner=user_name)["Value"] ] except KeyError: pass n_jobs = len(running_ids) if n_jobs > 0: print("Scanning {} running/waiting jobs... please wait...".format( n_jobs)) for i, id in enumerate(running_ids): if ((100 * i) / n_jobs) % 5 == 0: print("\r{} %".format(((20 * i) / n_jobs) * 5)), jobname = dirac.getJobAttributes(id)["Value"]["JobName"] running_names.append(jobname) else: print("\n... done") for bunch in list_run_to_loop_on: # for bunch in bunches_of_run: # from IPython import embed # embed() # this selects the `runxxx` part of the first and last file in the run # list and joins them with a dash so that we get a nice identifier in # the output file name. # if there is only one file in the list, use only that one # run_token = re.split('_', bunch[+0])[3] # JLK JLK run_token = re.split("_", bunch[0])[3] if len(bunch) > 1: run_token = "-".join([run_token, re.split("_", bunch[-1])[3]]) print("-" * 50) print("-" * 50) # setting output name output_filenames = dict() if switches["output_type"] in "DL2": job_name = "protopipe_{}_{}_{}_{}_{}".format( config_name, switches["output_type"], particle, run_token, mode) output_filenames[mode] = output_filename.format("_".join( [particle, mode, run_token])) else: job_name = "protopipe_{}_{}_{}_{}_{}_{}".format( config_name, switches["output_type"], step, particle, run_token, mode) output_filenames[mode] = output_filename.format("_".join( [step, particle, mode, run_token])) # if job already running / waiting, skip if job_name in running_names: print("\n WARNING> {} still running\n".format(job_name)) continue print("Output file name: {}".format(output_filenames[mode])) # if file already in GRID storage, skip # (you cannot overwrite it there, delete it and resubmit) # (assumes tail and wave will always be written out together) already_exist = False file_on_grid = os.path.join(output_path, output_filenames[mode]) print("DEBUG> check for existing file on GRID...") if file_on_grid in grid_filelist: print("\n WARNING> {} already on GRID SE\n".format(job_name)) continue if n_jobs_max == 0: print("WARNING> maximum number of jobs to submit reached") print("WARNING> breaking loop now") break else: n_jobs_max -= 1 j = Job() # runtime in seconds times 8 (CPU normalisation factor) j.setCPUTime(6 * 3600 * 8) j.setName(job_name) j.setInputSandbox(input_sandbox) if banned_sites: j.setBannedSites(banned_sites) # Add simtel files as input data j.setInputData(bunch) for run_file in bunch: file_token = re.split("_", run_file)[3] # wait for a random number of seconds (up to five minutes) before # starting to add a bit more entropy in the starting times of the # dirac queries. # if too many jobs try in parallel to access the SEs, # the interface crashes # #sleep = random.randint(0, 20) # seconds # #j.setExecutable('sleep', str(sleep)) # JLK: Try to stop doing that # consecutively downloads the data files, processes them, # deletes the input # and goes on to the next input file; # afterwards, the output files are merged # j.setExecutable('dirac-dms-get-file', "LFN:" + run_file) # source the miniconda ctapipe environment and # run the python script with all its arguments if switches["output_type"] in "DL2": output_filename_temp = output_filename.format("_".join( [particle, mode, file_token])) if switches["output_type"] in "TRAINING": output_filename_temp = output_filename.format("_".join( [step, particle, mode, file_token])) j.setExecutable( "./pilot.sh", pilot_args_write.format( outfile=output_filename_temp, infile_name=os.path.basename(run_file), mode=mode, ), ) # remove the current file to clear space j.setExecutable("rm", os.path.basename(run_file)) # simple `ls` for good measure j.setExecutable("ls", "-lh") # if there is more than one file per job, merge the output tables if len(bunch) > 1: names = [] names.append(("*_{}_".format(particle), output_filenames[mode])) for in_name, out_name in names: print("in_name: {}, out_name: {}".format(in_name, out_name)) j.setExecutable( "./pilot.sh", pilot_args_merge.format(in_name=in_name, out_name=out_name), ) print("DEBUG> args append: {}".format( pilot_args_merge.format(in_name=in_name, out_name=out_name))) bunch.extend(models_to_upload) bunch.extend(configs_to_upload) j.setInputData(bunch) print("Input data set to job = {}".format(bunch)) outputs = [] outputs.append(output_filenames[mode]) print("Output file path: {}{}".format(output_path, output_filenames[mode])) j.setOutputData(outputs, outputSE=None, outputPath=output_path) # check if we should somehow stop doing what we are doing if switches["dry"] is True: print("\nThis is a DRY RUN! -- NO job has been submitted!") print("Name of the job: {}".format(job_name)) print("Name of the output file: {}".format(outputs)) print("Output path from GRID home: {}".format(output_path)) break # this sends the job to the GRID and uploads all the # files into the input sandbox in the process print("\nSUBMITTING job with the following INPUT SANDBOX:") print(input_sandbox) print("Submission RESULT: {}\n".format(dirac.submitJob(j)["Value"])) # break if this is only a test submission if switches["test"] is True: print("This is a TEST RUN! -- Only ONE job will be submitted!") print("Name of the job: {}".format(job_name)) print("Name of the output file: {}".format(outputs)) print("Output path from GRID home: {}".format(output_path)) break # since there are two nested loops, need to break again if switches["test"] is True: break try: os.remove("datapipe.tar.gz") os.remove("modules.tar.gz") except: pass # Upload analysis configuration file for provenance SE_LIST = ['CC-IN2P3-USER', 'DESY-ZN-USER', 'CNAF-USER', 'CEA-USER'] analysis_config_local = os.path.join(config_path, config_file) # the configuration file is uploaded to the data directory because # the training samples (as well as their cleaning settings) are independent analysis_config_dirac = os.path.join(home_grid, output_path, config_file) print("Uploading {} to {}...".format(analysis_config_local, analysis_config_dirac)) if switches["dry"] is False: # Upload this file to all Dirac Storage Elements in SE_LIST for se in SE_LIST: # the uploaded config file overwrites any old copy ana_cfg_upload_cmd = "dirac-dms-add-file -f {} {} {}".format( analysis_config_dirac, analysis_config_local, se) ana_cfg_upload_result = subprocess.check_output(ana_cfg_upload_cmd, shell=True) print(ana_cfg_upload_result) else: print("This is a DRY RUN! -- analysis.yaml has NOT been uploaded.") print("\nall done -- exiting now") exit()
# (you cannot overwrite it there, delete it and resubmit) # (assumes tail and wave will always be written out together) if '/'.join([output_path.strip('/'), output_filename_wave]) in GRID_filelist: print("\n{} already on GRID SE\n".format(job_name)) continue if NJobs == 0: print("maximum number of jobs to submit reached") print("breaking loop now") break else: NJobs -= 1 j = Job() # runtime in seconds times 8 (CPU normalisation factor) j.setCPUTime(6 * 3600 * 8) j.setName(job_name) j.setInputSandbox(input_sandbox) if banned_sites: j.setBannedSites(banned_sites) # j.setDestination( 'LCG.IN2P3-CC.fr' ) # mr_filter loses its executable property by uploading it to the GRID SE; reset j.setExecutable('chmod', '+x mr_filter') j.setExecutable('ls -lah') for run_file in run_filelist: file_token = re.split('_', run_file)[3]
dexit(1) os.chmod(executable,0755) # make file executable input_sandbox_files.append(executable) j.setExecutable(str(executable)) else: gLogger.error("No executable defined.") dexit(1) j.setName("MC job") if not opts.name is None: j.setName(opts.name) j.setInputSandbox(input_sandbox_files) # all input files in the sandbox j.setOutputSandbox(output_sandbox_files) j.setCPUTime(opts.cpu) if not opts.site is None: j.setDestination(opts.site.split(","))#can also be a list if not opts.bannedSites is None: j.setBannedSites(opts.bannedSites.split(",")) if not opts.release is None: tag = opts.release cl = SoftwareTagClient() result = cl.getSitesForTag(tag,'Valid') # keyword doesn't work there. if not result['OK']: gLogger.error("*ERROR* Could not get sites for Tag %s"%tag,result['Message']) dexit(1) sites = result[ 'Value' ] j.setDestination(sites)
print("\n{} already on GRID SE\n".format(job_name)) already_exist = True break if already_exist == True: continue if NJobs == 0: print("maximum number of jobs to submit reached") print("breaking loop now") break else: NJobs -= 1 j = Job() # runtime in seconds times 8 (CPU normalisation factor) j.setCPUTime(6 * 3600 * 8) j.setName(job_name) j.setInputSandbox(input_sandbox) if banned_sites: j.setBannedSites(banned_sites) # mr_filter loses its executable property by uploading it to the GRID SE; reset j.setExecutable('chmod', '+x mr_filter') j.setExecutable('ls -lah') for run_file in run_filelist: file_token = re.split('_', run_file)[3] # wait for a random number of seconds (up to five minutes) before starting
print("Usage %s <scriptName> <jobName> <nbJobs>" % sys.argv[0]) sys.exit(1) scriptName = sys.argv[1] jobName = sys.argv[2] nbJobs = int(sys.argv[3]) if not os.path.exists(jobName): os.makedirs(jobName) os.makedirs("%s/Done" % jobName) os.makedirs("%s/Failed" % jobName) else: print("Folder %s exists" % jobName) sys.exit(1) f = open("%s/jobIdList.txt" % jobName, 'w') for i in range(nbJobs): j = Job() j.setCPUTime(10000) j.setExecutable(scriptName) j.addToOutputSandbox.append('myLog.txt') j.addToOutputSandbox.append('clock.txt') j.addToOutputSandbox.append('time.txt') dirac = Dirac() jobID = dirac.submitJob(j) realId = jobID.get('JobID') f.write("%s\n" % realId) f.close()
from DIRAC.Interfaces.API.Dirac import Dirac #from tests.Workflow.Integration.Test_UserJobs import createJob gLogger.setLevel('DEBUG') cwd = os.path.realpath('.') dirac = Dirac() # Simple Hello Word job to DIRAC.Jenkins.ch gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch") helloJ = Job() helloJ.setName("helloWorld-TEST-TO-Jenkins") helloJ.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]]) helloJ.setExecutable("exe-script.py", "", "helloWorld.log") helloJ.setCPUTime(1780) helloJ.setDestination('DIRAC.Jenkins.ch') helloJ.setLogLevel('DEBUG') result = dirac.submitJob(helloJ) gLogger.info("Hello world job: ", result) if not result['OK']: gLogger.error("Problem submitting job", result['Message']) exit(1) # Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN") helloJMP = Job() helloJMP.setName("helloWorld-TEST-TO-Jenkins-MP") helloJMP.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]]) helloJMP.setExecutable("exe-script.py", "", "helloWorld.log") helloJMP.setCPUTime(1780)
logfile = 'HelloWorldjob.log' # The executable here '' is later set, so don't confuse users later on diracJob = Job('', stdout, stderr) diracJob.setName(jobName) # Set the program/executable, arguments, logFile, ... diracJob.setExecutable('echo', arguments='\"Hello world!\"') # multiple executables can be set/appended # diracJob.setExecutable('ls', arguments='-l') # diracJob.setExecutable(executable, arguments='\"hello again\"') # Set the job length, but not needed in this example diracJob.setCPUTime(500) print 'submitting job', jobName dirac = Dirac() result = dirac.submitJob(diracJob) print 'Submission Result: ', result # try to create job id file try: jid = 'JobID' if jid in result.keys(): jid_file = open('%s.jid' % (jobName), 'w') jid_file.write('%s\n' % (str(result[jid]).strip())) jid_file.close() else: print 'Unable to creaate jid file for this job', jobName
from DIRAC.Interfaces.API.Dirac import Dirac #from tests.Workflow.Integration.Test_UserJobs import createJob gLogger.setLevel( 'DEBUG' ) cwd = os.path.realpath( '.' ) dirac = Dirac() # Simple Hello Word job to DIRAC.Jenkins.ch gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch" ) helloJ = Job() helloJ.setName( "helloWorld-TEST-TO-Jenkins" ) helloJ.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJ.setExecutable( "exe-script.py", "", "helloWorld.log" ) helloJ.setCPUTime( 17800 ) helloJ.setDestination( 'DIRAC.Jenkins.ch' ) result = dirac.submit( helloJ ) gLogger.info( "Hello world job: ", result ) if not result['OK']: gLogger.error( "Problem submitting job", result['Message'] ) exit( 1 ) # Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN" ) helloJMP = Job() helloJMP.setName( "helloWorld-TEST-TO-Jenkins-MP" ) helloJMP.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJMP.setExecutable( "exe-script.py", "", "helloWorld.log" ) helloJMP.setCPUTime( 17800 ) helloJMP.setDestination( 'DIRAC.Jenkins.ch' )
instcatname = visit + ".tar.gz" insidename = 'phosim_cat_' + str(int(visit)) + '.txt' #startsensor = idx * 4 #numsensors = 4 #if idx == 47: # numsensors = 1 startsensor = idx numsensors = 1 args = visit + ' ' + insidename + ' ' + str(startsensor) + ' ' + str( numsensors) + ' ' + str(idx) outputname = 'fits_' + visit + '_' + str(idx) + '.tar' j.setCPUTime(1209600) j.setExecutable('runimsim2.1.sh', arguments=args) j.stderr = "std.err" j.stdout = "std.out" #!!! May need the 2.1i directory here depending on visit number !!! j.setInputSandbox([ "runimsim2.1.sh", "run_imsim_nersc.py", "LFN:/lsst/user/j/james.perry/instcats/2.1i/" + instcatname ]) j.setOutputSandbox(["std.out", "std.err"]) j.setTag(["8Processors"]) #j.setOutputData([visit + "/" + outputname], outputPath="", outputSE=["IN2P3-CC-disk"]) j.setOutputData([visit + "/" + outputname], outputPath="", outputSE=["UKI-NORTHGRID-LANCS-HEP-disk"]) j.setPlatform("AnyPlatform")
from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job dirac = Dirac() j = Job() j.setName("compile_mrfilter") j.setCPUTime(80) j.setInputSandbox(["dirac_compile_mrfilter_pilot.sh"]) j.setExecutable("dirac_compile_mrfilter_pilot.sh", "") j.setOutputData(["mr_filter"], outputSE=None, outputPath="cta/bin/mr_filter/v3_1/") Dirac().submit(j)
# and a valid proxy: dirac-proxy-init -g [your vo here]_user -M # DIRAC does not work otherwise from DIRAC.Core.Base import Script Script.parseCommandLine(ignoreErrors=True) # end of DIRAC setup from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac import pprint dirac = Dirac() j = Job() j.setCPUTime(500) j.setExecutable('/bin/echo hello') j.setExecutable('/bin/hostname') j.setExecutable('/bin/echo hello again') j.setName('API') result = dirac.submitJob(j) print 'Submission Result: ' pprint.pprint(result) jobid = result['JobID'] # print job id to file for future reference joblog = open("jobid.log", "a") joblog.write(str(jobid) + '\n') joblog.close()
prodJobType = 'Merge' transName = 'testProduction_' + str(int(time.time())) desc = 'just test' prodJob = Job() prodJob._addParameter( prodJob.workflow, 'PRODUCTION_ID', 'string', '00012345', 'ProductionID' ) prodJob._addParameter( prodJob.workflow, 'JOB_ID', 'string', '00006789', 'ProductionJobID' ) prodJob._addParameter( prodJob.workflow, 'eventType', 'string', 'TestEventType', 'Event Type of the prodJobuction' ) prodJob._addParameter( prodJob.workflow, 'numberOfEvents', 'string', '-1', 'Number of events requested' ) prodJob._addParameter( prodJob.workflow, 'ProcessingType', 'JDL', str( 'Test' ), 'ProductionGroupOrType' ) prodJob._addParameter( prodJob.workflow, 'Priority', 'JDL', str( 9 ), 'UserPriority' ) prodJob.setType( prodJobType ) prodJob.workflow.setName(transName) prodJob.workflow.setDescrShort( desc ) prodJob.workflow.setDescription( desc ) prodJob.setCPUTime( 86400 ) prodJob.setInputDataPolicy( 'Download' ) prodJob.setExecutable('/bin/ls', '-l') #Let's submit the prodJobuction now #result = prodJob.create() name = prodJob.workflow.getName() name = name.replace( '/', '' ).replace( '\\', '' ) prodJob.workflow.toXMLFile( name ) print 'Workflow XML file name is: %s' % name workflowBody = '' if os.path.exists( name ): with open( name, 'r' ) as fopen:
# dirac job created by ganga from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac j = Job() dirac = Dirac() # default commands added by ganga j.setName("helloWorld-test") j.setInputSandbox( ['/afs/cern.ch/user/f/fstagni/userJobs/_inputHello.tar.bz2', '/afs/cern.ch/user/f/fstagni/userJobs/hello-script.py'] ) j.setExecutable("exe-script.py","","Ganga_Executable.log") # <-- user settings j.setCPUTime(172800) j.setBannedSites(['LCG.CERN.ch', 'LCG.CNAF.it', 'LCG.GRIDKA.de', 'LCG.IN2P3.fr', 'LCG.NIKHEF.nl', 'LCG.PIC.es', 'LCG.RAL.uk', 'LCG.SARA.nl']) # user settings --> #print j.workflow # submit the job to dirac result = dirac.submit(j) print result
"../tools/StorageElement.py" ] # these files are created by the job outputSandbox =\ [ stdout, stderr, logfile ] # the executible here '' is later set, so don't confuse users later on diracJob = Job('', stdout, stderr) # give a descriptive name diracJob.setName('ND280Custom') # set the program/executable, arguments, logFile, ... diracJob.setExecutable(exe, arguments=args, logFile=logfile) # set the job length diracJob.setCPUTime(3600) diracJob.setExecutionEnv(environmentDict) diracJob.setInputSandbox(inputSandbox) diracJob.setOutputSandbox(outputSandbox) print 'job being submitted...' dirac = Dirac() result = dirac.submit(diracJob) print 'Submission Result: ', result