def softinstall( args = None ) : from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args)!=2): Script.showHelp() version = args[0] site = args[1] if version not in ['prod-2_21122012','prod-2_08032013']: Script.gLogger.error('Version not valid') Script.showHelp() j = Job() j.setInputSandbox( ['cta-swinstall.py','SoftwareInstallation.py','CODE'] ) j.setExecutable('./cta-swinstall.py', version) j.setDestination([site]) j.setName('SoftInstall') j.setCPUTime(100000) Script.gLogger.info( j._toJDL() ) Dirac().submit( j )
def cleanProd3sw(args=None): """ Simple wrapper to remove Software package """ # get arguments package = args[0] version = args[1] site = args[2] # ## job = Job() dirac = Dirac() Step = job.setExecutable( './cta-cleansw.py', arguments = '%s %s' % ( package, version ), \ logFile = 'cleanSoftware_Log.txt' ) Step['Value']['name'] = 'Step_cleanSoftware' Step['Value']['descr_short'] = 'clean Software' # override for testing job.setName('CleanProd3Sw') # send job at Lyon CC job.setDestination([site]) # run job # res = dirac.submit( job, "local" ) res = dirac.submit(job) Script.gLogger.notice('Submission Result: ', res) return DIRAC.S_OK('Done')
def createJob(self): job = Job() job.setName(self.__stepName) job.setOutputSandbox(['*log']) job.setExecutable('/usr/bin/wget', arguments='"{0}/{1}"'.format(URL_ROOT, self.__executable)) job.setExecutable('/bin/chmod', arguments='+x "{0}"'.format(self.__executable)) arguments = '"{0}" "{1}" "{2}" "{3}" "{4}" "{5}" @{{JOB_ID}}'.format( self.__softwareVersion, self.__application, self.__outputPath, self.__outputPattern, self.__outputSE, self.__outputMode) if self.__extraArgs: arguments += ' ' + self.__extraArgs job.setExecutable(self.__executable, arguments=arguments) # failover for failed jobs job.setExecutable('/bin/ls -l', modulesList=['Script', 'FailoverRequest']) if self.__inputData: job.setInputData(self.__inputData) if self.__site: job.setDestination(self.__site) if self.__bannedsite: job.setBannedSites(self.__bannedsite) job.setOutputSandbox(['app.out', 'app.err', 'Script3_CodeOutput.log']) self.__job = job
def test_basicJob(): job = Job() job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setExecutable('someExe') job.setType('jobType') job.setDestination('ANY') xml = job._toXML() try: with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWF.xml') as fd: expected = fd.read() assert xml == expected try: with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML())) assert jdlSIO == expected
def __submit(self, site, CE, vo): """ set the job and submit. """ job = Job() job.setName(self.testType) job.setJobGroup('CE-Test') job.setExecutable(self.executable) job.setInputSandbox('%s/%s' % (self.__scriptPath, self.executable)) if site and not CE: job.setDestination(site) if CE: job.setDestinationCE(CE) LOCK.acquire() proxyPath = BESUtils.getProxyByVO('zhangxm', vo) if not proxyPath['OK']: LOCK.release() return proxyPath proxyPath = proxyPath['Value'] oldProxy = os.environ.get('X509_USER_PROXY') os.environ['X509_USER_PROXY'] = proxyPath result = self.dirac.submitJob(job) if oldProxy is None: del os.environ['X509_USER_PROXY'] else: os.environ['X509_USER_PROXY'] = oldProxy LOCK.release() return result
def SoftClean(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) != 3): Script.showHelp() package = args[0] version = args[1] site = args[2] j = Job() j.setInputSandbox(['cta-swclean.py']) arguments = package + ' ' + version j.setExecutable('./cta-swclean.py', arguments) j.setDestination([site]) name = 'SoftClean_' + package + '_' + version j.setName('SoftClean') j.setJobGroup('SoftClean') j.setCPUTime(100000) Script.gLogger.info(j._toJDL()) res = Dirac().submit(j) print res['Value']
def test_execute(self): job = Job() job.setName("helloWorld-test") job.setExecutable(find_all("helloWorld.py", '.', 'Integration')[0], arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ('applicationLog', 'string', '', "Log file name"), ('someCustomOne', 'string', '', "boh")], paramValues=[('someCustomOne', 'aCustomValue')]) job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org']) job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setType('jobType') job.setDestination('DIRAC.someSite.ch') job.setCPUTime(12345) job.setLogLevel('DEBUG') res = job.runLocal(self.d) self.assertTrue(res['OK'])
def test_execute( self ): job = Job() job.setName( "helloWorld-test" ) job.setExecutable( find_all( "helloWorld.py", '.', 'Integration' )[0], arguments = "This is an argument", logFile = "aLogFileForTest.txt" , parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ( 'applicationLog', 'string', '', "Log file name" ), ( 'someCustomOne', 'string', '', "boh" )], paramValues = [( 'someCustomOne', 'aCustomValue' )] ) job.setBannedSites( ['LCG.SiteA.com', 'DIRAC.SiteB.org'] ) job.setOwner( 'ownerName' ) job.setOwnerGroup( 'ownerGroup' ) job.setName( 'jobName' ) job.setJobGroup( 'jobGroup' ) job.setType( 'jobType' ) job.setDestination( 'DIRAC.someSite.ch' ) job.setCPUTime( 12345 ) job.setLogLevel( 'DEBUG' ) res = job.runLocal( self.d ) self.assertTrue( res['OK'] )
def __submit( self, site, CE, vo ): """ set the job and submit. """ job = Job() job.setName( self.testType ) job.setJobGroup( 'CE-Test' ) job.setExecutable( self.executable ) job.setInputSandbox( '%s/%s' % ( self.__scriptPath, self.executable ) ) if site and not CE: job.setDestination( site ) if CE: job.setDestinationCE( CE ) LOCK.acquire() proxyPath = BESUtils.getProxyByVO( 'zhangxm', vo ) if not proxyPath[ 'OK' ]: LOCK.release() return proxyPath proxyPath = proxyPath[ 'Value' ] oldProxy = os.environ.get( 'X509_USER_PROXY' ) os.environ[ 'X509_USER_PROXY' ] = proxyPath result = self.dirac.submit( job ) if oldProxy is None: del os.environ[ 'X509_USER_PROXY' ] else: os.environ[ 'X509_USER_PROXY' ] = oldProxy LOCK.release() return result
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''): if (not owner) or (not ownerGroup): res = getProxyInfo(False,False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] oJob = Job(transBody) for taskNumber in sortList(taskDict.keys()): paramsDict = taskDict[taskNumber] transID = paramsDict['TransformationID'] self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup)) oJob.setOwner(owner) oJob.setOwnerGroup(ownerGroup) transGroup = str(transID).zfill(8) self.log.verbose('Adding default transformation group of %s' % (transGroup)) oJob.setJobGroup(transGroup) constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8) self.log.verbose('Setting task name to %s' % constructedName) oJob.setName(constructedName) oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8)) oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8)) inputData = None for paramName,paramValue in paramsDict.items(): self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue)) if paramName=='InputData': if paramValue: self.log.verbose('Setting input data to %s' %paramValue) oJob.setInputData(paramValue) elif paramName=='Site': if paramValue: self.log.verbose('Setting allocated site to: %s' %(paramValue)) oJob.setDestination(paramValue) elif paramValue: self.log.verbose('Setting %s to %s' % (paramName,paramValue)) oJob._addJDLParameter(paramName,paramValue) hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])] if int(transID) in hospitalTrans: hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch') hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[]) oJob.setType('Hospital') oJob.setDestination(hospitalSite) oJob.setInputDataPolicy('download',dataScheduling=False) if hospitalCEs: oJob._addJDLParameter('GridRequiredCEs',hospitalCEs) taskDict[taskNumber]['TaskObject'] = '' res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData}) if not res ['OK']: self.log.error("Failed to generate output data",res['Message']) continue for name,output in res['Value'].items(): oJob._addJDLParameter(name,string.join(output,';')) taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML()) return S_OK(taskDict)
def base(): job = Job() job.setName("helloWorld-TEST-TO-Jenkins") job.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]]) job.setExecutable("exe-script.py", "", "helloWorld.log") job.setCPUTime(1780) job.setDestination('DIRAC.Jenkins.ch') job.setLogLevel('DEBUG') return job
def submit(name, job_group, task_id, input_sandbox, output_sandbox, executable, site=None, banned_site=None, sub_ids=[]): dirac = Dirac() submit_result = {'backend_job_ids': {}} jobInfos = {} for run in range(int((len(sub_ids) + 99) / 100)): ids_this_run = [x for x in sub_ids[run * 100:(run + 1) * 100]] job_names = ['%s.%s' % (name, sub_id) for sub_id in ids_this_run] j = Job() j.setName(name) j.setExecutable(executable) j.setParameterSequence('JobName', job_names, addToWorkflow=True) j.setParameterSequence('arguments', ids_this_run, addToWorkflow=True) if input_sandbox: j.setInputSandbox(input_sandbox) if output_sandbox: j.setOutputSandbox(output_sandbox) if job_group: j.setJobGroup(job_group) if site: # set destination to a certain site; list not allowed j.setDestination(site) if banned_site: j.setBannedSites(banned_site) result = dirac.submitJob(j) if not result['OK']: sys.stdout.write('DIRAC job submit error: %s\n' % result['Message']) sys.exit(1) for sub_id, dirac_id in zip(ids_this_run, result['Value']): submit_result['backend_job_ids'][sub_id] = dirac_id jobInfos[dirac_id] = {'SubID': sub_id} #Register on Task-manager Webapp of IHEPDIRAC task = RPCClient('WorkloadManagement/TaskManager') taskInfo = {'TaskName': name, 'JobGroup': job_group, 'JSUB-ID': task_id} task_result = task.createTask(name, taskInfo, jobInfos) task_web_id = task_result['Value'] submit_result['backend_task_id'] = task_web_id return submit_result
def base(): job = Job() job.setName("helloWorld-TEST-TO-Jenkins") executablePath = find_all("exe-script.py", "..", "/DIRAC/tests/Workflow/")[0] job.setInputSandbox([executablePath]) job.setExecutable(executablePath, "", "helloWorld.log") job.setCPUTime(1780) job.setDestination("DIRAC.Jenkins.ch") job.setLogLevel("DEBUG") return job
def softinstall(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) < 2): Script.showHelp() version = args[0] site = args[1] if version not in ['18122013', 'v0r7p0']: Script.gLogger.error('Version not valid') Script.showHelp() j = Job() j.setInputSandbox(['cta-ctools-install.py', 'SoftwareInstallation.py']) j.setExecutable('./cta-ctools-install.py', version) j.setDestination([site]) j.setName('ctoolsInstall') j.setCPUTime(100000) Script.gLogger.info(j._toJDL()) if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']: if site == 'LCG.GRIF.fr': ceList = [ 'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr', 'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr', 'node74.datagrid.cea.fr' ] elif site == 'LCG.M3PEC.fr': # ceList = ['ce0.bordeaux.inra.fr','ce0.m3pec.u-bordeaux1.fr'] ceList = ['ce0.bordeaux.inra.fr'] for ce in ceList: j.setDestinationCE(ce) name = 'ctoolsInstall' + '_' + ce j.setName(name) res = Dirac().submit(j) print res DIRAC.exit() else: name = 'ctoolsInstall' j.setName(name) res = Dirac().submit(j) print res
def _submitJob(self, result_id, executable, test_name, site_name): executable = executable.split('&') j = Job() j.setExecutable('python', arguments=executable[0] + " " + str(result_id)) sandBox = [] for file_name in executable: sandBox.append(SAM_TEST_DIR + file_name) j.setInputSandbox(sandBox) j.setName(test_name) j.setJobGroup('sam_test') j.setDestination(site_name) result = self.dirac.submit(j) return result
def do_installonsite(self, argss): """ Install a release on a grid site : installonsite tag site """ args = argss.split() if len(args) < 2: print self.do_installonsite.__doc__ return tag = args[0] site = args[1] #print "Check if the software with the tag '"+tag+"' exists on the rsync server..." #res = self.client.getSitesForTag(tag) #if not res['OK']: #print res['Message'] #return #print "tag found !" from DIRAC.Interfaces.API.Dirac import Dirac d = Dirac() from DIRAC.Interfaces.API.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations import os ops = Operations() scriptname = "InstallSoftware.py" j = Job() j.setDestination(site) j.setCPUTime(1000) j.setName("Installation " + tag) j.setExecutable(os.environ['DIRAC'] + "/GlastDIRAC/ResourceStatusSystem/Client/externals/" + scriptname, logFile='SoftwareInstallation.log') j.setOutputSandbox('*.log') res = d.submit(j) if not res['OK']: print "Could not submit the installation at site %s, message %s" % ( site, res['Message']) return print "Job submitted, id = " + str(res['Value']) print "Add tag :" res = self.client.addTagAtSite(tag, site) if not res['OK']: print "Could not register tag %s at site %s, message %s" % ( tag, site, res['Message']) return print "Added %s to %i CEs" % (tag, len(res['Value'][tag]))
def do_installonsite(self,argss): """ Install a release on a grid site : installonsite tag site """ args = argss.split() if len(args)<2: print self.do_installonsite.__doc__ return tag = args[0] site = args[1] #print "Check if the software with the tag '"+tag+"' exists on the rsync server..." #res = self.client.getSitesForTag(tag) #if not res['OK']: #print res['Message'] #return #print "tag found !" from DIRAC.Interfaces.API.Dirac import Dirac d = Dirac() from DIRAC.Interfaces.API.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations import os ops = Operations() scriptname = "InstallSoftware.py" j = Job() j.setDestination(site) j.setCPUTime(1000) j.setName("Installation "+tag) j.setExecutable(os.environ['DIRAC']+"/GlastDIRAC/ResourceStatusSystem/Client/"+scriptname , logFile='SoftwareInstallation.log') j.setOutputSandbox('*.log') res = d.submit(j) if not res['OK']: print "Could not submit the installation at site %s, message %s"%(site,res['Message']) return print "Job submitted, id = "+str(res['Value']) print "Add tag :" res = self.client.addTagAtSite(tag,site) if not res['OK']: print "Could not register tag %s at site %s, message %s"%(tag,site,res['Message']) return print "Added %s to %i CEs"%(tag,len(res['Value'][tag]))
def CorsikaSimtelInstall(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) != 2): Script.showHelp() version = args[0] site = args[1] if version not in ['prod-2_13112014']: Script.gLogger.error('Version not valid') Script.showHelp() j = Job() CorsikaSimtelPack = os.path.join('corsika_simhessarray', version, 'corsika_simhessarray') CorsikaSimtelLFN = 'LFN:' + os.path.join('/vo.cta.in2p3.fr/software', CorsikaSimtelPack) + '.tar.gz' j.setInputSandbox(['cta-corsikasimtel-install.py', CorsikaSimtelLFN]) j.setExecutable('./cta-corsikasimtel-install.py', version) j.setDestination([site]) j.setJobGroup('SoftInstall') j.setCPUTime(100000) if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']: if site == 'LCG.GRIF.fr': ceList = [ 'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr', 'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr', 'node74.datagrid.cea.fr' ] if site == 'LCG.M3PEC.fr': ceList = ['ce0.bordeaux.inra.fr', 'ce0.m3pec.u-bordeaux1.fr'] for ce in ceList: j.setDestinationCE(ce) name = 'corsikasimtelInstall' + '_' + ce j.setName(name) Dirac().submit(j) DIRAC.exit() j.setName('corsikasimtelInstall') Script.gLogger.info(j._toJDL()) Dirac().submit(j)
def submitJob(jobPara): dirac = Dirac() j = Job() j.setName(jobPara['jobName']) j.setJobGroup(jobPara['jobGroup']) j.setExecutable(jobPara['jobScript'], logFile = jobPara['jobScriptLog']) j.setInputSandbox(jobPara['inputSandbox']) j.setOutputSandbox(jobPara['outputSandbox']) j.setOutputData(jobPara['outputData'], jobPara['SE']) j.setDestination(jobPara['sites']) j.setCPUTime(jobPara['CPUTime']) result = dirac.submit(j) if result['OK']: print 'Job %s submitted successfully. ID = %d' %(jobPara['jobName'],result['Value']) else: print 'Job %s submitted failed' %jobPara['jobName'] return result
def submitJob(jobPara): dirac = Dirac() j = Job() j.setName(jobPara['jobName']) j.setJobGroup(jobPara['jobGroup']) j.setExecutable(jobPara['jobScript'], logFile=jobPara['jobScriptLog']) j.setInputSandbox(jobPara['inputSandbox']) j.setOutputSandbox(jobPara['outputSandbox']) j.setOutputData(jobPara['outputData'], jobPara['SE']) j.setDestination(jobPara['sites']) j.setCPUTime(jobPara['CPUTime']) result = dirac.submit(j) if result['OK']: print 'Job %s submitted successfully. ID = %d' % (jobPara['jobName'], result['Value']) else: print 'Job %s submitted failed' % jobPara['jobName'] return result
def basicTest(self): j = Job() j.setCPUTime(50000) j.setExecutable( '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py' ) # j.setExecutable('/bin/echo hello') j.setOwner('paterson') j.setType('test') j.setName('MyJobName') #j.setAncestorDepth(1) j.setInputSandbox([ '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts', '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts' ]) j.setOutputSandbox(['firstfile.txt', 'anotherfile.root']) j.setInputData([ '/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst', '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst' ]) j.setOutputData(['my.dst', 'myfile.log']) j.setDestination('LCG.CERN.ch') j.setPlatform('LCG') j.setSystemConfig('x86_64-slc5-gcc43-opt') j.setSoftwareTags(['VO-lhcb-Brunel-v30r17', 'VO-lhcb-Boole-v12r10']) #print j._toJDL() #print j.printObj() xml = j._toXML() testFile = 'jobDescription.xml' if os.path.exists(testFile): os.remove(testFile) xmlfile = open(testFile, 'w') xmlfile.write(xml) xmlfile.close() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow' print j.createCode() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow' j.execute() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file' workflow = fromXMLFile(testFile) code = workflow.createCode() print code workflow.execute()
def submit(self, param): j = Job() j.setName(param['jobName']) j.setExecutable(param['jobScript'],logFile = param['jobScriptLog']) if self.site: j.setDestination(self.site) if self.jobGroup: j.setJobGroup(self.jobGroup) j.setInputSandbox(param['inputSandbox']) j.setOutputSandbox(param['outputSandbox']) j.setOutputData(param['outputData'], outputSE = self.outputSE, outputPath = self.outputPath) dirac = GridDirac() result = dirac.submit(j) status = {} status['submit'] = result['OK'] if status['submit']: status['job_id'] = result['Value'] return status
def test_execute(self): job = Job() job._siteSet = {"DIRAC.someSite.ch"} job.setName("helloWorld-test") job.setExecutable( self.helloWorld, arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[ ("executable", "string", "", "Executable Script"), ("arguments", "string", "", "Arguments for executable Script"), ("applicationLog", "string", "", "Log file name"), ("someCustomOne", "string", "", "boh"), ], paramValues=[("someCustomOne", "aCustomValue")], ) job.setBannedSites(["LCG.SiteA.com", "DIRAC.SiteB.org"]) job.setOwner("ownerName") job.setOwnerGroup("ownerGroup") job.setName("jobName") job.setJobGroup("jobGroup") job.setType("jobType") job.setDestination("DIRAC.someSite.ch") job.setCPUTime(12345) job.setLogLevel("DEBUG") try: # This is the standard location in Jenkins job.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): job.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) job.setConfigArgs("pilot.cfg") res = job.runLocal(self.d) self.assertTrue(res["OK"])
def test_basicJob(): job = Job() job.setOwner("ownerName") job.setOwnerGroup("ownerGroup") job.setName("jobName") job.setJobGroup("jobGroup") job.setExecutable("someExe") job.setType("jobType") job.setDestination("ANY") xml = job._toXML() with open(join(dirname(__file__), "testWF.xml")) as fd: expected = fd.read() assert xml == expected with open(join(dirname(__file__), "testWFSIO.jdl")) as fd: expected = fd.read() jdlSIO = job._toJDL(jobDescriptionObject=StringIO(job._toXML())) assert jdlSIO == expected
def submit(self, param): j = Job() j.setName(param['jobName']) j.setExecutable(param['jobScript'], logFile=param['jobScriptLog']) if self.site: j.setDestination(self.site) if self.jobGroup: j.setJobGroup(self.jobGroup) j.setInputSandbox(param['inputSandbox']) j.setOutputSandbox(param['outputSandbox']) j.setOutputData(param['outputData'], outputSE=self.outputSE, outputPath=self.outputPath) dirac = GridDirac() result = dirac.submit(j) status = {} status['submit'] = result['OK'] if status['submit']: status['job_id'] = result['Value'] return status
def test_execute(self): job = Job() job._siteSet = {'DIRAC.someSite.ch'} job.setName("helloWorld-test") job.setExecutable(self.helloWorld, arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ('applicationLog', 'string', '', "Log file name"), ('someCustomOne', 'string', '', "boh")], paramValues=[('someCustomOne', 'aCustomValue')]) job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org']) job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setType('jobType') job.setDestination('DIRAC.someSite.ch') job.setCPUTime(12345) job.setLogLevel('DEBUG') try: # This is the standard location in Jenkins job.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): job.setInputSandbox(find_all('pilot.cfg', rootPath)[0]) job.setConfigArgs('pilot.cfg') res = job.runLocal(self.d) self.assertTrue(res['OK'])
def basicTest(self): j = Job() j.setCPUTime(50000) j.setExecutable('/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py') # j.setExecutable('/bin/echo hello') j.setOwner('paterson') j.setType('test') j.setName('MyJobName') #j.setAncestorDepth(1) j.setInputSandbox(['/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts','/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts']) j.setOutputSandbox(['firstfile.txt','anotherfile.root']) j.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst', '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst']) j.setOutputData(['my.dst','myfile.log']) j.setDestination('LCG.CERN.ch') j.setPlatform('LCG') j.setSystemConfig('x86_64-slc5-gcc43-opt') j.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10']) #print j._toJDL() #print j.printObj() xml = j._toXML() testFile = 'jobDescription.xml' if os.path.exists(testFile): os.remove(testFile) xmlfile = open(testFile,'w') xmlfile.write(xml) xmlfile.close() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow' print j.createCode() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow' j.execute() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file' workflow = fromXMLFile(testFile) code = workflow.createCode() print code workflow.execute()
input_sandbox_files.append(executable) j.setExecutable(str(executable)) else: gLogger.error("No executable defined.") dexit(1) j.setName("MC job") if not opts.name is None: j.setName(opts.name) j.setInputSandbox(input_sandbox_files) # all input files in the sandbox j.setOutputSandbox(output_sandbox_files) j.setCPUTime(opts.cpu) if not opts.site is None: j.setDestination(opts.site.split(","))#can also be a list if not opts.bannedSites is None: j.setBannedSites(opts.bannedSites.split(",")) if not opts.release is None: tag = opts.release cl = SoftwareTagClient() result = cl.getSitesForTag(tag,'Valid') # keyword doesn't work there. if not result['OK']: gLogger.error("*ERROR* Could not get sites for Tag %s"%tag,result['Message']) dexit(1) sites = result[ 'Value' ] j.setDestination(sites) if not opts.stagein is None:
j.setInputSandbox(file_dict.keys()) #...and added to the desired storage element with the corresponding # LFN via the job's OutputData. You may wish to change: # * The Storage Element - by changing the outputSE parameter; # * The LFN base name - by changing the outputPath parameter. j.setOutputData(file_dict.keys(), \ outputSE='%s' % (se), \ outputPath='/%s/' % (gridoutdir)\ ) # These are the files retrieved with the local job output. j.setOutputSandbox(['StdOut', 'StdErr']) # You can set your preferred site here. j.setDestination(sitename) ## The DIRAC instance. dirac = Dirac() # # Submit the job and print the result. # result = dirac.submit(j) # print 'Submission result: ', result # Delete the (temporary) data files. for fn in file_dict.keys(): os.remove(fn) ## The dataset name (chip ID + start time). dn = sorted(file_dict.keys())[0][:-4]
def main(dataset, chunksize, test): ''' The DATASET argument is a list of paths to MC files on the grid. Like the output of cta-prod3-dump-dataset for example. See also https://forge.in2p3.fr/projects/cta_dirac/wiki/CTA-DIRAC_MC_PROD3_Status Keep in mind that for some effing reason this needs to be executed within this weird 'dirac' environment which comes with its own glibc, python and pip. I guess the real Mr. Dirac would turn in his grave. ''' dirac = Dirac() with open(dataset) as f: simtel_files = f.readlines() print('Analysing {}'.format(len(simtel_files))) server_list = [ "TORINO-USER", "CYF-STORM-USER", "CYF-STORM-Disk", "M3PEC-Disk", "OBSPM-Disk", "POLGRID-Disk", "FRASCATI-USER", "LAL-Disk", "CIEMAT-Disk", "CIEMAT-USER", "CPPM-Disk", "LAL-USER", "CYFRONET-Disk", "DESY-ZN-USER", "M3PEC-USER", "LPNHE-Disk", "LPNHE-USER", "LAPP-USER", "LAPP-Disk" ] desy_server = 'DESY-ZN-USER' servers_with_miniconda = [ 'LCG.IN2P3-CC.fr', 'LCG.DESY-ZEUTHEN.de', 'LCG.CNAF.it', 'LCG.GRIF.fr', 'LCG.CYFRONET.pl', 'LCG.Prague.cz', 'LCG.CIEMAT.es' ] chunks = np.array_split(sorted(simtel_files), int(len(simtel_files) / chunksize)) print('Got a total of {} chunks'.format(len(chunks))) for c, simtel_filenames in tqdm(enumerate( chunks[0:2])): # send just 2 jobs for now. # convert chunk to a list of strings. becasue this dirac thing cant take numpy arrays simtel_filenames = [ str(s).strip() for s in simtel_filenames if 'SCT' not in s ] print('Starting processing for chunk {}'.format(c)) print(simtel_filenames) j = Job() # set runtime to 0.5h j.setCPUTime(30 * 60) j.setName('cta_preprocessing_{}'.format(c)) j.setInputData(simtel_filenames) j.setOutputData(['./processing_output/*.hdf5'], outputSE=None, outputPath='cta_preprocessing/') j.setInputSandbox( ['../process_simtel.py', './install_dependencies.py']) j.setOutputSandbox(['cta_preprocessing.log']) j.setExecutable('./job_script.sh') # These servers seem to have mini conda installed # destination = np.random.choice(servers_with_miniconda) j.setDestination(servers_with_miniconda) value = dirac.submit(j) print('Number {} Submission Result: {}'.format(c, value))
inputdata_list.append(lfn + 'second/HPC_data.tar.gz.' + str(i)) #SitesList = ['LCG.UKI-NORTHGRID-MAN-HEP.uk', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.UKI-LT2-QMUL.uk', 'LCG.UKI-NORTHGRID-LANCS-HEP.uk'] #SEList = ['UKI-NORTHGRID-MAN-HEP-disk', 'UKI-LT2-IC-HEP-disk', 'UKI-LT2-QMUL2-disk', 'UKI-NORTHGRID-LANCS-HEP-disk'] SitesList = ['LCG.UKI-NORTHGRID-MAN-HEP.uk'] SEList = ['UKI-NORTHGRID-MAN-HEP-disk'] print("Input data list:\n") print(inputdata_list) for i in range(0, total_pixels, chunks): id_start = i id_end = i + chunks dirac = Dirac() j.setName('CS Faraday Rotation Measurement Reconstruction - Pixels from ' + str(id_start) + ' to ' + str(id_end - 1)) j.setPlatform('EL7') j.setTag([str(nprocs) + 'Processors']) j.setDestination(SitesList) j.setExecutable('RMSynthesis2.sh', arguments=str(nprocs) + ' ' + str(id_start) + ' ' + str(id_end) + ' ' + str(expmnt)) # Input data j.setInputData(inputdata_list) j.setInputSandbox([ 'RMSynthesis2.sh', 'run2.sh', 'prmon_1.0.1_x86_64-static-gnu72-opt.tar.gz' ]) # Output data j.setOutputSandbox([ 'StdOut', 'StdErr', 'outputtxt_' + str(id_start) + '_' + str(id_end - 1) + '.txt', 'prmon' + str(id_start) + '_' + str(id_end - 1) + '.txt' ])
#from tests.Workflow.Integration.Test_UserJobs import createJob gLogger.setLevel( 'DEBUG' ) cwd = os.path.realpath( '.' ) dirac = Dirac() # Simple Hello Word job to DIRAC.Jenkins.ch gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch" ) helloJ = Job() helloJ.setName( "helloWorld-TEST-TO-Jenkins" ) helloJ.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJ.setExecutable( "exe-script.py", "", "helloWorld.log" ) helloJ.setCPUTime( 17800 ) helloJ.setDestination( 'DIRAC.Jenkins.ch' ) result = dirac.submit( helloJ ) gLogger.info( "Hello world job: ", result ) if not result['OK']: gLogger.error( "Problem submitting job", result['Message'] ) exit( 1 ) # Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN" ) helloJMP = Job() helloJMP.setName( "helloWorld-TEST-TO-Jenkins-MP" ) helloJMP.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJMP.setExecutable( "exe-script.py", "", "helloWorld.log" ) helloJMP.setCPUTime( 17800 ) helloJMP.setDestination( 'DIRAC.Jenkins.ch' ) helloJMP.setTag('MultiProcessor')
#from tests.Workflow.Integration.Test_UserJobs import createJob gLogger.setLevel('DEBUG') cwd = os.path.realpath('.') dirac = Dirac() # Simple Hello Word job to DIRAC.Jenkins.ch gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch") helloJ = Job() helloJ.setName("helloWorld-TEST-TO-Jenkins") helloJ.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]]) helloJ.setExecutable("exe-script.py", "", "helloWorld.log") helloJ.setCPUTime(1780) helloJ.setDestination('DIRAC.Jenkins.ch') helloJ.setLogLevel('DEBUG') result = dirac.submitJob(helloJ) gLogger.info("Hello world job: ", result) if not result['OK']: gLogger.error("Problem submitting job", result['Message']) exit(1) # Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN") helloJMP = Job() helloJMP.setName("helloWorld-TEST-TO-Jenkins-MP") helloJMP.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]]) helloJMP.setExecutable("exe-script.py", "", "helloWorld.log") helloJMP.setCPUTime(1780) helloJMP.setDestination('DIRAC.Jenkins.ch')
else: ## base_output_path = PWD j.setOutputData(output_files, outputSE=se, outputPath=outdir) print 'Output files = ', output_files print 'outputPath = ', outdir ##################### ## PREPARE JOB ## ##################### if (DO_NOT_SUBMIT): sys.exit(os.EX_USAGE) ### ALWAYS, INFO, VERBOSE, WARN, DEBUG j.setLogLevel('debug') j.setDestination(site_dirac) JOB_IDX = first_job + 1 + idx JOB_NAME = PROD_NAME + " IDX_" + str(JOB_IDX) print '\nJOB NAME is : ', JOB_NAME j.setName(JOB_NAME) j.setCPUTime(JOB_CPUTIME) ## 4 days run_corsika_sim_args = input_file_base + " " + corsika_version + " " + corsika_bin j.setExecutable( './run_corsika_sim', arguments = run_corsika_sim_args, logFile='run_sim.log') if (TEST_JOB) : jobID = dirac.submit(j,mode='local') else : jobID = dirac.submit(j)
print("test run -- only submitting one job") exit() try: os.remove("datapipe.tar.gz") os.remove("tino_cta.tar.gz") except: pass print("\nall done -- exiting now") exit() # ######## ### ####### # ## ## ## ## ## # ## ## ## ## ## # ###### ## ## ## ## # ## ######### ## ## ## # ## ## ## ## ## # ## ## ## ##### ## # specify allowed sites to send the job to j.setDestination(['LCG.IN2P3-CC.fr', 'LCG.DESY-ZEUTHEN.de', 'LCG.CNAF.it', 'LCG.GRIF.fr', 'LCG.CYFRONET.pl', 'LCG.PRAGUE-CESNET.cz', 'LCG.Prague.cz', 'LCG.LAPP.fr', 'LCG.CIEMAT.es', 'LCG.PIC.es']) # to specify input GRID files independent of the site the job is send to file1 = 'LFN:/vo.cta.in2p3.fr/user/c/ciro.bigongiari/MiniArray9/Simtel/'\ 'gamma/run1011.simtel.gz' j.setInputSandbox([file1])
args = visit + ' ' + insidename + ' ' + str(startsensor) + ' ' + str( numsensors) + ' ' + str(idx) outputname = 'fits_' + visit + '_' + str(idx) + '.tar' j.setCPUTime(1209600) j.setExecutable('runimsim2.1.sh', arguments=args) j.stderr = "std.err" j.stdout = "std.out" #!!! May need the 2.1i directory here depending on visit number !!! j.setInputSandbox([ "runimsim2.1.sh", "run_imsim_nersc.py", "LFN:/lsst/user/j/james.perry/instcats/2.1i/" + instcatname ]) j.setOutputSandbox(["std.out", "std.err"]) j.setTag(["8Processors"]) #j.setOutputData([visit + "/" + outputname], outputPath="", outputSE=["IN2P3-CC-disk"]) j.setOutputData([visit + "/" + outputname], outputPath="", outputSE=["UKI-NORTHGRID-LANCS-HEP-disk"]) j.setPlatform("AnyPlatform") j.setDestination(site) jobID = dirac.submitJob(j) print("Submitted job to " + site + " as ID " + str(jobID)) print "Status is:", dirac.status(jobID['JobID']) joblistfile.write(str(jobID['JobID']) + '\n') joblistfile.close()
j.stdout = "std.out" #!!! May need the 2.1i directory here depending on visit number !!! j.setInputSandbox([ "launch_container_userns.sh", "docker_run.sh", "run_imsim_nersc.py", "parsl_imsim_configs", "finals2000A.all", "LFN:/lsst/user/j/james.perry/instcats/2.2i/y04/" + instcatname ]) j.setOutputSandbox(["std.out", "std.err"]) #j.setTag(["4Processors"]) j.setOutputData([outputname], outputPath="", outputSE=["UKI-NORTHGRID-LANCS-HEP-disk"]) #j.setOutputData([outputname], outputPath="", outputSE=["IN2P3-CC-disk"]) j.setPlatform("EL7") # FIXME: remove these once those sites are working again #j.setBannedSites(["VAC.UKI-NORTHGRID-MAN-HEP.uk", "LCG.IN2P3-CC.fr"]) #j.setDestination("LCG.UKI-LT2-IC-HEP.uk") #j.setDestination("LCG.UKI-SOUTHGRID-OX-HEP.uk") j.setDestination("CLOUD.UK-CAM-CUMULUS-backfill.uk") #j.setDestination("LCG.RAL-LCG2.uk") #j.setDestination("LCG.IN2P3-CC.fr") jobID = dirac.submitJob(j) print("Submitted job as ID " + str(jobID)) print "Status is:", dirac.status(jobID['JobID']) joblistfile.write(str(jobID['JobID']) + '\n') joblistfile.close()
# Here we add the list of LFNs we have obtained from the metadata # query. j.setInputSandbox(lfns) #...and added to the desried storage element with the corresponding # LFN via the job's OutputData. You may wish to change: # * The Storage Element - by changing the outputSE parameter; # * The LFN base name - by changing the outputPath parameter. #j.setOutputData(file_dict.keys(), \ # outputSE='GLASGOW-disk', \ # outputPath='/diractest%03d/' % (jobnum)\ # ) # These are the files retrieved with the local job output. j.setOutputSandbox(['StdOut', 'StdErr']) # You can set your preferred site here. #j.setDestination('LCG.Liverpool.uk') #j.setDestination('LCG.Glasgow.uk') j.setDestination('LCG.UKI-LT2-QMUL.uk') ## The DIRAC instance. dirac = Dirac() # Submit the job and print the result. #result = dirac.submit(j) print print 'Submission result: ', result print
break try: os.remove("datapipe.tar.gz") os.remove("modules.tar.gz") except: pass print("\nall done -- exiting now") exit() # ######## ### ####### # ## ## ## ## ## # ## ## ## ## ## # ###### ## ## ## ## # ## ######### ## ## ## # ## ## ## ## ## # ## ## ## ##### ## # specify allowed sites to send the job to j.setDestination([ 'LCG.IN2P3-CC.fr', 'LCG.DESY-ZEUTHEN.de', 'LCG.CNAF.it', 'LCG.GRIF.fr', 'LCG.CYFRONET.pl', 'LCG.PRAGUE-CESNET.cz', 'LCG.Prague.cz', 'LCG.LAPP.fr', 'LCG.CIEMAT.es', 'LCG.PIC.es' ]) # to specify input GRID files independent of the site the job is send to file1 = 'LFN:/vo.cta.in2p3.fr/user/c/ciro.bigongiari/MiniArray9/Simtel/'\ 'gamma/run1011.simtel.gz' j.setInputSandbox([file1])