def submitProbeJobs(self, ce): """ Submit some jobs to the CEs """ #need credentials, should be there since the initialize from DIRAC.Interfaces.API.Dirac import Dirac d = Dirac() from DIRAC.Interfaces.API.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations import os ops = Operations("glast.org") scriptname = ops.getValue("ResourceStatus/SofwareManagementScript", self.script) j = Job() j.setDestinationCE(ce) j.setCPUTime(1000) j.setName("Probe %s" % ce) j.setJobGroup("SoftwareProbe") j.setExecutable("%s/GlastDIRAC/ResourceStatusSystem/Client/%s" % (os.environ['DIRAC'], scriptname), logFile='SoftwareProbe.log') j.setOutputSandbox('*.log') res = d.submit(j) if not res['OK']: return res return S_OK()
def __submit( self, site, CE, vo ): """ set the job and submit. """ job = Job() job.setName( self.testType ) job.setJobGroup( 'CE-Test' ) job.setExecutable( self.executable ) job.setInputSandbox( '%s/%s' % ( self.__scriptPath, self.executable ) ) if site and not CE: job.setDestination( site ) if CE: job.setDestinationCE( CE ) LOCK.acquire() proxyPath = BESUtils.getProxyByVO( 'zhangxm', vo ) if not proxyPath[ 'OK' ]: LOCK.release() return proxyPath proxyPath = proxyPath[ 'Value' ] oldProxy = os.environ.get( 'X509_USER_PROXY' ) os.environ[ 'X509_USER_PROXY' ] = proxyPath result = self.dirac.submit( job ) if oldProxy is None: del os.environ[ 'X509_USER_PROXY' ] else: os.environ[ 'X509_USER_PROXY' ] = oldProxy LOCK.release() return result
def test_basicJob(): job = Job() job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setExecutable('someExe') job.setType('jobType') job.setDestination('ANY') xml = job._toXML() try: with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWF.xml') as fd: expected = fd.read() assert xml == expected try: with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML())) assert jdlSIO == expected
def SoftClean(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) != 3): Script.showHelp() package = args[0] version = args[1] site = args[2] j = Job() j.setInputSandbox(['cta-swclean.py']) arguments = package + ' ' + version j.setExecutable('./cta-swclean.py', arguments) j.setDestination([site]) name = 'SoftClean_' + package + '_' + version j.setName('SoftClean') j.setJobGroup('SoftClean') j.setCPUTime(100000) Script.gLogger.info(j._toJDL()) res = Dirac().submit(j) print res['Value']
def test_execute(self): job = Job() job.setName("helloWorld-test") job.setExecutable(find_all("helloWorld.py", '.', 'Integration')[0], arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ('applicationLog', 'string', '', "Log file name"), ('someCustomOne', 'string', '', "boh")], paramValues=[('someCustomOne', 'aCustomValue')]) job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org']) job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setType('jobType') job.setDestination('DIRAC.someSite.ch') job.setCPUTime(12345) job.setLogLevel('DEBUG') res = job.runLocal(self.d) self.assertTrue(res['OK'])
def submitProbeJobs(self, ce): """ Submit some jobs to the CEs """ #need credentials, should be there since the initialize from DIRAC.Interfaces.API.Dirac import Dirac d = Dirac() from DIRAC.Interfaces.API.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations import DIRAC ops = Operations() scriptname = ops.getValue("ResourceStatus/SofwareManagementScript", self.script) j = Job() j.setDestinationCE(ce) j.setCPUTime(1000) j.setName("Probe %s" % ce) j.setJobGroup("SoftwareProbe") j.setExecutable("%s/GlastDIRAC/ResourceStatusSystem/Client/%s" % (DIRAC.rootPath, scriptname), logFile='SoftwareProbe.log') j.setOutputSandbox('*.log') res = d.submit(j) if not res['OK']: return res return S_OK()
def test_basicJob(): job = Job() job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setExecutable('someExe') job.setType('jobType') job.setDestination('ANY') xml = job._toXML() try: with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWF.xml') as fd: expected = fd.read() assert xml == expected try: with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML())) assert jdlSIO == expected
def __submit(self, site, CE, vo): """ set the job and submit. """ job = Job() job.setName(self.testType) job.setJobGroup('CE-Test') job.setExecutable(self.executable) job.setInputSandbox('%s/%s' % (self.__scriptPath, self.executable)) if site and not CE: job.setDestination(site) if CE: job.setDestinationCE(CE) LOCK.acquire() proxyPath = BESUtils.getProxyByVO('zhangxm', vo) if not proxyPath['OK']: LOCK.release() return proxyPath proxyPath = proxyPath['Value'] oldProxy = os.environ.get('X509_USER_PROXY') os.environ['X509_USER_PROXY'] = proxyPath result = self.dirac.submitJob(job) if oldProxy is None: del os.environ['X509_USER_PROXY'] else: os.environ['X509_USER_PROXY'] = oldProxy LOCK.release() return result
def test_execute( self ): job = Job() job.setName( "helloWorld-test" ) job.setExecutable( find_all( "helloWorld.py", '.', 'Integration' )[0], arguments = "This is an argument", logFile = "aLogFileForTest.txt" , parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ( 'applicationLog', 'string', '', "Log file name" ), ( 'someCustomOne', 'string', '', "boh" )], paramValues = [( 'someCustomOne', 'aCustomValue' )] ) job.setBannedSites( ['LCG.SiteA.com', 'DIRAC.SiteB.org'] ) job.setOwner( 'ownerName' ) job.setOwnerGroup( 'ownerGroup' ) job.setName( 'jobName' ) job.setJobGroup( 'jobGroup' ) job.setType( 'jobType' ) job.setDestination( 'DIRAC.someSite.ch' ) job.setCPUTime( 12345 ) job.setLogLevel( 'DEBUG' ) res = job.runLocal( self.d ) self.assertTrue( res['OK'] )
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''): if (not owner) or (not ownerGroup): res = getProxyInfo(False,False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] oJob = Job(transBody) for taskNumber in sortList(taskDict.keys()): paramsDict = taskDict[taskNumber] transID = paramsDict['TransformationID'] self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup)) oJob.setOwner(owner) oJob.setOwnerGroup(ownerGroup) transGroup = str(transID).zfill(8) self.log.verbose('Adding default transformation group of %s' % (transGroup)) oJob.setJobGroup(transGroup) constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8) self.log.verbose('Setting task name to %s' % constructedName) oJob.setName(constructedName) oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8)) oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8)) inputData = None for paramName,paramValue in paramsDict.items(): self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue)) if paramName=='InputData': if paramValue: self.log.verbose('Setting input data to %s' %paramValue) oJob.setInputData(paramValue) elif paramName=='Site': if paramValue: self.log.verbose('Setting allocated site to: %s' %(paramValue)) oJob.setDestination(paramValue) elif paramValue: self.log.verbose('Setting %s to %s' % (paramName,paramValue)) oJob._addJDLParameter(paramName,paramValue) hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])] if int(transID) in hospitalTrans: hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch') hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[]) oJob.setType('Hospital') oJob.setDestination(hospitalSite) oJob.setInputDataPolicy('download',dataScheduling=False) if hospitalCEs: oJob._addJDLParameter('GridRequiredCEs',hospitalCEs) taskDict[taskNumber]['TaskObject'] = '' res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData}) if not res ['OK']: self.log.error("Failed to generate output data",res['Message']) continue for name,output in res['Value'].items(): oJob._addJDLParameter(name,string.join(output,';')) taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML()) return S_OK(taskDict)
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''): if (not owner) or (not ownerGroup): res = getProxyInfo(False,False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] oJob = Job(transBody) for taskNumber in sortList(taskDict.keys()): paramsDict = taskDict[taskNumber] transID = paramsDict['TransformationID'] self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup)) oJob.setOwner(owner) oJob.setOwnerGroup(ownerGroup) transGroup = str(transID).zfill(8) self.log.verbose('Adding default transformation group of %s' % (transGroup)) oJob.setJobGroup(transGroup) constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8) self.log.verbose('Setting task name to %s' % constructedName) oJob.setName(constructedName) oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8)) oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8)) inputData = None for paramName,paramValue in paramsDict.items(): self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue)) if paramName=='InputData': if paramValue: self.log.verbose('Setting input data to %s' %paramValue) oJob.setInputData(paramValue) elif paramName=='Site': if paramValue: self.log.verbose('Setting allocated site to: %s' %(paramValue)) oJob.setDestination(paramValue) elif paramValue: self.log.verbose('Setting %s to %s' % (paramName,paramValue)) oJob._addJDLParameter(paramName,paramValue) hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])] if int(transID) in hospitalTrans: hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch') hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[]) oJob.setType('Hospital') oJob.setDestination(hospitalSite) oJob.setInputDataPolicy('download',dataScheduling=False) if hospitalCEs: oJob._addJDLParameter('GridRequiredCEs',hospitalCEs) taskDict[taskNumber]['TaskObject'] = '' res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData}) if not res ['OK']: self.log.error("Failed to generate output data",res['Message']) continue for name,output in res['Value'].items(): oJob._addJDLParameter(name,string.join(output,';')) taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML()) return S_OK(taskDict)
def submit(name, job_group, task_id, input_sandbox, output_sandbox, executable, site=None, banned_site=None, sub_ids=[]): dirac = Dirac() submit_result = {'backend_job_ids': {}} jobInfos = {} for run in range(int((len(sub_ids) + 99) / 100)): ids_this_run = [x for x in sub_ids[run * 100:(run + 1) * 100]] job_names = ['%s.%s' % (name, sub_id) for sub_id in ids_this_run] j = Job() j.setName(name) j.setExecutable(executable) j.setParameterSequence('JobName', job_names, addToWorkflow=True) j.setParameterSequence('arguments', ids_this_run, addToWorkflow=True) if input_sandbox: j.setInputSandbox(input_sandbox) if output_sandbox: j.setOutputSandbox(output_sandbox) if job_group: j.setJobGroup(job_group) if site: # set destination to a certain site; list not allowed j.setDestination(site) if banned_site: j.setBannedSites(banned_site) result = dirac.submitJob(j) if not result['OK']: sys.stdout.write('DIRAC job submit error: %s\n' % result['Message']) sys.exit(1) for sub_id, dirac_id in zip(ids_this_run, result['Value']): submit_result['backend_job_ids'][sub_id] = dirac_id jobInfos[dirac_id] = {'SubID': sub_id} #Register on Task-manager Webapp of IHEPDIRAC task = RPCClient('WorkloadManagement/TaskManager') taskInfo = {'TaskName': name, 'JobGroup': job_group, 'JSUB-ID': task_id} task_result = task.createTask(name, taskInfo, jobInfos) task_web_id = task_result['Value'] submit_result['backend_task_id'] = task_web_id return submit_result
def _submitJob(self, result_id, executable, test_name, site_name): executable = executable.split('&') j = Job() j.setExecutable('python', arguments=executable[0] + " " + str(result_id)) sandBox = [] for file_name in executable: sandBox.append(SAM_TEST_DIR + file_name) j.setInputSandbox(sandBox) j.setName(test_name) j.setJobGroup('sam_test') j.setDestination(site_name) result = self.dirac.submit(j) return result
def CorsikaSimtelInstall(args=None): from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job if (len(args) != 2): Script.showHelp() version = args[0] site = args[1] if version not in ['prod-2_13112014']: Script.gLogger.error('Version not valid') Script.showHelp() j = Job() CorsikaSimtelPack = os.path.join('corsika_simhessarray', version, 'corsika_simhessarray') CorsikaSimtelLFN = 'LFN:' + os.path.join('/vo.cta.in2p3.fr/software', CorsikaSimtelPack) + '.tar.gz' j.setInputSandbox(['cta-corsikasimtel-install.py', CorsikaSimtelLFN]) j.setExecutable('./cta-corsikasimtel-install.py', version) j.setDestination([site]) j.setJobGroup('SoftInstall') j.setCPUTime(100000) if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']: if site == 'LCG.GRIF.fr': ceList = [ 'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr', 'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr', 'node74.datagrid.cea.fr' ] if site == 'LCG.M3PEC.fr': ceList = ['ce0.bordeaux.inra.fr', 'ce0.m3pec.u-bordeaux1.fr'] for ce in ceList: j.setDestinationCE(ce) name = 'corsikasimtelInstall' + '_' + ce j.setName(name) Dirac().submit(j) DIRAC.exit() j.setName('corsikasimtelInstall') Script.gLogger.info(j._toJDL()) Dirac().submit(j)
def submitJob(jobPara): dirac = Dirac() j = Job() j.setName(jobPara['jobName']) j.setJobGroup(jobPara['jobGroup']) j.setExecutable(jobPara['jobScript'], logFile = jobPara['jobScriptLog']) j.setInputSandbox(jobPara['inputSandbox']) j.setOutputSandbox(jobPara['outputSandbox']) j.setOutputData(jobPara['outputData'], jobPara['SE']) j.setDestination(jobPara['sites']) j.setCPUTime(jobPara['CPUTime']) result = dirac.submit(j) if result['OK']: print 'Job %s submitted successfully. ID = %d' %(jobPara['jobName'],result['Value']) else: print 'Job %s submitted failed' %jobPara['jobName'] return result
def submitJob(jobPara): dirac = Dirac() j = Job() j.setName(jobPara['jobName']) j.setJobGroup(jobPara['jobGroup']) j.setExecutable(jobPara['jobScript'], logFile=jobPara['jobScriptLog']) j.setInputSandbox(jobPara['inputSandbox']) j.setOutputSandbox(jobPara['outputSandbox']) j.setOutputData(jobPara['outputData'], jobPara['SE']) j.setDestination(jobPara['sites']) j.setCPUTime(jobPara['CPUTime']) result = dirac.submit(j) if result['OK']: print 'Job %s submitted successfully. ID = %d' % (jobPara['jobName'], result['Value']) else: print 'Job %s submitted failed' % jobPara['jobName'] return result
def launch_batch_pict( pitch_start, step, n_pict ): j = Job() j.setCPUTime(500) j.setName('%s_%f' % (EXEC, pitch_start)) j.setJobGroup(JOBGROUP) j.setInputSandbox([EXEC]) out_bmp_list=[] pitch=pitch_start for i in range(n_pict): out_bmp='out_%f.bmp' % pitch out_bmp_list.append(out_bmp) j.setExecutable(EXEC,arguments="-W 600 -H 600 -X -0.77568377 -Y -0.13646737 -P %f -M 500 %s" % (pitch, out_bmp)) pitch+=step j.setOutputSandbox(out_bmp_list + ["StdOut"] + ["StdErr"]) result = dirac.submit(j) print 'Submission Result: ',result return result
def submit(self, param): j = Job() j.setName(param['jobName']) j.setExecutable(param['jobScript'],logFile = param['jobScriptLog']) if self.site: j.setDestination(self.site) if self.jobGroup: j.setJobGroup(self.jobGroup) j.setInputSandbox(param['inputSandbox']) j.setOutputSandbox(param['outputSandbox']) j.setOutputData(param['outputData'], outputSE = self.outputSE, outputPath = self.outputPath) dirac = GridDirac() result = dirac.submit(j) status = {} status['submit'] = result['OK'] if status['submit']: status['job_id'] = result['Value'] return status
def test_execute(self): job = Job() job._siteSet = {"DIRAC.someSite.ch"} job.setName("helloWorld-test") job.setExecutable( self.helloWorld, arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[ ("executable", "string", "", "Executable Script"), ("arguments", "string", "", "Arguments for executable Script"), ("applicationLog", "string", "", "Log file name"), ("someCustomOne", "string", "", "boh"), ], paramValues=[("someCustomOne", "aCustomValue")], ) job.setBannedSites(["LCG.SiteA.com", "DIRAC.SiteB.org"]) job.setOwner("ownerName") job.setOwnerGroup("ownerGroup") job.setName("jobName") job.setJobGroup("jobGroup") job.setType("jobType") job.setDestination("DIRAC.someSite.ch") job.setCPUTime(12345) job.setLogLevel("DEBUG") try: # This is the standard location in Jenkins job.setInputSandbox( find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) except (IndexError, KeyError): job.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) job.setConfigArgs("pilot.cfg") res = job.runLocal(self.d) self.assertTrue(res["OK"])
def submit(self, param): j = Job() j.setName(param['jobName']) j.setExecutable(param['jobScript'], logFile=param['jobScriptLog']) if self.site: j.setDestination(self.site) if self.jobGroup: j.setJobGroup(self.jobGroup) j.setInputSandbox(param['inputSandbox']) j.setOutputSandbox(param['outputSandbox']) j.setOutputData(param['outputData'], outputSE=self.outputSE, outputPath=self.outputPath) dirac = GridDirac() result = dirac.submit(j) status = {} status['submit'] = result['OK'] if status['submit']: status['job_id'] = result['Value'] return status
def test_basicJob(): job = Job() job.setOwner("ownerName") job.setOwnerGroup("ownerGroup") job.setName("jobName") job.setJobGroup("jobGroup") job.setExecutable("someExe") job.setType("jobType") job.setDestination("ANY") xml = job._toXML() with open(join(dirname(__file__), "testWF.xml")) as fd: expected = fd.read() assert xml == expected with open(join(dirname(__file__), "testWFSIO.jdl")) as fd: expected = fd.read() jdlSIO = job._toJDL(jobDescriptionObject=StringIO(job._toXML())) assert jdlSIO == expected
def test_execute(self): job = Job() job._siteSet = {'DIRAC.someSite.ch'} job.setName("helloWorld-test") job.setExecutable(self.helloWorld, arguments="This is an argument", logFile="aLogFileForTest.txt", parameters=[('executable', 'string', '', "Executable Script"), ('arguments', 'string', '', 'Arguments for executable Script'), ('applicationLog', 'string', '', "Log file name"), ('someCustomOne', 'string', '', "boh")], paramValues=[('someCustomOne', 'aCustomValue')]) job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org']) job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setType('jobType') job.setDestination('DIRAC.someSite.ch') job.setCPUTime(12345) job.setLogLevel('DEBUG') try: # This is the standard location in Jenkins job.setInputSandbox( find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]) except (IndexError, KeyError): job.setInputSandbox(find_all('pilot.cfg', rootPath)[0]) job.setConfigArgs('pilot.cfg') res = job.runLocal(self.d) self.assertTrue(res['OK'])
j.setOutputSandbox([ 'StdOut', 'StdErr', 'outputtxt_' + str(id_start) + '_' + str(id_end - 1) + '.txt', 'prmon' + str(id_start) + '_' + str(id_end - 1) + '.txt' ]) o_data_file = lfn + 'second/results_experiment_' + str( expmnt) + '/' + 'LOS_' + str(id_start) + '_to_' + str(id_end - 1) + '.npy' try: output_process = subprocess.check_output('dirac-dms-remove-files ' + o_data_file, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: print 'Failed: ' + str(e.returncode) + ' ' + e.output else: print "Output: ", output_process j.setOutputData( ['LOS_' + str(id_start) + '_to_' + str(id_end - 1) + '.npy'], outputSE=SEList, outputPath='/second/results_experiment_' + str(expmnt)) try: diracUsername = getProxyInfo()['Value']['username'] except: print 'Failed to get DIRAC username. No proxy set up?' sys.exit(1) j.setJobGroup('rmsynthesis_by_' + expmnt + '_' + timestamp) jobID = dirac.submitJob(j) print 'Submission Result: ', j._toJDL() print '\n'