def test_SimpleParametricJob(): job = Job() job.setExecutable("myExec") job.setLogLevel("DEBUG") parList = [1, 2, 3] job.setParameterSequence("JOB_ID", parList, addToWorkflow=True) inputDataList = [ ["/lhcb/data/data1", "/lhcb/data/data2"], ["/lhcb/data/data3", "/lhcb/data/data4"], ["/lhcb/data/data5", "/lhcb/data/data6"], ] job.setParameterSequence("InputData", inputDataList, addToWorkflow=True) jdl = job._toJDL() with open(join(dirname(__file__), "testWF.jdl")) as fd: expected = fd.read() assert jdl == expected clad = ClassAd("[" + jdl + "]") arguments = clad.getAttributeString("Arguments") job_id = clad.getAttributeString("JOB_ID") inputData = clad.getAttributeString("InputData") assert job_id == "%(JOB_ID)s" assert inputData == "%(InputData)s" assert "jobDescription.xml" in arguments assert "-o LogLevel=DEBUG" in arguments assert "-p JOB_ID=%(JOB_ID)s" in arguments assert "-p InputData=%(InputData)s" in arguments
def test_SimpleParametricJob(self): job = Job() job.setExecutable('myExec') job.setLogLevel('DEBUG') parList = [1, 2, 3] job.setParameterSequence('JOB_ID', parList, addToWorkflow=True) inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'], ['/lhcb/data/data3', '/lhcb/data/data4'], ['/lhcb/data/data5', '/lhcb/data/data6']] job.setParameterSequence('InputData', inputDataList, addToWorkflow=True) jdl = job._toJDL() print jdl clad = ClassAd('[' + jdl + ']') arguments = clad.getAttributeString('Arguments') job_id = clad.getAttributeString('JOB_ID') inputData = clad.getAttributeString('InputData') print "arguments", arguments self.assertEqual(job_id, '%(JOB_ID)s') self.assertEqual(inputData, '%(InputData)s') self.assertIn('jobDescription.xml', arguments) self.assertIn('-o LogLevel=DEBUG', arguments) self.assertIn('-p JOB_ID=%(JOB_ID)s', arguments) self.assertIn('-p InputData=%(InputData)s', arguments)
def test_SimpleParametricJob(): job = Job() job.setExecutable('myExec') job.setLogLevel('DEBUG') parList = [1, 2, 3] job.setParameterSequence('JOB_ID', parList, addToWorkflow=True) inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'], ['/lhcb/data/data3', '/lhcb/data/data4'], ['/lhcb/data/data5', '/lhcb/data/data6']] job.setParameterSequence('InputData', inputDataList, addToWorkflow=True) jdl = job._toJDL() try: with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWF.jdl') as fd: expected = fd.read() assert jdl == expected clad = ClassAd('[' + jdl + ']') arguments = clad.getAttributeString('Arguments') job_id = clad.getAttributeString('JOB_ID') inputData = clad.getAttributeString('InputData') assert job_id == '%(JOB_ID)s' assert inputData == '%(InputData)s' assert 'jobDescription.xml' in arguments assert '-o LogLevel=DEBUG' in arguments assert '-p JOB_ID=%(JOB_ID)s' in arguments assert '-p InputData=%(InputData)s' in arguments
def parametricJob(): job = Job() job.setName("parametric_helloWorld_%n") exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0] job.setInputSandbox(exeScriptLocation) job.setParameterSequence("args", ['one', 'two', 'three']) job.setParameterSequence("iargs", [1, 2, 3]) job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log') return job
def parametricJob(): job = Job() job.setName("parametric_helloWorld_%n") exeScriptLocation = find_all("exe-script.py", "..", "/DIRAC/tests/Integration")[0] job.setInputSandbox(exeScriptLocation) job.setParameterSequence("args", ["one", "two", "three"]) job.setParameterSequence("iargs", [1, 2, 3]) job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile="helloWorld_%n.log") return job
def submit(name, job_group, task_id, input_sandbox, output_sandbox, executable, site=None, banned_site=None, sub_ids=[]): dirac = Dirac() submit_result = {'backend_job_ids': {}} jobInfos = {} for run in range(int((len(sub_ids) + 99) / 100)): ids_this_run = [x for x in sub_ids[run * 100:(run + 1) * 100]] job_names = ['%s.%s' % (name, sub_id) for sub_id in ids_this_run] j = Job() j.setName(name) j.setExecutable(executable) j.setParameterSequence('JobName', job_names, addToWorkflow=True) j.setParameterSequence('arguments', ids_this_run, addToWorkflow=True) if input_sandbox: j.setInputSandbox(input_sandbox) if output_sandbox: j.setOutputSandbox(output_sandbox) if job_group: j.setJobGroup(job_group) if site: # set destination to a certain site; list not allowed j.setDestination(site) if banned_site: j.setBannedSites(banned_site) result = dirac.submitJob(j) if not result['OK']: sys.stdout.write('DIRAC job submit error: %s\n' % result['Message']) sys.exit(1) for sub_id, dirac_id in zip(ids_this_run, result['Value']): submit_result['backend_job_ids'][sub_id] = dirac_id jobInfos[dirac_id] = {'SubID': sub_id} #Register on Task-manager Webapp of IHEPDIRAC task = RPCClient('WorkloadManagement/TaskManager') taskInfo = {'TaskName': name, 'JobGroup': job_group, 'JSUB-ID': task_id} task_result = task.createTask(name, taskInfo, jobInfos) task_web_id = task_result['Value'] submit_result['backend_task_id'] = task_web_id return submit_result
def test_SimpleParametricJob(): job = Job() job.setExecutable('myExec') job.setLogLevel('DEBUG') parList = [1, 2, 3] job.setParameterSequence('JOB_ID', parList, addToWorkflow=True) inputDataList = [ [ '/lhcb/data/data1', '/lhcb/data/data2' ], [ '/lhcb/data/data3', '/lhcb/data/data4' ], [ '/lhcb/data/data5', '/lhcb/data/data6' ] ] job.setParameterSequence('InputData', inputDataList, addToWorkflow=True) jdl = job._toJDL() try: with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWF.jdl') as fd: expected = fd.read() assert jdl == expected clad = ClassAd('[' + jdl + ']') arguments = clad.getAttributeString('Arguments') job_id = clad.getAttributeString('JOB_ID') inputData = clad.getAttributeString('InputData') assert job_id == '%(JOB_ID)s' assert inputData == '%(InputData)s' assert 'jobDescription.xml' in arguments assert '-o LogLevel=DEBUG' in arguments assert'-p JOB_ID=%(JOB_ID)s' in arguments assert'-p InputData=%(InputData)s' in arguments
def test_SimpleParametricJob( self ): job = Job() job.setExecutable( 'myExec' ) job.setLogLevel( 'DEBUG' ) parList = [1,2,3] job.setParameterSequence( 'JOB_ID', parList, addToWorkflow=True ) inputDataList = [ [ '/lhcb/data/data1', '/lhcb/data/data2' ], [ '/lhcb/data/data3', '/lhcb/data/data4' ], [ '/lhcb/data/data5', '/lhcb/data/data6' ] ] job.setParameterSequence( 'InputData', inputDataList, addToWorkflow=True ) jdl = job._toJDL() print jdl clad = ClassAd( '[' + jdl + ']' ) arguments = clad.getAttributeString( 'Arguments' ) job_id = clad.getAttributeString( 'JOB_ID' ) inputData = clad.getAttributeString( 'InputData' ) print "arguments", arguments self.assertEqual( job_id, '%(JOB_ID)s' ) self.assertEqual( inputData, '%(InputData)s' ) self.assertIn( 'jobDescription.xml', arguments ) self.assertIn( '-o LogLevel=DEBUG', arguments ) self.assertIn( '-p JOB_ID=%(JOB_ID)s', arguments ) self.assertIn( '-p InputData=%(InputData)s', arguments )
def submit_job(self, request_id, executable, macro, starting_seed=8000000, njobs=10, platform='ANY', output_data_site='UKI-LT2-IC-HEP-disk', output_log='lzproduction_output.log'): """ Submit LZProduction job to DIRAC. Args: request_id (int): The id number of the associated request executable (str): The full path to the executable job script macro (str): The full path to the macro for this job starting_seed (int): The random seed for the first of the parametric jobs njobs (int): The number of parametric jobs to create platform (str): The required platform output_data_site (str): The name of the grid site to store the output data at output_log (str): The file name for the output log file Returns: list: The list of created parametric job DIRAC ids """ j = Job() j.setName(os.path.splitext(os.path.basename(macro))[0] + '%(args)s') j.setExecutable(os.path.basename(executable), os.path.basename(macro) + ' %(args)s', output_log) j.setInputSandbox([executable, macro]) j.setOutputData('*.root', output_data_site, str(request_id)) j.setParameterSequence( "args", [str(i) for i in xrange(starting_seed, starting_seed + njobs)], addToWorkflow=True) j.setPlatform(platform) return self.status(self._dirac_api.submit(j).get("Value", []))