def test_SimpleParametricJob():
job = Job()
job.setExecutable("myExec")
job.setLogLevel("DEBUG")
parList = [1, 2, 3]
job.setParameterSequence("JOB_ID", parList, addToWorkflow=True)
inputDataList = [
["/lhcb/data/data1", "/lhcb/data/data2"],
["/lhcb/data/data3", "/lhcb/data/data4"],
["/lhcb/data/data5", "/lhcb/data/data6"],
]
job.setParameterSequence("InputData", inputDataList, addToWorkflow=True)
jdl = job._toJDL()
with open(join(dirname(__file__), "testWF.jdl")) as fd:
expected = fd.read()
assert jdl == expected
clad = ClassAd("[" + jdl + "]")
arguments = clad.getAttributeString("Arguments")
job_id = clad.getAttributeString("JOB_ID")
inputData = clad.getAttributeString("InputData")
assert job_id == "%(JOB_ID)s"
assert inputData == "%(InputData)s"
assert "jobDescription.xml" in arguments
assert "-o LogLevel=DEBUG" in arguments
assert "-p JOB_ID=%(JOB_ID)s" in arguments
assert "-p InputData=%(InputData)s" in arguments
def test_SimpleParametricJob(self):
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
parList = [1, 2, 3]
job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'],
['/lhcb/data/data3', '/lhcb/data/data4'],
['/lhcb/data/data5', '/lhcb/data/data6']]
job.setParameterSequence('InputData',
inputDataList,
addToWorkflow=True)
jdl = job._toJDL()
print jdl
clad = ClassAd('[' + jdl + ']')
arguments = clad.getAttributeString('Arguments')
job_id = clad.getAttributeString('JOB_ID')
inputData = clad.getAttributeString('InputData')
print "arguments", arguments
self.assertEqual(job_id, '%(JOB_ID)s')
self.assertEqual(inputData, '%(InputData)s')
self.assertIn('jobDescription.xml', arguments)
self.assertIn('-o LogLevel=DEBUG', arguments)
self.assertIn('-p JOB_ID=%(JOB_ID)s', arguments)
self.assertIn('-p InputData=%(InputData)s', arguments)
def test_SimpleParametricJob():
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
parList = [1, 2, 3]
job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'],
['/lhcb/data/data3', '/lhcb/data/data4'],
['/lhcb/data/data5', '/lhcb/data/data6']]
job.setParameterSequence('InputData', inputDataList, addToWorkflow=True)
jdl = job._toJDL()
try:
with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
assert jdl == expected
clad = ClassAd('[' + jdl + ']')
arguments = clad.getAttributeString('Arguments')
job_id = clad.getAttributeString('JOB_ID')
inputData = clad.getAttributeString('InputData')
assert job_id == '%(JOB_ID)s'
assert inputData == '%(InputData)s'
assert 'jobDescription.xml' in arguments
assert '-o LogLevel=DEBUG' in arguments
assert '-p JOB_ID=%(JOB_ID)s' in arguments
assert '-p InputData=%(InputData)s' in arguments
def parametricJob():
job = Job()
job.setName("parametric_helloWorld_%n")
exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
job.setInputSandbox(exeScriptLocation)
job.setParameterSequence("args", ['one', 'two', 'three'])
job.setParameterSequence("iargs", [1, 2, 3])
job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log')
return job
def parametricJob():
job = Job()
job.setName("parametric_helloWorld_%n")
exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
job.setInputSandbox(exeScriptLocation)
job.setParameterSequence("args", ['one', 'two', 'three'])
job.setParameterSequence("iargs", [1, 2, 3])
job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log')
return job
def parametricJob():
job = Job()
job.setName("parametric_helloWorld_%n")
exeScriptLocation = find_all("exe-script.py", "..", "/DIRAC/tests/Integration")[0]
job.setInputSandbox(exeScriptLocation)
job.setParameterSequence("args", ["one", "two", "three"])
job.setParameterSequence("iargs", [1, 2, 3])
job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile="helloWorld_%n.log")
return job
def submit(name,
job_group,
task_id,
input_sandbox,
output_sandbox,
executable,
site=None,
banned_site=None,
sub_ids=[]):
dirac = Dirac()
submit_result = {'backend_job_ids': {}}
jobInfos = {}
for run in range(int((len(sub_ids) + 99) / 100)):
ids_this_run = [x for x in sub_ids[run * 100:(run + 1) * 100]]
job_names = ['%s.%s' % (name, sub_id) for sub_id in ids_this_run]
j = Job()
j.setName(name)
j.setExecutable(executable)
j.setParameterSequence('JobName', job_names, addToWorkflow=True)
j.setParameterSequence('arguments', ids_this_run, addToWorkflow=True)
if input_sandbox:
j.setInputSandbox(input_sandbox)
if output_sandbox:
j.setOutputSandbox(output_sandbox)
if job_group:
j.setJobGroup(job_group)
if site: # set destination to a certain site; list not allowed
j.setDestination(site)
if banned_site:
j.setBannedSites(banned_site)
result = dirac.submitJob(j)
if not result['OK']:
sys.stdout.write('DIRAC job submit error: %s\n' %
result['Message'])
sys.exit(1)
for sub_id, dirac_id in zip(ids_this_run, result['Value']):
submit_result['backend_job_ids'][sub_id] = dirac_id
jobInfos[dirac_id] = {'SubID': sub_id}
#Register on Task-manager Webapp of IHEPDIRAC
task = RPCClient('WorkloadManagement/TaskManager')
taskInfo = {'TaskName': name, 'JobGroup': job_group, 'JSUB-ID': task_id}
task_result = task.createTask(name, taskInfo, jobInfos)
task_web_id = task_result['Value']
submit_result['backend_task_id'] = task_web_id
return submit_result
def test_SimpleParametricJob():
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
parList = [1, 2, 3]
job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
inputDataList = [
[
'/lhcb/data/data1',
'/lhcb/data/data2'
],
[
'/lhcb/data/data3',
'/lhcb/data/data4'
],
[
'/lhcb/data/data5',
'/lhcb/data/data6'
]
]
job.setParameterSequence('InputData', inputDataList, addToWorkflow=True)
jdl = job._toJDL()
try:
with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
assert jdl == expected
clad = ClassAd('[' + jdl + ']')
arguments = clad.getAttributeString('Arguments')
job_id = clad.getAttributeString('JOB_ID')
inputData = clad.getAttributeString('InputData')
assert job_id == '%(JOB_ID)s'
assert inputData == '%(InputData)s'
assert 'jobDescription.xml' in arguments
assert '-o LogLevel=DEBUG' in arguments
assert'-p JOB_ID=%(JOB_ID)s' in arguments
assert'-p InputData=%(InputData)s' in arguments
def test_SimpleParametricJob( self ):
job = Job()
job.setExecutable( 'myExec' )
job.setLogLevel( 'DEBUG' )
parList = [1,2,3]
job.setParameterSequence( 'JOB_ID', parList, addToWorkflow=True )
inputDataList = [
[
'/lhcb/data/data1',
'/lhcb/data/data2'
],
[
'/lhcb/data/data3',
'/lhcb/data/data4'
],
[
'/lhcb/data/data5',
'/lhcb/data/data6'
]
]
job.setParameterSequence( 'InputData', inputDataList, addToWorkflow=True )
jdl = job._toJDL()
print jdl
clad = ClassAd( '[' + jdl + ']' )
arguments = clad.getAttributeString( 'Arguments' )
job_id = clad.getAttributeString( 'JOB_ID' )
inputData = clad.getAttributeString( 'InputData' )
print "arguments", arguments
self.assertEqual( job_id, '%(JOB_ID)s' )
self.assertEqual( inputData, '%(InputData)s' )
self.assertIn( 'jobDescription.xml', arguments )
self.assertIn( '-o LogLevel=DEBUG', arguments )
self.assertIn( '-p JOB_ID=%(JOB_ID)s', arguments )
self.assertIn( '-p InputData=%(InputData)s', arguments )
def submit_job(self,
request_id,
executable,
macro,
starting_seed=8000000,
njobs=10,
platform='ANY',
output_data_site='UKI-LT2-IC-HEP-disk',
output_log='lzproduction_output.log'):
"""
Submit LZProduction job to DIRAC.
Args:
request_id (int): The id number of the associated request
executable (str): The full path to the executable job script
macro (str): The full path to the macro for this job
starting_seed (int): The random seed for the first of the parametric jobs
njobs (int): The number of parametric jobs to create
platform (str): The required platform
output_data_site (str): The name of the grid site to store the output data at
output_log (str): The file name for the output log file
Returns:
list: The list of created parametric job DIRAC ids
"""
j = Job()
j.setName(os.path.splitext(os.path.basename(macro))[0] + '%(args)s')
j.setExecutable(os.path.basename(executable),
os.path.basename(macro) + ' %(args)s', output_log)
j.setInputSandbox([executable, macro])
j.setOutputData('*.root', output_data_site, str(request_id))
j.setParameterSequence(
"args",
[str(i) for i in xrange(starting_seed, starting_seed + njobs)],
addToWorkflow=True)
j.setPlatform(platform)
return self.status(self._dirac_api.submit(j).get("Value", []))