def helloWorldJob():
job = Job()
job.setName("helloWorld")
exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
job.setInputSandbox(exeScriptLocation)
job.setExecutable(exeScriptLocation, "", "helloWorld.log")
return job
def helloWorldJob(): job = Job() job.setName( "helloWorld" ) exeScriptLocation = find_all( 'exe-script.py', '.', 'WorkloadManagementSystem' )[0] job.setInputSandbox( exeScriptLocation ) job.setExecutable( exeScriptLocation, "", "helloWorld.log" ) return job
def __submit( self, site, CE, vo ):
"""
set the job and submit.
"""
job = Job()
job.setName( self.testType )
job.setJobGroup( 'CE-Test' )
job.setExecutable( self.executable )
job.setInputSandbox( '%s/%s' % ( self.__scriptPath, self.executable ) )
if site and not CE:
job.setDestination( site )
if CE:
job.setDestinationCE( CE )
LOCK.acquire()
proxyPath = BESUtils.getProxyByVO( 'zhangxm', vo )
if not proxyPath[ 'OK' ]:
LOCK.release()
return proxyPath
proxyPath = proxyPath[ 'Value' ]
oldProxy = os.environ.get( 'X509_USER_PROXY' )
os.environ[ 'X509_USER_PROXY' ] = proxyPath
result = self.dirac.submit( job )
if oldProxy is None:
del os.environ[ 'X509_USER_PROXY' ]
else:
os.environ[ 'X509_USER_PROXY' ] = oldProxy
LOCK.release()
return result
def job():
from DIRAC.Interfaces.API.Job import Job
job = Job(stdout="printer", stderr="/dev/null")
job.setInputSandbox([
"LFN:/vo/user/i/initial/important.tar.gz", "/abspath/absfile.xml",
"file_in_pwd.xml"
])
return job
def job():
from DIRAC.Interfaces.API.Job import Job
job = Job(stdout='printer', stderr='/dev/null')
job.setInputSandbox([
'LFN:/vo/user/i/initial/important.tar.gz',
'/abspath/absfile.xml',
'file_in_pwd.xml',
])
return job
def parametricJob():
job = Job()
job.setName("parametric_helloWorld_%n")
exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
job.setInputSandbox(exeScriptLocation)
job.setParameterSequence("args", ['one', 'two', 'three'])
job.setParameterSequence("iargs", [1, 2, 3])
job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log')
return job
def base():
job = Job()
job.setName("helloWorld-TEST-TO-Jenkins")
job.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
job.setExecutable("exe-script.py", "", "helloWorld.log")
job.setCPUTime(1780)
job.setDestination('DIRAC.Jenkins.ch')
job.setLogLevel('DEBUG')
return job
def parametricJob():
job = Job()
job.setName("parametric_helloWorld_%n")
exeScriptLocation = find_all("exe-script.py", "..", "/DIRAC/tests/Integration")[0]
job.setInputSandbox(exeScriptLocation)
job.setParameterSequence("args", ["one", "two", "three"])
job.setParameterSequence("iargs", [1, 2, 3])
job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile="helloWorld_%n.log")
return job
def submit(name,
job_group,
task_id,
input_sandbox,
output_sandbox,
executable,
site=None,
banned_site=None,
sub_ids=[]):
dirac = Dirac()
submit_result = {'backend_job_ids': {}}
jobInfos = {}
for run in range(int((len(sub_ids) + 99) / 100)):
ids_this_run = [x for x in sub_ids[run * 100:(run + 1) * 100]]
job_names = ['%s.%s' % (name, sub_id) for sub_id in ids_this_run]
j = Job()
j.setName(name)
j.setExecutable(executable)
j.setParameterSequence('JobName', job_names, addToWorkflow=True)
j.setParameterSequence('arguments', ids_this_run, addToWorkflow=True)
if input_sandbox:
j.setInputSandbox(input_sandbox)
if output_sandbox:
j.setOutputSandbox(output_sandbox)
if job_group:
j.setJobGroup(job_group)
if site: # set destination to a certain site; list not allowed
j.setDestination(site)
if banned_site:
j.setBannedSites(banned_site)
result = dirac.submitJob(j)
if not result['OK']:
sys.stdout.write('DIRAC job submit error: %s\n' %
result['Message'])
sys.exit(1)
for sub_id, dirac_id in zip(ids_this_run, result['Value']):
submit_result['backend_job_ids'][sub_id] = dirac_id
jobInfos[dirac_id] = {'SubID': sub_id}
#Register on Task-manager Webapp of IHEPDIRAC
task = RPCClient('WorkloadManagement/TaskManager')
taskInfo = {'TaskName': name, 'JobGroup': job_group, 'JSUB-ID': task_id}
task_result = task.createTask(name, taskInfo, jobInfos)
task_web_id = task_result['Value']
submit_result['backend_task_id'] = task_web_id
return submit_result
def base():
job = Job()
job.setName("helloWorld-TEST-TO-Jenkins")
executablePath = find_all("exe-script.py", "..",
"/DIRAC/tests/Workflow/")[0]
job.setInputSandbox([executablePath])
job.setExecutable(executablePath, "", "helloWorld.log")
job.setCPUTime(1780)
job.setDestination("DIRAC.Jenkins.ch")
job.setLogLevel("DEBUG")
return job
def test_execute( self ):
""" this one tests that I can execute a job that requires multi-processing
"""
j = Job()
j.setName( "MP-test" )
j.setExecutable( self.mpExe )
j.setInputSandbox( find_all( 'mpTest.py', '.', 'Utilities' )[0] )
j.setTag( 'MultiProcessor' )
res = j.runLocal( self.d )
self.assertTrue( res['OK'] )
def read_hessjob(args=None):
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
if (len(args) != 1):
Script.showHelp()
version = args[0]
user_script = './read_hess2dst.sh'
sim_file = 'simtel_file.list'
infileLFNList = [
'/vo.cta.in2p3.fr/MC/PROD2/Config_310113/prod-2_21122012_corsika/gamma/prod-2_06052013_simtel_STD/Data/002xxx/gamma_20.0_180.0_alt2662.0_run002997.simtel.gz',
'/vo.cta.in2p3.fr/MC/PROD2/Config_310113/prod-2_21122012_corsika/gamma/prod-2_06052013_simtel_STD/Data/002xxx/gamma_20.0_180.0_alt2662.0_run002998.simtel.gz'
]
f = open(sim_file, 'w')
for infileLFN in infileLFNList:
filein = os.path.basename(infileLFN)
f.write(filein)
f.write('\n')
f.close()
j = Job()
j.setInputData(infileLFNList)
options = []
options = [sim_file]
executablestr = "%s %s %s" % (version, user_script, ' '.join(options))
j.setExecutable('./cta-read_hess.py', executablestr)
j.setInputSandbox(['cta-read_hess.py', user_script, sim_file])
j.setOutputSandbox(['read_hess.log'])
j.setOutputData(['*dst.gz'])
j.setName(user_script)
j.setCPUTime(100000)
Script.gLogger.info(j._toJDL())
Dirac().submit(j)
def softinstall(args=None):
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
if (len(args) < 2):
Script.showHelp()
version = args[0]
site = args[1]
if version not in ['18122013', 'v0r7p0']:
Script.gLogger.error('Version not valid')
Script.showHelp()
j = Job()
j.setInputSandbox(['cta-ctools-install.py', 'SoftwareInstallation.py'])
j.setExecutable('./cta-ctools-install.py', version)
j.setDestination([site])
j.setName('ctoolsInstall')
j.setCPUTime(100000)
Script.gLogger.info(j._toJDL())
if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']:
if site == 'LCG.GRIF.fr':
ceList = [
'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr',
'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr',
'node74.datagrid.cea.fr'
]
elif site == 'LCG.M3PEC.fr':
# ceList = ['ce0.bordeaux.inra.fr','ce0.m3pec.u-bordeaux1.fr']
ceList = ['ce0.bordeaux.inra.fr']
for ce in ceList:
j.setDestinationCE(ce)
name = 'ctoolsInstall' + '_' + ce
j.setName(name)
res = Dirac().submit(j)
print res
DIRAC.exit()
else:
name = 'ctoolsInstall'
j.setName(name)
res = Dirac().submit(j)
print res
def _submitJob(self, result_id, executable, test_name, site_name):
executable = executable.split('&')
j = Job()
j.setExecutable('python', arguments=executable[0] + " " + str(result_id))
sandBox = []
for file_name in executable:
sandBox.append(SAM_TEST_DIR + file_name)
j.setInputSandbox(sandBox)
j.setName(test_name)
j.setJobGroup('sam_test')
j.setDestination(site_name)
result = self.dirac.submit(j)
return result
def test_execute( self ):
""" this one tests that I can execute a job that requires multi-processing
"""
j = Job()
j.setName( "MP-test" )
j.setExecutable( self.mpExe )
j.setInputSandbox( find_all( 'mpTest.py', '.', 'Utilities' )[0] )
j.setTag( 'MultiProcessor' )
res = j.runLocal( self.d )
if multiprocessing.cpu_count() > 1:
self.assertTrue( res['OK'] )
else:
self.assertFalse( res['OK'] )
def test_execute(self):
""" this one tests that I can execute a job that requires multi-processing
"""
j = Job()
j.setName("MP-test")
j.setExecutable(self.mpExe)
j.setInputSandbox(find_all('mpTest.py', rootPath, 'DIRAC/tests/Utilities')[0])
j.setTag('MultiProcessor')
j.setLogLevel('DEBUG')
res = j.runLocal(self.d)
if multiprocessing.cpu_count() > 1:
self.assertTrue(res['OK'])
else:
self.assertFalse(res['OK'])
def submitJob(jobPara):
dirac = Dirac()
j = Job()
j.setName(jobPara['jobName'])
j.setJobGroup(jobPara['jobGroup'])
j.setExecutable(jobPara['jobScript'], logFile = jobPara['jobScriptLog'])
j.setInputSandbox(jobPara['inputSandbox'])
j.setOutputSandbox(jobPara['outputSandbox'])
j.setOutputData(jobPara['outputData'], jobPara['SE'])
j.setDestination(jobPara['sites'])
j.setCPUTime(jobPara['CPUTime'])
result = dirac.submit(j)
if result['OK']:
print 'Job %s submitted successfully. ID = %d' %(jobPara['jobName'],result['Value'])
else:
print 'Job %s submitted failed' %jobPara['jobName']
return result
def CorsikaSimtelInstall(args=None):
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
if (len(args) != 2):
Script.showHelp()
version = args[0]
site = args[1]
if version not in ['prod-2_13112014']:
Script.gLogger.error('Version not valid')
Script.showHelp()
j = Job()
CorsikaSimtelPack = os.path.join('corsika_simhessarray', version,
'corsika_simhessarray')
CorsikaSimtelLFN = 'LFN:' + os.path.join('/vo.cta.in2p3.fr/software',
CorsikaSimtelPack) + '.tar.gz'
j.setInputSandbox(['cta-corsikasimtel-install.py', CorsikaSimtelLFN])
j.setExecutable('./cta-corsikasimtel-install.py', version)
j.setDestination([site])
j.setJobGroup('SoftInstall')
j.setCPUTime(100000)
if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']:
if site == 'LCG.GRIF.fr':
ceList = [
'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr',
'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr',
'node74.datagrid.cea.fr'
]
if site == 'LCG.M3PEC.fr':
ceList = ['ce0.bordeaux.inra.fr', 'ce0.m3pec.u-bordeaux1.fr']
for ce in ceList:
j.setDestinationCE(ce)
name = 'corsikasimtelInstall' + '_' + ce
j.setName(name)
Dirac().submit(j)
DIRAC.exit()
j.setName('corsikasimtelInstall')
Script.gLogger.info(j._toJDL())
Dirac().submit(j)
def test_execute(self):
j = Job()
j.setName("helloWorld-test")
j.setExecutable(self.exeScriptLocation)
j.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
j.setConfigArgs('pilot.cfg')
res = j.runLocal(self.d)
self.assertTrue(res['OK'])
def test_execute(self):
j = Job()
j.setName("helloWorld-test")
j.setExecutable(self.exeScriptLocation)
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
self.assertTrue(res["OK"])
def test_execute(self):
""" this one tests that I can execute a job that requires multi-processing
"""
j = Job()
j.setName("MP-test")
j.setExecutable(self.mpExe)
j.setInputSandbox(
find_all('mpTest.py', rootPath, 'DIRAC/tests/Utilities')[0])
j.setNumberOfProcessors(4)
j.setLogLevel('DEBUG')
res = j.runLocal(self.d)
if multiprocessing.cpu_count() > 1:
self.assertTrue(res['OK'])
else:
self.assertFalse(res['OK'])
def submitJob(jobPara):
dirac = Dirac()
j = Job()
j.setName(jobPara['jobName'])
j.setJobGroup(jobPara['jobGroup'])
j.setExecutable(jobPara['jobScript'], logFile=jobPara['jobScriptLog'])
j.setInputSandbox(jobPara['inputSandbox'])
j.setOutputSandbox(jobPara['outputSandbox'])
j.setOutputData(jobPara['outputData'], jobPara['SE'])
j.setDestination(jobPara['sites'])
j.setCPUTime(jobPara['CPUTime'])
result = dirac.submit(j)
if result['OK']:
print 'Job %s submitted successfully. ID = %d' % (jobPara['jobName'],
result['Value'])
else:
print 'Job %s submitted failed' % jobPara['jobName']
return result
def test_execute(self):
"""just testing unix "ls" """
job = Job()
job.setName("ls-test")
job.setExecutable("/bin/ls", "-l")
job.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
job.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
job.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
job.setConfigArgs("pilot.cfg")
res = job.runLocal(self.d)
self.assertTrue(res["OK"])
def test_submit(self):
print "**********************************************************************************************************"
gLogger.info("\n Submitting hello world job")
helloJ = Job()
helloJ.setName("helloWorld-test-T2s")
helloJ.setInputSandbox(
[find_all('exe-script.py', '.', 'GridTestSubmission')[0]])
helloJ.setExecutable("exe-script.py", "", "helloWorld.log")
helloJ.setCPUTime(17800)
result = self.dirac.submit(helloJ)
gLogger.info("Hello world job: ", result)
jobID = int(result['Value'])
jobsSubmittedList.append(jobID)
self.assert_(result['OK'])
print "**********************************************************************************************************"
gLogger.info("\n Submitting a job that uploads an output")
helloJ = Job()
helloJ.setName("upload-Output-test")
helloJ.setInputSandbox(
[find_all('testFileUpload.txt', '.', 'GridTestSubmission')[0]])
helloJ.setExecutable("exe-script.py", "", "helloWorld.log")
helloJ.setCPUTime(17800)
helloJ.setOutputData(['testFileUpload.txt'])
result = self.dirac.submit(helloJ)
gLogger.info("Hello world with output: ", result)
jobID = int(result['Value'])
jobsSubmittedList.append(jobID)
self.assert_(result['OK'])
def test_execute(self):
""" just testing unix "ls"
"""
job = Job()
job.setName("ls-test")
job.setExecutable("/bin/ls", '-l')
job.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
job.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
job.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
job.setConfigArgs('pilot.cfg')
res = job.runLocal(self.d)
self.assertTrue(res['OK'])
def basicTest(self):
j = Job()
j.setCPUTime(50000)
j.setExecutable(
'/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py'
)
# j.setExecutable('/bin/echo hello')
j.setOwner('paterson')
j.setType('test')
j.setName('MyJobName')
#j.setAncestorDepth(1)
j.setInputSandbox([
'/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts',
'/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts'
])
j.setOutputSandbox(['firstfile.txt', 'anotherfile.root'])
j.setInputData([
'/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst',
'/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst'
])
j.setOutputData(['my.dst', 'myfile.log'])
j.setDestination('LCG.CERN.ch')
j.setPlatform('LCG')
j.setSystemConfig('x86_64-slc5-gcc43-opt')
j.setSoftwareTags(['VO-lhcb-Brunel-v30r17', 'VO-lhcb-Boole-v12r10'])
#print j._toJDL()
#print j.printObj()
xml = j._toXML()
testFile = 'jobDescription.xml'
if os.path.exists(testFile):
os.remove(testFile)
xmlfile = open(testFile, 'w')
xmlfile.write(xml)
xmlfile.close()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow'
print j.createCode()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow'
j.execute()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file'
workflow = fromXMLFile(testFile)
code = workflow.createCode()
print code
workflow.execute()
def simteljob(args = None ):
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
if (len(args)!=1):
Script.showHelp()
version = args[0]
user_script = './run_simtel.sh'
infileLFNList = ['/vo.cta.in2p3.fr/MC/PROD2/Config_120213/prod-2_21122012_corsika/proton/Data/044xxx/proton_20.0_180.0_alt2662.0_run044019.corsika.gz',
'/vo.cta.in2p3.fr/MC/PROD2/Config_120213/prod-2_21122012_corsika/proton/Data/044xxx/proton_20.0_180.0_alt2662.0_run044085.corsika.gz']
for infileLFN in infileLFNList:
filein = os.path.basename(infileLFN)
j = Job()
j.setInputSandbox( ['cta-simtel.py', user_script] )
j.setInputData(infileLFN)
user_args = []
user_args = [filein]
executablestr = "%s %s %s" % ( version, user_script, ' '.join( user_args ) )
j.setExecutable('./cta-simtel.py', executablestr)
sim_out = 'Data/sim_telarray/cta-ultra5/0.0deg/Data/*.simtel.gz'
log_out = 'Data/sim_telarray/cta-ultra5/0.0deg/Log/*.log.gz'
hist_out = 'Data/sim_telarray/cta-ultra5/0.0deg/Histograms/*.hdata.gz'
j.setOutputData([sim_out,log_out,hist_out])
j.setOutputSandbox('simtel.log')
j.setName(user_script)
j.setCPUTime(100000)
Script.gLogger.info( j._toJDL() )
Dirac().submit( j )
def launch_batch_pict( pitch_start, step, n_pict ):
j = Job()
j.setCPUTime(500)
j.setName('%s_%f' % (EXEC, pitch_start))
j.setJobGroup(JOBGROUP)
j.setInputSandbox([EXEC])
out_bmp_list=[]
pitch=pitch_start
for i in range(n_pict):
out_bmp='out_%f.bmp' % pitch
out_bmp_list.append(out_bmp)
j.setExecutable(EXEC,arguments="-W 600 -H 600 -X -0.77568377 -Y -0.13646737 -P %f -M 500 %s" % (pitch, out_bmp))
pitch+=step
j.setOutputSandbox(out_bmp_list + ["StdOut"] + ["StdErr"])
result = dirac.submit(j)
print 'Submission Result: ',result
return result
def submit(self, param):
j = Job()
j.setName(param['jobName'])
j.setExecutable(param['jobScript'],logFile = param['jobScriptLog'])
if self.site:
j.setDestination(self.site)
if self.jobGroup:
j.setJobGroup(self.jobGroup)
j.setInputSandbox(param['inputSandbox'])
j.setOutputSandbox(param['outputSandbox'])
j.setOutputData(param['outputData'], outputSE = self.outputSE, outputPath = self.outputPath)
dirac = GridDirac()
result = dirac.submit(j)
status = {}
status['submit'] = result['OK']
if status['submit']:
status['job_id'] = result['Value']
return status
def test_min2max4(self):
""" this tests executes a job that requires 2 to 4 processors
"""
j = Job()
j.setName("MP-test-min2max4")
j.setExecutable(self.mpExeFlex, arguments='2')
j.setInputSandbox(
find_all('mpTest-flexible.py', rootPath,
'DIRAC/tests/Utilities')[0])
j.setNumberOfProcessors(
minNumberOfProcessors=2,
maxNumberOfProcessors=4) # This requires 2 to 4 processors
j.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
j.setConfigArgs('pilot.cfg')
res = j.runLocal(self.d)
self.assertTrue(res['OK'])
def test_min2(self):
"""this tests executes a job that requires at least 2 processors"""
j = Job()
j.setName("MP-test-min2")
# FIXME: the number of processors should be discovered at runtime using JobParameters.getNumberOfJobProcessors()
# here, and later
j.setExecutable(self.mpExeFlex, arguments="2")
j.setInputSandbox(
find_all("mpTest-flexible.py", rootPath,
"DIRAC/tests/Utilities")[0])
j.setNumberOfProcessors(
minNumberOfProcessors=2) # This requires at least 2 processors
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
self.assertTrue(res["OK"])
def test_fixed(self):
"""this tests executes a job that requires exactly 4 processors"""
j = Job()
j.setName("MP-test")
j.setExecutable(self.mpExe)
j.setInputSandbox(
find_all("mpTest.py", rootPath, "DIRAC/tests/Utilities")[0])
j.setNumberOfProcessors(
4) # This requires a fixed number of processors
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
if multiprocessing.cpu_count() > 1:
self.assertTrue(res["OK"])
else:
self.assertFalse(res["OK"])
def test_execute(self):
job = Job()
job._siteSet = {"DIRAC.someSite.ch"}
job.setName("helloWorld-test")
job.setExecutable(
self.helloWorld,
arguments="This is an argument",
logFile="aLogFileForTest.txt",
parameters=[
("executable", "string", "", "Executable Script"),
("arguments", "string", "", "Arguments for executable Script"),
("applicationLog", "string", "", "Log file name"),
("someCustomOne", "string", "", "boh"),
],
paramValues=[("someCustomOne", "aCustomValue")],
)
job.setBannedSites(["LCG.SiteA.com", "DIRAC.SiteB.org"])
job.setOwner("ownerName")
job.setOwnerGroup("ownerGroup")
job.setName("jobName")
job.setJobGroup("jobGroup")
job.setType("jobType")
job.setDestination("DIRAC.someSite.ch")
job.setCPUTime(12345)
job.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
job.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
job.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
job.setConfigArgs("pilot.cfg")
res = job.runLocal(self.d)
self.assertTrue(res["OK"])
def submit_job(self,
request_id,
executable,
macro,
starting_seed=8000000,
njobs=10,
platform='ANY',
output_data_site='UKI-LT2-IC-HEP-disk',
output_log='lzproduction_output.log'):
"""
Submit LZProduction job to DIRAC.
Args:
request_id (int): The id number of the associated request
executable (str): The full path to the executable job script
macro (str): The full path to the macro for this job
starting_seed (int): The random seed for the first of the parametric jobs
njobs (int): The number of parametric jobs to create
platform (str): The required platform
output_data_site (str): The name of the grid site to store the output data at
output_log (str): The file name for the output log file
Returns:
list: The list of created parametric job DIRAC ids
"""
j = Job()
j.setName(os.path.splitext(os.path.basename(macro))[0] + '%(args)s')
j.setExecutable(os.path.basename(executable),
os.path.basename(macro) + ' %(args)s', output_log)
j.setInputSandbox([executable, macro])
j.setOutputData('*.root', output_data_site, str(request_id))
j.setParameterSequence(
"args",
[str(i) for i in xrange(starting_seed, starting_seed + njobs)],
addToWorkflow=True)
j.setPlatform(platform)
return self.status(self._dirac_api.submit(j).get("Value", []))
def submit(self, param):
j = Job()
j.setName(param['jobName'])
j.setExecutable(param['jobScript'], logFile=param['jobScriptLog'])
if self.site:
j.setDestination(self.site)
if self.jobGroup:
j.setJobGroup(self.jobGroup)
j.setInputSandbox(param['inputSandbox'])
j.setOutputSandbox(param['outputSandbox'])
j.setOutputData(param['outputData'],
outputSE=self.outputSE,
outputPath=self.outputPath)
dirac = GridDirac()
result = dirac.submit(j)
status = {}
status['submit'] = result['OK']
if status['submit']:
status['job_id'] = result['Value']
return status
def test_execute(self):
job = Job()
job._siteSet = {'DIRAC.someSite.ch'}
job.setName("helloWorld-test")
job.setExecutable(self.helloWorld,
arguments="This is an argument",
logFile="aLogFileForTest.txt",
parameters=[('executable', 'string', '',
"Executable Script"),
('arguments', 'string', '',
'Arguments for executable Script'),
('applicationLog', 'string', '',
"Log file name"),
('someCustomOne', 'string', '', "boh")],
paramValues=[('someCustomOne', 'aCustomValue')])
job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org'])
job.setOwner('ownerName')
job.setOwnerGroup('ownerGroup')
job.setName('jobName')
job.setJobGroup('jobGroup')
job.setType('jobType')
job.setDestination('DIRAC.someSite.ch')
job.setCPUTime(12345)
job.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
job.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
job.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
job.setConfigArgs('pilot.cfg')
res = job.runLocal(self.d)
self.assertTrue(res['OK'])
def basicTest(self):
j = Job()
j.setCPUTime(50000)
j.setExecutable('/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py')
# j.setExecutable('/bin/echo hello')
j.setOwner('paterson')
j.setType('test')
j.setName('MyJobName')
#j.setAncestorDepth(1)
j.setInputSandbox(['/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts','/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts'])
j.setOutputSandbox(['firstfile.txt','anotherfile.root'])
j.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst',
'/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst'])
j.setOutputData(['my.dst','myfile.log'])
j.setDestination('LCG.CERN.ch')
j.setPlatform('LCG')
j.setSystemConfig('x86_64-slc5-gcc43-opt')
j.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10'])
#print j._toJDL()
#print j.printObj()
xml = j._toXML()
testFile = 'jobDescription.xml'
if os.path.exists(testFile):
os.remove(testFile)
xmlfile = open(testFile,'w')
xmlfile.write(xml)
xmlfile.close()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow'
print j.createCode()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow'
j.execute()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file'
workflow = fromXMLFile(testFile)
code = workflow.createCode()
print code
workflow.execute()
def test_min1(self):
"""this tests executes a job that requires at least 1 processor"""
j = Job()
j.setName("MP-test-min1")
j.setExecutable(self.mpExeFlex, arguments="2")
j.setInputSandbox(
find_all("mpTest-flexible.py", rootPath,
"DIRAC/tests/Utilities")[0])
j.setNumberOfProcessors(
minNumberOfProcessors=1) # This requires 1 to infinite processors
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
self.assertTrue(res["OK"])
# Setup the job.
## The DIRAC job to submit.
j = Job(stdout='StdOut', stderr='StdErr')
# Set the name of the job (viewable in the web portal).
j.setName("CERNatschool_test_%03d" % (jobnum))
# As we're just copying the input sandbox to the storage element
# via OutputData, we'll just list the files as a check for the
# output written to StdOut.
j.setExecutable('/bin/ls -l')
# Here we add the list of LFNs we have obtained from the metadata
# query.
j.setInputSandbox(lfns)
#...and added to the desried storage element with the corresponding
# LFN via the job's OutputData. You may wish to change:
# * The Storage Element - by changing the outputSE parameter;
# * The LFN base name - by changing the outputPath parameter.
#j.setOutputData(file_dict.keys(), \
# outputSE='GLASGOW-disk', \
# outputPath='/diractest%03d/' % (jobnum)\
# )
# These are the files retrieved with the local job output.
j.setOutputSandbox(['StdOut', 'StdErr'])
# You can set your preferred site here.
#j.setDestination('LCG.Liverpool.uk')
if '/'.join([output_path.strip('/'), output_filename_wave]) in GRID_filelist:
print("\n{} already on GRID SE\n".format(job_name))
continue
if NJobs == 0:
print("maximum number of jobs to submit reached")
print("breaking loop now")
break
else:
NJobs -= 1
j = Job()
# runtime in seconds times 8 (CPU normalisation factor)
j.setCPUTime(6 * 3600 * 8)
j.setName(job_name)
j.setInputSandbox(input_sandbox)
if banned_sites:
j.setBannedSites(banned_sites)
# j.setDestination( 'LCG.IN2P3-CC.fr' )
# mr_filter loses its executable property by uploading it to the GRID SE; reset
j.setExecutable('chmod', '+x mr_filter')
j.setExecutable('ls -lah')
for run_file in run_filelist:
file_token = re.split('_', run_file)[3]
# wait for a random number of seconds (up to five minutes) before starting
# to add a bit more entropy in the starting times of the dirac querries.
for fn in retrieved_frames:
#print("* Found: '%s'." % (fn))
inputfiles.append(fn)
print("*")
## The DIRAC job to submit.
j = Job(stdout='StdOut', stderr='StdErr')
# Set the name of the job (viewable in the web portal).
j.setName(jobname)
#
j.setExecutable('/bin/sh', arguments='%s %s %s' % ('run.sh', '/cvmfs/cernatschool.gridpp.ac.uk/grid-klustering-001-00-07/', 'process-frames.py'))
#
j.setInputSandbox(inputfiles)
#...and added to the desired storage element with the corresponding
# LFN via the job's OutputData. You may wish to change:
# * The Storage Element - by changing the outputSE parameter;
# * The LFN base name - by changing the outputPath parameter.
j.setOutputData(kluster_file_names, \
outputSE='%s' % (se), \
outputPath='/%s/' % (gridoutdir)\
)
# These are the files retrieved with the local job output.
j.setOutputSandbox(['StdOut', 'StdErr', 'klusters.json', 'log_process_frames.log'])
# You can set your preferred site here.
j.setDestination(sitename)
executable = args[0].replace("bash ","").replace("./","")
if not os.path.isfile(executable):
gLogger.error("file %s not found."%executable)
dexit(1)
os.chmod(executable,0755) # make file executable
input_sandbox_files.append(executable)
j.setExecutable(str(executable))
else:
gLogger.error("No executable defined.")
dexit(1)
j.setName("MC job")
if not opts.name is None:
j.setName(opts.name)
j.setInputSandbox(input_sandbox_files) # all input files in the sandbox
j.setOutputSandbox(output_sandbox_files)
j.setCPUTime(opts.cpu)
if not opts.site is None:
j.setDestination(opts.site.split(","))#can also be a list
if not opts.bannedSites is None:
j.setBannedSites(opts.bannedSites.split(","))
if not opts.release is None:
tag = opts.release
cl = SoftwareTagClient()
result = cl.getSitesForTag(tag,'Valid') # keyword doesn't work there.
if not result['OK']:
gLogger.error("*ERROR* Could not get sites for Tag %s"%tag,result['Message'])
from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac #from tests.Workflow.Integration.Test_UserJobs import createJob gLogger.setLevel( 'DEBUG' ) cwd = os.path.realpath( '.' ) dirac = Dirac() # Simple Hello Word job to DIRAC.Jenkins.ch gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch" ) helloJ = Job() helloJ.setName( "helloWorld-TEST-TO-Jenkins" ) helloJ.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJ.setExecutable( "exe-script.py", "", "helloWorld.log" ) helloJ.setCPUTime( 17800 ) helloJ.setDestination( 'DIRAC.Jenkins.ch' ) result = dirac.submit( helloJ ) gLogger.info( "Hello world job: ", result ) if not result['OK']: gLogger.error( "Problem submitting job", result['Message'] ) exit( 1 ) # Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN" ) helloJMP = Job() helloJMP.setName( "helloWorld-TEST-TO-Jenkins-MP" ) helloJMP.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJMP.setExecutable( "exe-script.py", "", "helloWorld.log" )
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
dirac = Dirac()
j = Job()
j.setName("compile_mrfilter")
j.setCPUTime(80)
j.setInputSandbox(["dirac_compile_mrfilter_pilot.sh"])
j.setExecutable("dirac_compile_mrfilter_pilot.sh", "")
j.setOutputData(["mr_filter"], outputSE=None,
outputPath="cta/bin/mr_filter/v3_1/")
Dirac().submit(j)
f.close()
if runnr == -1 :
print "Something is wrong with determination of run number from input file"
break
print "runnr = '{0}'".format(runnr)
######################################
## BEGIN JOB DESCRIPTION
######################################
dirac = Dirac(withRepo=True, repoLocation='jobid.list', useCertificates=False)
j = Job(script=None, stdout='submission.out', stderr='submission.err')
j.setInputSandbox(input_file)
input_file_base = os.path.basename(input_file)
## prepare the list of output files
run_log = input_file_base + ".log"
dat = input_file_base.replace('aug', 'DAT')
datlong = dat + ".long"
output_files= [run_log, 'fluka11.out', 'fluka15.err', dat, datlong ]
## prepare the output location in GRID storage; the input path will be the used also for GRID storage
outdir = grid_basedir_output + path + "/" + str(runnr)
## ALWAYS, INFO, VERBOSE, WARN, DEBUG
j.setLogLevel('debug')
j.setName('AUGER test simulation')
# dirac job created by ganga from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac j = Job() dirac = Dirac() # default commands added by ganga j.setName( "helloWorld-test" ) j.setInputSandbox( ['/afs/cern.ch/user/f/fstagni/userJobs/_inputHello.tar.bz2', '/afs/cern.ch/user/f/fstagni/userJobs/hello-script.py'] ) j.setExecutable( "exe-script.py", "", "Ganga_Executable.log" ) # <-- user settings j.setCPUTime( 172800 ) tier1s = DMSHelpers().getTiers( tier = ( 0, 1 ) ) j.setBannedSites( tier1s ) # user settings --> # print j.workflow # submit the job to dirac result = dirac.submit( j ) print result
from DIRAC.Interfaces.API.Job import Job
from DIRAC.Interfaces.API.Dirac import Dirac
#from tests.Workflow.Integration.Test_UserJobs import createJob
gLogger.setLevel('DEBUG')
cwd = os.path.realpath('.')
dirac = Dirac()
# Simple Hello Word job to DIRAC.Jenkins.ch
gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch")
helloJ = Job()
helloJ.setName("helloWorld-TEST-TO-Jenkins")
helloJ.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
helloJ.setExecutable("exe-script.py", "", "helloWorld.log")
helloJ.setCPUTime(1780)
helloJ.setDestination('DIRAC.Jenkins.ch')
helloJ.setLogLevel('DEBUG')
result = dirac.submitJob(helloJ)
gLogger.info("Hello world job: ", result)
if not result['OK']:
gLogger.error("Problem submitting job", result['Message'])
exit(1)
# Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN
gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN")
helloJMP = Job()
helloJMP.setName("helloWorld-TEST-TO-Jenkins-MP")
helloJMP.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
print "runnr = '{0}'".format(runnr)
print "prmpar = '{0}'".format(prmpar)
print "Energy Min = '{0}'".format(e_min)
print "Energy Max = '{0}'".format(e_max)
print "Theta Min = '{0}'".format(theta_min)
print "Theta Max = '{0}'".format(theta_max)
######################################
## BEGIN JOB DESCRIPTION
######################################
dirac = Dirac(withRepo=True, repoLocation='jobid.list', useCertificates=False)
j = Job(script=None, stdout='submission.out', stderr='submission.err')
input_files_list = []
input_files_list.append(input_file)
j.setInputSandbox(input_files_list)
input_file_base = os.path.basename(input_file)
# set the list of output files
output_files = [ 'data.tar.gz', 'logs.tar.gz' ]
## prepare the output location in GRID storage; the input path will be the used also for GRID storage
# outdir = grid_basedir_output + PROD_NAME + "/" + str(e_min) + "_" + str(e_max) + "/" + str(theta_min) + "_" + str(theta_max) + "/" + str(prmpar) + "/" + str(runnr)
# outdir = "/" + PROD_NAME + "/" + str(e_min) + "_" + str(e_max) + "/" + str(theta_min) + "_" + str(theta_max) + "/" + str(prmpar) + "/" + str(runnr)
outdir = "/" + PROD_NAME + "/" + str(e_min) + "/" + str(theta_min) + "/" + str(prmpar) + "/" + str(runnr)
print 'SE = ',se
lfns_list = []
if ( group == "auger_prod" ):
base_output_path = prod_path
outputfiles.append("StdErr")
## The DIRAC job to submit.
j = Job(stdout='StdOut', stderr='StdErr')
# Set the name of the job (viewable in the web portal).
j.setName(jobname)
# As we're just copying the input sandbox to the storage element
# via OutputData, we'll just list the files as a check for the
# output written to StdOut.
j.setExecutable('/bin/ls -l')
# Here we add the names of the temporary copies of the frame data
# files in the dataset to the input sandbox. These will be uploaded
# to the grid with the job...
j.setInputSandbox(retrieved_clusters)
# These are the files retrieved with the local job output.
j.setOutputSandbox(outputfiles)
# You can set your preferred site here.
j.setDestination(sitename)
## The DIRAC instance.
dirac = Dirac()
# # Submit the job and print the result.
# result = dirac.submit(j)
# print 'Submission result: ', result
## The DIRAC job to submit.
j = Job(stdout='StdOut', stderr='StdErr')
# Set the name of the job (viewable in the web portal).
j.setName(jobname)
# As we're just copying the input sandbox to the storage element
# via OutputData, we'll just list the files as a check for the
# output written to StdOut.
j.setExecutable('/bin/ls -l')
# Here we add the names of the temporary copies of the frame data
# files in the dataset to the input sandbox. These will be uploaded
# to the grid with the job...
j.setInputSandbox(file_dict.keys())
#...and added to the desired storage element with the corresponding
# LFN via the job's OutputData. You may wish to change:
# * The Storage Element - by changing the outputSE parameter;
# * The LFN base name - by changing the outputPath parameter.
j.setOutputData(file_dict.keys(), \
outputSE='%s' % (se), \
outputPath='/%s/' % (gridoutdir)\
)
# These are the files retrieved with the local job output.
j.setOutputSandbox(['StdOut', 'StdErr'])
# You can set your preferred site here.
j.setDestination(sitename)
# dirac job created by ganga
from DIRAC.Interfaces.API.Job import Job
from DIRAC.Interfaces.API.Dirac import Dirac
j = Job()
dirac = Dirac()
# default commands added by ganga
j.setName("helloWorld-test")
j.setInputSandbox( ['/afs/cern.ch/user/f/fstagni/userJobs/_inputHello.tar.bz2', '/afs/cern.ch/user/f/fstagni/userJobs/hello-script.py'] )
j.setExecutable("exe-script.py","","Ganga_Executable.log")
# <-- user settings
j.setCPUTime(172800)
j.setBannedSites(['LCG.CERN.ch', 'LCG.CNAF.it', 'LCG.GRIDKA.de',
'LCG.IN2P3.fr', 'LCG.NIKHEF.nl', 'LCG.PIC.es', 'LCG.RAL.uk',
'LCG.SARA.nl'])
# user settings -->
#print j.workflow
# submit the job to dirac
result = dirac.submit(j)
print result
def helloWorldJob(): job = Job() job.setName( "helloWorld" ) job.setInputSandbox( '../../Integration/exe-script.py' ) job.setExecutable( "exe-script.py", "", "helloWorld.log" ) return job
