def test_execute( self ):
""" this one tests that I can execute a job that requires multi-processing
"""
j = Job()
j.setName( "MP-test" )
j.setExecutable( self.mpExe )
j.setInputSandbox( find_all( 'mpTest.py', '.', 'Utilities' )[0] )
j.setTag( 'MultiProcessor' )
res = j.runLocal( self.d )
self.assertTrue( res['OK'] )
def test_execute( self ):
""" this one tests that I can execute a job that requires multi-processing
"""
j = Job()
j.setName( "MP-test" )
j.setExecutable( self.mpExe )
j.setInputSandbox( find_all( 'mpTest.py', '.', 'Utilities' )[0] )
j.setTag( 'MultiProcessor' )
res = j.runLocal( self.d )
if multiprocessing.cpu_count() > 1:
self.assertTrue( res['OK'] )
else:
self.assertFalse( res['OK'] )
def test_execute(self):
""" this one tests that I can execute a job that requires multi-processing
"""
j = Job()
j.setName("MP-test")
j.setExecutable(self.mpExe)
j.setInputSandbox(find_all('mpTest.py', rootPath, 'DIRAC/tests/Utilities')[0])
j.setTag('MultiProcessor')
j.setLogLevel('DEBUG')
res = j.runLocal(self.d)
if multiprocessing.cpu_count() > 1:
self.assertTrue(res['OK'])
else:
self.assertFalse(res['OK'])
args = visit + ' ' + insidename + ' ' + str(startsensor) + ' ' + str(
numsensors) + ' ' + str(idx)
outputname = 'fits_' + visit + '_' + str(idx) + '.tar'
j.setCPUTime(1209600)
j.setExecutable('runimsim2.1.sh', arguments=args)
j.stderr = "std.err"
j.stdout = "std.out"
#!!! May need the 2.1i directory here depending on visit number !!!
j.setInputSandbox([
"runimsim2.1.sh", "run_imsim_nersc.py",
"LFN:/lsst/user/j/james.perry/instcats/2.1i/" + instcatname
])
j.setOutputSandbox(["std.out", "std.err"])
j.setTag(["8Processors"])
#j.setOutputData([visit + "/" + outputname], outputPath="", outputSE=["IN2P3-CC-disk"])
j.setOutputData([visit + "/" + outputname],
outputPath="",
outputSE=["UKI-NORTHGRID-LANCS-HEP-disk"])
j.setPlatform("AnyPlatform")
j.setDestination(site)
jobID = dirac.submitJob(j)
print("Submitted job to " + site + " as ID " + str(jobID))
print "Status is:", dirac.status(jobID['JobID'])
joblistfile.write(str(jobID['JobID']) + '\n')
joblistfile.close()
else:
inputdata_list.append(lfn + 'second/HPC_data.tar.gz.' + str(i))
#SitesList = ['LCG.UKI-NORTHGRID-MAN-HEP.uk', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.UKI-LT2-QMUL.uk', 'LCG.UKI-NORTHGRID-LANCS-HEP.uk']
#SEList = ['UKI-NORTHGRID-MAN-HEP-disk', 'UKI-LT2-IC-HEP-disk', 'UKI-LT2-QMUL2-disk', 'UKI-NORTHGRID-LANCS-HEP-disk']
SitesList = ['LCG.UKI-NORTHGRID-MAN-HEP.uk']
SEList = ['UKI-NORTHGRID-MAN-HEP-disk']
print("Input data list:\n")
print(inputdata_list)
for i in range(0, total_pixels, chunks):
id_start = i
id_end = i + chunks
dirac = Dirac()
j.setName('CS Faraday Rotation Measurement Reconstruction - Pixels from ' +
str(id_start) + ' to ' + str(id_end - 1))
j.setPlatform('EL7')
j.setTag([str(nprocs) + 'Processors'])
j.setDestination(SitesList)
j.setExecutable('RMSynthesis2.sh',
arguments=str(nprocs) + ' ' + str(id_start) + ' ' +
str(id_end) + ' ' + str(expmnt))
# Input data
j.setInputData(inputdata_list)
j.setInputSandbox([
'RMSynthesis2.sh', 'run2.sh',
'prmon_1.0.1_x86_64-static-gnu72-opt.tar.gz'
])
# Output data
j.setOutputSandbox([
'StdOut', 'StdErr',
'outputtxt_' + str(id_start) + '_' + str(id_end - 1) + '.txt',
'prmon' + str(id_start) + '_' + str(id_end - 1) + '.txt'
gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch")
helloJ = Job()
helloJ.setName("helloWorld-TEST-TO-Jenkins")
helloJ.setInputSandbox(
[find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
helloJ.setExecutable("exe-script.py", "", "helloWorld.log")
helloJ.setCPUTime(17800)
helloJ.setDestination('DIRAC.Jenkins.ch')
result = dirac.submit(helloJ)
gLogger.info("Hello world job: ", result)
if not result['OK']:
gLogger.error("Problem submitting job", result['Message'])
exit(1)
# Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN
gLogger.info(
"\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN")
helloJMP = Job()
helloJMP.setName("helloWorld-TEST-TO-Jenkins-MP")
helloJMP.setInputSandbox(
[find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
helloJMP.setExecutable("exe-script.py", "", "helloWorld.log")
helloJMP.setCPUTime(17800)
helloJMP.setDestination('DIRAC.Jenkins.ch')
helloJMP.setTag('MultiProcessor')
result = dirac.submit(helloJMP) # this should make the difference!
gLogger.info("Hello world job MP: ", result)
if not result['OK']:
gLogger.error("Problem submitting job", result['Message'])
exit(1)
startsensor = idx * 4
numsensors = 4
if idx == 47:
numsensors = 1
args = visit + ' ' + insidename + ' ' + str(startsensor) + ' ' + str(numsensors) + ' ' + str(idx)
outputname = 'fits_' + visit + '_' + str(idx) + '.tar'
j.setCPUTime(1209600)
j.setExecutable('runimsim2.1.sh', arguments=args)
j.stderr="std.err"
j.stdout="std.out"
#!!! May need the 2.1i directory here depending on visit number !!!
j.setInputSandbox(["runimsim2.1.sh","run_imsim_nersc.py","LFN:/lsst/user/j/james.perry/instcats/2.1.1i/" + instcatname])
j.setOutputSandbox(["std.out","std.err"])
j.setTag(["4Processors"])
j.setOutputData([visit + "/" + outputname], outputPath="", outputSE=["UKI-NORTHGRID-LANCS-HEP-disk"])
j.setPlatform("AnyPlatform")
j.setBannedSites(["VAC.UKI-NORTHGRID-MAN-HEP.uk", "LCG.IN2P3-CC.fr"])
#print("Would submit job for sensors", sensorstring)
jobID = dirac.submitJob(j)
print("Submitted job as ID " + str(jobID))
print "Status is:", dirac.status(jobID['JobID'])
joblistfile.write(str(jobID['JobID']) + '\n')
joblistfile.close()
dirac = Dirac()
# Simple Hello Word job to DIRAC.Jenkins.ch
gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch" )
helloJ = Job()
helloJ.setName( "helloWorld-TEST-TO-Jenkins" )
helloJ.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] )
helloJ.setExecutable( "exe-script.py", "", "helloWorld.log" )
helloJ.setCPUTime( 17800 )
helloJ.setDestination( 'DIRAC.Jenkins.ch' )
result = dirac.submit( helloJ )
gLogger.info( "Hello world job: ", result )
if not result['OK']:
gLogger.error( "Problem submitting job", result['Message'] )
exit( 1 )
# Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN
gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN" )
helloJMP = Job()
helloJMP.setName( "helloWorld-TEST-TO-Jenkins-MP" )
helloJMP.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] )
helloJMP.setExecutable( "exe-script.py", "", "helloWorld.log" )
helloJMP.setCPUTime( 17800 )
helloJMP.setDestination( 'DIRAC.Jenkins.ch' )
helloJMP.setTag('MultiProcessor')
result = dirac.submit( helloJMP ) # this should make the difference!
gLogger.info( "Hello world job MP: ", result )
if not result['OK']:
gLogger.error( "Problem submitting job", result['Message'] )
exit( 1 )
