def __runJobLocally(jobID, basepath):
"""
Runs the job!
"""
from LHCbDIRAC.Interfaces.API.LHCbJob import LHCbJob
localJob = LHCbJob(basepath + "InputSandbox" + str(jobID) + os.path.sep +
"jobDescription.xml")
localJob.setInputSandbox(os.getcwd() + "pilot.cfg")
localJob.setConfigArgs(os.getcwd() + "pilot.cfg")
os.chdir(basepath)
localJob.runLocal()
class UserJobTestCase(IntegrationTest):
""" Base class for the UserJob test cases
"""
def setUp(self):
super(UserJobTestCase, self).setUp()
print "\n \n ********************************* \n Running a new test \n *********************************"
self.dLHCb = DiracLHCb()
try:
self.exeScriptLocation = find_all('exe-script.py',
os.environ['WORKSPACE'],
'/tests/Workflow/Integration')[0]
except (IndexError, KeyError):
self.exeScriptLocation = find_all('exe-script.py', rootPath,
'/tests/Workflow/Integration')[0]
try:
self.exeScriptFromDIRACLocation = find_all(
'exe-script-fromDIRAC.py', os.environ['WORKSPACE'],
'/tests/Workflow/Integration')[0]
except (IndexError, KeyError):
self.exeScriptFromDIRACLocation = find_all(
'exe-script-fromDIRAC.py', rootPath,
'/tests/Workflow/Integration')[0]
self.lhcbJobTemplate = LHCbJob()
self.lhcbJobTemplate.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
self.lhcbJobTemplate.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
self.lhcbJobTemplate.setInputSandbox(
find_all('pilot.cfg', rootPath)[0])
self.lhcbJobTemplate.setConfigArgs('pilot.cfg')
def tearDown(self):
del self.lhcbJobTemplate
def createJob(local=True):
gaudirunJob = LHCbJob()
gaudirunJob.setName("gaudirun-Gauss-test")
if local:
try:
gaudirunJob.setInputSandbox([
find_all('prodConf_Gauss_00012345_00067890_1.py',
os.environ['WORKSPACE'],
'/tests/System/GridTestSubmission')[0],
find_all('wrongConfig.cfg', os.environ['WORKSPACE'],
'/tests/System/GridTestSubmission')[0],
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0]
])
except (IndexError, KeyError):
gaudirunJob.setInputSandbox([
find_all('prodConf_Gauss_00012345_00067890_1.py', rootPath,
'/tests/System/GridTestSubmission')[0],
find_all('wrongConfig.cfg', rootPath,
'/tests/System/GridTestSubmission')[0],
find_all('pilot.cfg', rootPath)[0]
])
else:
try:
gaudirunJob.setInputSandbox([
find_all('prodConf_Gauss_00012345_00067890_1.py',
os.environ['WORKSPACE'],
'/tests/System/GridTestSubmission')[0],
find_all('wrongConfig.cfg', os.environ['WORKSPACE'],
'/tests/System/GridTestSubmission')[0]
])
except (IndexError, KeyError):
gaudirunJob.setInputSandbox([
find_all('prodConf_Gauss_00012345_00067890_1.py', rootPath,
'/tests/System/GridTestSubmission')[0],
find_all('wrongConfig.cfg', rootPath,
'/tests/System/GridTestSubmission')[0]
])
gaudirunJob.setOutputSandbox('00012345_00067890_1.sim')
optGauss = "$APPCONFIGOPTS/Gauss/Sim08-Beam3500GeV-md100-2011-nu2.py;"
optDec = "$DECFILESROOT/options/34112104.py;"
optPythia = "$LBPYTHIAROOT/options/Pythia.py;"
optOpts = "$APPCONFIGOPTS/Gauss/G4PL_FTFP_BERT_EmNoCuts.py;"
optCompr = "$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py;"
optPConf = "prodConf_Gauss_00012345_00067890_1.py"
options = optGauss + optDec + optPythia + optOpts + optCompr + optPConf
gaudirunJob.setApplication(
'Gauss',
'v45r5',
options,
extraPackages='AppConfig.v3r171;Gen/DecFiles.v27r14p1;ProdConf.v1r9',
systemConfig='x86_64-slc5-gcc43-opt',
modulesNameList=[
'CreateDataFile', 'GaudiApplication', 'FileUsage',
'UploadOutputData', 'UploadLogFile', 'FailoverRequest',
'UserJobFinalization'
],
parametersList=[('applicationName', 'string', '', 'Application Name'),
('applicationVersion', 'string', '',
'Application Version'),
('applicationLog', 'string', '',
'Name of the output file of the application'),
('optionsFile', 'string', '', 'Options File'),
('extraOptionsLine', 'string', '',
'This is appended to standard options'),
('inputDataType', 'string', '', 'Input Data Type'),
('inputData', 'string', '', 'Input Data'),
('numberOfEvents', 'string', '', 'Events treated'),
('extraPackages', 'string', '', 'ExtraPackages'),
('listoutput', 'list', [], 'StepOutputList'),
('SystemConfig', 'string', '', 'binary tag')])
gaudirunJob._addParameter(gaudirunJob.workflow, 'PRODUCTION_ID', 'string',
'00012345', 'ProductionID')
gaudirunJob._addParameter(gaudirunJob.workflow, 'JOB_ID', 'string',
'00067890', 'JobID')
gaudirunJob._addParameter(gaudirunJob.workflow, 'configName', 'string',
'testCfg', 'ConfigName')
gaudirunJob._addParameter(gaudirunJob.workflow, 'configVersion', 'string',
'testVer', 'ConfigVersion')
outputList = [{
'stepName': 'GaussStep1',
'outputDataType': 'sim',
'outputBKType': 'SIM',
'outputDataSE': 'Tier1_MC-DST',
'outputDataName': '00012345_00067890_1.sim'
}]
gaudirunJob._addParameter(gaudirunJob.workflow, 'outputList', 'list',
outputList, 'outputList')
gaudirunJob._addParameter(gaudirunJob.workflow, 'outputDataFileMask',
'string', '', 'outputFM')
gaudirunJob._addParameter(gaudirunJob.workflow, 'outputMode', 'string',
'Local', 'OM')
gaudirunJob._addParameter(gaudirunJob.workflow, 'LogLevel', 'string',
'DEBUG', 'LL')
outputFilesDict = [{
'outputDataName': '00012345_00067890_1.sim',
'outputDataSE': 'Tier1_MC-DST',
'outputDataType': 'SIM'
}]
gaudirunJob._addParameter(gaudirunJob.workflow.step_instances[0],
'listoutput', 'list', outputFilesDict,
'listoutput')
gaudirunJob.setLogLevel('DEBUG')
gaudirunJob.setDIRACPlatform()
if local:
gaudirunJob.setConfigArgs('pilot.cfg wrongConfig.cfg')
else:
gaudirunJob.setConfigArgs('wrongConfig.cfg')
gaudirunJob.setCPUTime(172800)
return gaudirunJob
# dirac job created by ganga
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers
from LHCbDIRAC.Interfaces.API.LHCbJob import LHCbJob
from LHCbDIRAC.Interfaces.API.DiracLHCb import DiracLHCb
j = LHCbJob()
dirac = DiracLHCb()
# default commands added by ganga
j.setName("helloWorld-test")
j.setInputSandbox([
'/afs/cern.ch/user/f/fstagni/userJobs/_inputHello.tgz',
'/afs/cern.ch/user/f/fstagni/userJobs/hello-script.py'
])
j.setExecutable("exe-script.py", "", "Ganga_Executable.log")
# <-- user settings
j.setCPUTime(172800)
try:
tier1s = DMSHelpers().getTiers(tier=(0, 1))
except AttributeError:
tier1s = [
'LCG.CERN.cern', 'LCG.CNAF.it', 'LCG.GRIDKA.de', 'LCG.IN2P3.fr',
'LCG.NIKHEF.nl', 'LCG.PIC.es', 'LCG.RAL.uk', 'LCG.RRCKI.ru',
'LCG.SARA.nl'
]
j.setBannedSites(tier1s)
# user settings -->
# print j.workflow
#for i in range(1,2):
inputList = [workdir+'gammaDh']
# "CLEO_K3PiScan_2014_pdf.root",
# "CLEO_K3PiScan_2009_GaussianRise_pdf2.root",
# "CLEO_KPiPi0Scan_2014_pdf.root",
# "CLEO_KSKpiScan_2012_Kst_pdf.root"]
# arg = "tot 1500000000 1";
arg = "tot 500000000 1";
j1 = LHCbJob()
j1.setName('GammaDh.'+mode+'.'+str(i))
j1.setExecutable('run_gamma_Dh.sh',arg,'gamma.log')
j1.setInputSandbox(inputList)
j1.setOutputSandbox(['*.root','__postprocesslocations__','std.out','std.err','*.log'])
j1.setOutputData(['*.root'])
j1.setCPUTime(172800)
j1.setBannedSites(['LCG.CERN.ch', 'LCG.CNAF.it', 'LCG.GRIDKA.de', 'LCG.IN2P3.fr', 'LCG.NIKHEF.nl', 'LCG.PIC.es', 'LCG.RAL.uk', 'LCG.SARA.nl'])
jobList.append({ 'Job' : j1, 'Name' : 'GammaDh.'+mode+'.'+str(i) })
#SPLITTING JOBLIST BY THREAD
nThread = int(10)
if len(jobList) < nThread: nThread = len(jobList)
njobsPerList = len(jobList)/nThread #Number of jobs to submit for each thread
jobListPerThread = [jobList[x:x+njobsPerList] for x in xrange(0, len(jobList), njobsPerList)]
from LHCbDIRAC.Interfaces.API.LHCbJob import LHCbJob
from LHCbDIRAC.Interfaces.API.DiracLHCb import DiracLHCb
#from tests.Workflow.Integration.Test_UserJobs import createJob
gLogger.setLevel('DEBUG')
cwd = os.path.realpath('.')
dirac = DiracLHCb()
# Simple Hello Word job to DIRAC.Jenkins.ch
gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch")
helloJ = LHCbJob()
helloJ.setName("helloWorld-TEST-TO-Jenkins")
try:
helloJ.setInputSandbox([find_all('exe-script.py', rootPath, '/DIRAC/tests/Workflow/')[0]])
except IndexError:
helloJ.setInputSandbox([find_all('exe-script.py', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow/')[0]])
helloJ.setExecutable("exe-script.py", "", "helloWorld.log")
helloJ.setCPUTime(17800)
helloJ.setDestination('DIRAC.Jenkins.ch')
result = dirac.submitJob(helloJ)
gLogger.info("Hello world job: ", result)
if not result['OK']:
gLogger.error("Problem submitting job", result['Message'])
exit(1)
# Simple Hello Word job to DIRAC.Jenkins.ch, with an input file
gLogger.info("\n Submitting hello world job, with input, targeting DIRAC.Jenkins.ch")
inputJ = LHCbJob()
def test_LHCbsubmit(self):
print "**********************************************************************************************************"
gLogger.info(
"\n Submitting hello world job targeting DIRAC.JobDebugger.cern")
helloJ = LHCbJob()
helloJ.setName("helloWorld-test-CERN")
helloJ.setInputSandbox([
find_all('exe-script.py', '..',
'/LHCbDIRAC/tests/System/GridTestSubmission')[0]
])
helloJ.setExecutable("exe-script.py", "", "helloWorld.log")
helloJ.setCPUTime(17800)
helloJ.setDestination('DIRAC.JobDebugger.cern')
result = self.dirac.submitJob(helloJ)
gLogger.info("Hello world job: ", result)
jobID = int(result['Value'])
jobsSubmittedList.append(jobID)
self.assertTrue(result['OK'])
print "**********************************************************************************************************"
gLogger.info("\n Submitting hello world job targeting slc6 machines")
helloJ = LHCbJob()
helloJ.setName("helloWorld-test-slc6")
helloJ.setInputSandbox([
find_all('exe-script.py', '.',
'/LHCbDIRAC/tests/System/GridTestSubmission')[0]
])
helloJ.setExecutable("exe-script.py", "", "helloWorld.log")
helloJ.setCPUTime(17800)
helloJ.setPlatform('x86_64-slc6')
result = self.dirac.submitJob(helloJ)
gLogger.info("Hello world job: ", result)
jobID = int(result['Value'])
jobsSubmittedList.append(jobID)
self.assertTrue(result['OK'])
print "**********************************************************************************************************"
gLogger.info("\n Submitting a job that uploads an output")
helloJ = LHCbJob()
helloJ.setName("upload-Output-test")
helloJ.setInputSandbox( [find_all( 'testFileUpload.txt', '.',
'/LHCbDIRAC/tests/System/GridTestSubmission' )[0]]\
+ [find_all( 'exe-script.py', '.',
'/LHCbDIRAC/tests/System/GridTestSubmission' )[0]] )
helloJ.setExecutable("exe-script.py", "", "helloWorld.log")
helloJ.setCPUTime(17800)
helloJ.setDestination('DIRAC.JobDebugger.cern')
helloJ.setOutputData(['testFileUpload.txt'])
result = self.dirac.submitJob(helloJ)
gLogger.info("Hello world with output: ", result)
jobID = int(result['Value'])
jobsSubmittedList.append(jobID)
self.assertTrue(result['OK'])
from tests.Workflow.Integration.Test_UserJobs import createJob
gLogger.setLevel('DEBUG')
cwd = os.path.realpath('.')
########################################################################################
gLogger.info("\n Submitting hello world job banning T1s")
helloJ = LHCbJob()
dirac = DiracLHCb()
helloJ.setName("helloWorld-test-T2s")
helloJ.setInputSandbox([
find_all('exe-script.py', rootPath, '/tests/System/GridTestSubmission')[0]
])
helloJ.setExecutable("exe-script.py", "", "helloWorld.log")
helloJ.setCPUTime(17800)
try:
tier1s = DMSHelpers().getTiers(tier=(0, 1))
except AttributeError:
tier1s = [
'LCG.CERN.cern', 'LCG.CNAF.it', 'LCG.GRIDKA.de', 'LCG.IN2P3.fr',
'LCG.NIKHEF.nl', 'LCG.PIC.es', 'LCG.RAL.uk', 'LCG.RRCKI.ru',
'LCG.SARA.nl'
]
cernSite = [s for s in tier1s if '.CERN.' in s][0]
helloJ.setBannedSites(tier1s)