def jobWithOutputs():
"""Creates a job that uploads an output to 2 SEs.
The output SE is set here to ['RAL-SE', 'IN2P3-SE']
"""
timenow = time.strftime("%s")
inp1 = [os.path.join(os.getcwd(), timenow + "testFileUpload.txt")]
with open(inp1[0], "wt") as f:
f.write(timenow)
J = baseToAllJobs("jobWithOutputs")
try:
inp2 = [find_all("exe-script.py", rootPath, "DIRAC/tests/Workflow")[0]]
J.setInputSandbox(inp1 + inp2)
except IndexError: # we are in Jenkins
inp2 = [
find_all("exe-script.py", os.environ["WORKSPACE"],
"DIRAC/tests/Workflow")[0]
]
J.setInputSandbox(inp1 + inp2)
J.setExecutable("exe-script.py", "", "helloWorld.log")
J.setOutputData([timenow + "testFileUpload.txt"],
outputSE=["RAL-SE", "IN2P3-SE"])
res = endOfAllJobs(J)
try:
os.remove(os.path.join(os.getcwd(), timenow + "testFileUpload.txt"))
except OSError as e:
return e.errno == errno.ENOENT
return res
def test_min2(self):
"""this tests executes a job that requires at least 2 processors"""
j = Job()
j.setName("MP-test-min2")
# FIXME: the number of processors should be discovered at runtime using JobParameters.getNumberOfJobProcessors()
# here, and later
j.setExecutable(self.mpExeFlex, arguments="2")
j.setInputSandbox(
find_all("mpTest-flexible.py", rootPath,
"DIRAC/tests/Utilities")[0])
j.setNumberOfProcessors(
minNumberOfProcessors=2) # This requires at least 2 processors
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
self.assertTrue(res["OK"])
def test_Integration_Production(self):
lfns = ['/lhcb/MC/2012/BDSTH.STRIP.DST/00051752/0000/00051752_00001263_2.bdsth.Strip.dst']
# From request 31139
optionFiles = '$APPCONFIGOPTS/Merging/DVMergeDST.py;$APPCONFIGOPTS/DaVinci/DataType-2012.py;'
optionFiles += '$APPCONFIGOPTS/Merging/WriteFSR.py;$APPCONFIGOPTS/Merging/MergeFSR.py'
stepsInProd = [{'StepId': 129424,
'StepName': 'Stripping24NoPrescalingFlagged',
'ApplicationName': 'DaVinci',
'ApplicationVersion': 'v40r1p2',
'ExtraPackages': 'AppConfig.v3r263',
'ProcessingPass': '******',
'Visible': 'N', 'Usable': 'Yes',
'DDDB': 'dddb-20130929-1',
'CONDDB': 'sim-20130522-1-vc-md100',
'DQTag': '',
'OptionsFormat': 'Merge',
'OptionFiles': optionFiles,
'mcTCK': '', 'ExtraOptions': '',
'isMulticore': 'N',
'SystemConfig': '',
'fileTypesIn': ['BDSTH.STRIP.DST'],
'fileTypesOut':['BDSTH.STRIP.DST'],
'visibilityFlag':[{'Visible': 'Y', 'FileType': 'BDSTH.STRIP.DST'}]}]
prod = self.pr._buildProduction('Merge', stepsInProd, {'BDSTH.STRIP.DST': 'Tier1_MC-DST'}, 0, 100,
inputDataPolicy='protocol', inputDataList=lfns)
try:
# This is the standard location in Jenkins
prod.LHCbJob.setInputSandbox(find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
prod.LHCbJob.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
prod.LHCbJob.setConfigArgs('pilot.cfg')
res = self.diracProduction.launchProduction(prod, False, True, 0)
self.assertTrue(res['OK'])
def test_Integration_Production(self):
# From request 41740
lfns = ['/lhcb/LHCb/Collision17/BRUNELHIST/00064899/0006/Brunel_00064899_00067792_1.Hist.root',
'/lhcb/LHCb/Collision17/BRUNELHIST/00064899/0006/Brunel_00064899_00067793_1.Hist.root']
stepsInProd = [{'StepId': 132150,
'StepName': 'Histo-merging-Reco17-6500GeV-MagUp-Full',
'ApplicationName': 'Noether',
'ApplicationVersion': 'v1r4',
'ExtraPackages': 'AppConfig.v3r337',
'ProcessingPass': '******',
'Visible': 'Y',
'Usable': 'Yes',
'DDDB': '',
'CONDDB': '',
'DQTag': '',
'OptionsFormat': '',
'OptionFiles': ' $APPCONFIGOPTS/DataQuality/DQMergeRun.py',
'mcTCK': '', 'ExtraOptions': '',
'isMulticore': 'N',
'SystemConfig': '',
'fileTypesIn': ['BRUNELHIST', 'DAVINCIHIST'],
'fileTypesOut':['HIST.ROOT'],
'visibilityFlag':[{'Visible': 'Y', 'FileType': 'HIST.ROOT'}]}]
prod = self.pr._buildProduction('HistoMerge', stepsInProd, {'HIST.ROOT': 'CERN-EOS-HIST'}, 0, 100,
inputDataPolicy='protocol', inputDataList=lfns)
try:
# This is the standard location in Jenkins
prod.LHCbJob.setInputSandbox(find_all('pilot.cfg', os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
prod.LHCbJob.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
prod.LHCbJob.setConfigArgs('pilot.cfg')
res = self.diracProduction.launchProduction(prod, False, True, 0)
self.assertTrue(res['OK'])
def test_fixed(self):
"""this tests executes a job that requires exactly 4 processors"""
j = Job()
j.setName("MP-test")
j.setExecutable(self.mpExe)
j.setInputSandbox(
find_all("mpTest.py", rootPath, "DIRAC/tests/Utilities")[0])
j.setNumberOfProcessors(
4) # This requires a fixed number of processors
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
if multiprocessing.cpu_count() > 1:
self.assertTrue(res["OK"])
else:
self.assertFalse(res["OK"])
def setUp(self):
super(UserJobTestCase, self).setUp()
self.d = Dirac()
self.exeScriptLocation = find_all('exe-script.py', '.',
'Integration')[0]
self.mpExe = find_all('testMpJob.sh', '.', 'Utilities')[0]
def jobWithOutput():
"""Creates a job that uploads an output.
The output SE is not set here, so it would use the default /Resources/StorageElementGroups/SE-USER
And possibly use for failover /Resources/StorageElementGroups/Tier1-Failover
"""
timenow = time.strftime("%s")
inp1 = [os.path.join(os.getcwd(), timenow + "testFileUpload.txt")]
with open(inp1[0], "wt") as f:
f.write(timenow)
J = baseToAllJobs("jobWithOutput")
try:
inp2 = [find_all("exe-script.py", rootPath, "DIRAC/tests/Workflow")[0]]
J.setInputSandbox(inp1 + inp2)
except IndexError: # we are in Jenkins
inp2 = [
find_all("exe-script.py", os.environ["WORKSPACE"],
"DIRAC/tests/Workflow")[0]
]
J.setInputSandbox(inp1 + inp2)
J.setExecutable("exe-script.py", "", "helloWorld.log")
J.setOutputData([timenow + "testFileUpload.txt"])
res = endOfAllJobs(J)
try:
os.remove(os.path.join(os.getcwd(), timenow + "testFileUpload.txt"))
except OSError as e:
return e.errno == errno.ENOENT
return res
def test_min2max4(self):
""" this tests executes a job that requires 2 to 4 processors
"""
j = Job()
j.setName("MP-test-min2max4")
j.setExecutable(self.mpExeFlex, arguments='2')
j.setInputSandbox(
find_all('mpTest-flexible.py', rootPath,
'DIRAC/tests/Utilities')[0])
j.setNumberOfProcessors(
minNumberOfProcessors=2,
maxNumberOfProcessors=4) # This requires 2 to 4 processors
j.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
j.setConfigArgs('pilot.cfg')
res = j.runLocal(self.d)
self.assertTrue(res['OK'])
def mpJob():
J = baseToAllJobs('mpJob')
J.setInputSandbox(
[find_all('mpTest.py', rootPath, 'DIRAC/tests/Utilities')[0]] +
[find_all('testMpJob.sh', rootPath, 'DIRAC/tests/Utilities')[0]])
J.setExecutable('testMpJob.sh mpTest.py')
J.setTag('MultiProcessor')
return endOfAllJobs(J)
def test_Integration_Production(self):
options = '$APPCONFIGOPTS/Gauss/Beam6500GeV-md100-2015-nu1.6.py;'
options += '$APPCONFIGOPTS/Gauss/EnableSpillover-25ns.py;'
options += '$APPCONFIGOPTS/Gauss/DataType-2015.py;'
options += '$APPCONFIGOPTS/Gauss/RICHRandomHits.py;'
options += '$DECFILESROOT/options/28144011.py;'
options += '$LBPYTHIA8ROOT/options/Pythia8.py;'
options += '$APPCONFIGOPTS/Gauss/G4PL_FTFP_BERT_EmNoCuts.py'
# From request 48257
stepsInProd = [{
'StepId': 133659,
'StepName': 'Sim09d',
'ApplicationName': 'Gauss',
'ApplicationVersion': 'v49r10',
'ExtraPackages': 'AppConfig.v3r359;Gen/DecFiles.v30r17',
'ProcessingPass': '******',
'Visible': 'Y',
'Usable': 'Yes',
'DDDB': 'dddb-20170721-3',
'CONDDB': 'sim-20161124-vc-md100',
'DQTag': '',
'OptionsFormat': '',
'OptionFiles': options,
'isMulticore': 'N',
'SystemConfig': 'x86_64-slc6-gcc48-opt',
'mcTCK': '',
'ExtraOptions': '',
'fileTypesIn': [],
'fileTypesOut': ['SIM'],
'visibilityFlag': [{
'Visible': 'N',
'FileType': 'SIM'
}]
}]
# First create the production object
prod = self.pr._buildProduction(prodType='MCSimulation',
stepsInProd=stepsInProd,
outputSE={'DIGI': 'Tier1_MC-DST'},
priority=0,
cpu=100,
outputFileMask='DIGI')
try:
# This is the standard location in Jenkins
prod.LHCbJob.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
prod.LHCbJob.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
prod.LHCbJob.setConfigArgs('pilot.cfg')
prod.setParameter('numberOfEvents', 'string', 2,
'Number of events to test')
# Then launch it
res = self.diracProduction.launchProduction(prod, False, True, 0)
self.assertTrue(res['OK'])
def test_Regression_Production(self):
try:
location66219 = find_all('66219.xml', os.environ['WORKSPACE'], '/LHCbDIRAC/tests/Workflow/Regression')[0]
except (IndexError, KeyError):
location66219 = find_all('66219.xml', rootPath, '/LHCbDIRAC/tests/Workflow/Regression')[0]
j_mergeHISTO_66219 = LHCbJob(location66219)
j_mergeHISTO_66219.setConfigArgs('pilot.cfg')
res = j_mergeHISTO_66219.runLocal(self.diracLHCb)
self.assertTrue(res['OK'])
def helloWorld():
""" simple hello world job
"""
J = baseToAllJobs('helloWorld')
try:
J.setInputSandbox([find_all('exe-script.py', rootPath, 'DIRAC/tests/Workflow')[0]])
except IndexError: # we are in Jenkins
J.setInputSandbox([find_all('exe-script.py', os.environ['WORKSPACE'], 'DIRAC/tests/Workflow')[0]])
J.setExecutable("exe-script.py", "", "helloWorld.log")
return endOfAllJobs(J)
def helloWorld():
""" simple hello world job
"""
J = baseToAllJobs('helloWorld')
try:
J.setInputSandbox([find_all('exe-script.py', rootPath, 'DIRAC/tests/Workflow')[0]])
except IndexError: # we are in Jenkins
J.setInputSandbox([find_all('exe-script.py', os.environ['WORKSPACE'], 'DIRAC/tests/Workflow')[0]])
J.setExecutable("exe-script.py", "", "helloWorld.log")
return endOfAllJobs(J)
def setUp(self):
super(UserJobTestCase, self).setUp()
self.d = Dirac()
self.exeScriptLocation = find_all('exe-script.py', '..',
'/DIRAC/tests/Workflow')[0]
self.helloWorld = find_all("helloWorld.py", '..',
'/DIRAC/tests/Workflow')[0]
self.mpExe = find_all('testMpJob.sh', '..',
'/DIRAC/tests/Utilities')[0]
gLogger.setLevel('DEBUG')
def parametricJob():
""" Creates a parametric job with 3 subjobs which are simple hello world jobs
"""
J = baseToAllJobs('helloWorld')
try:
J.setInputSandbox([find_all('exe-script.py', rootPath, 'DIRAC/tests/Workflow')[0]])
except IndexError: # we are in Jenkins
J.setInputSandbox([find_all('exe-script.py', os.environ['WORKSPACE'], 'DIRAC/tests/Workflow')[0]])
J.setParameterSequence("args", ['one', 'two', 'three'])
J.setParameterSequence("iargs", [1, 2, 3])
J.setExecutable("exe-script.py", arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log')
return endOfAllJobs(J)
def parametricJob():
""" Creates a parametric job with 3 subjobs which are simple hello world jobs
"""
J = baseToAllJobs('helloWorld')
try:
J.setInputSandbox([find_all('exe-script.py', rootPath, 'DIRAC/tests/Workflow')[0]])
except IndexError: # we are in Jenkins
J.setInputSandbox([find_all('exe-script.py', os.environ['WORKSPACE'], 'DIRAC/tests/Workflow')[0]])
J.setParameterSequence("args", ['one', 'two', 'three'])
J.setParameterSequence("iargs", [1, 2, 3])
J.setExecutable("exe-script.py", arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log')
return endOfAllJobs(J)
def test_Regression_Production(self):
# Reco from Reco17 # FIXME: the input file is not there.
try:
location63284 = find_all('63284.xml', os.environ['WORKSPACE'],
'/LHCbDIRAC/tests/Workflow/Regression')[0]
except (IndexError, KeyError):
location63284 = find_all('63284.xml', rootPath,
'/LHCbDIRAC/tests/Workflow/Regression')[0]
j_reco_63284 = LHCbJob(location63284)
j_reco_63284.setConfigArgs('pilot.cfg')
res = j_reco_63284.runLocal(self.diracLHCb)
self.assertTrue(res['OK'])
def test_Regression_Production(self):
# Turbo Stripping Collision15em
try:
location46403 = find_all('46403.xml', os.environ['WORKSPACE'],
'/LHCbDIRAC/tests/Workflow/Regression')[0]
except (IndexError, KeyError):
location46403 = find_all('46403.xml', rootPath,
'/LHCbDIRAC/tests/Workflow/Regression')[0]
j_stripp_46403 = LHCbJob(location46403)
j_stripp_46403.setConfigArgs('pilot.cfg')
res = j_stripp_46403.runLocal(self.diracLHCb)
self.assertTrue(res['OK'])
def setUp(self):
super(RegressionTestCase, self).setUp()
gLogger.setLevel('DEBUG')
self.dirac = Dirac()
try:
exeScriptLoc = find_all('exe-script.py', rootPath, '/DIRAC/tests/Workflow/Regression')[0]
helloWorldLoc = find_all('helloWorld.py', rootPath, '/DIRAC/tests/Workflow/Regression')[0]
except IndexError: # we are in Jenkins
exeScriptLoc = find_all('exe-script.py', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow/Regression')[0]
helloWorldLoc = find_all('helloWorld.py', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow/Regression')[0]
shutil.copyfile(exeScriptLoc, './exe-script.py')
shutil.copyfile(helloWorldLoc, './helloWorld.py')
try:
helloWorldXMLLocation = find_all('helloWorld.xml', rootPath, '/DIRAC/tests/Workflow/Regression')[0]
except IndexError: # we are in Jenkins
helloWorldXMLLocation = find_all('helloWorld.xml', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow/Regression')[0]
self.j_u_hello = Job(helloWorldXMLLocation)
try:
helloWorldXMLFewMoreLocation = find_all('helloWorld.xml', rootPath, '/DIRAC/tests/Workflow/Regression')[0]
except IndexError: # we are in Jenkins
helloWorldXMLFewMoreLocation = find_all(
'helloWorld.xml',
os.environ['WORKSPACE'],
'/DIRAC/tests/Workflow/Regression')[0]
self.j_u_helloPlus = Job(helloWorldXMLFewMoreLocation)
def setUp(self):
super(UserJobTestCase, self).setUp()
self.d = Dirac()
integration_test_dir = "/DIRAC/tests/Workflow/Integration"
try:
self.exeScriptLocation = find_all("exe-script.py", rootPath,
integration_test_dir)[0]
self.helloWorld = find_all("helloWorld.py", rootPath,
integration_test_dir)[0]
self.mpExe = find_all("mpTest.py", rootPath,
"/DIRAC/tests/Utilities")[0]
self.mpExeFlex = find_all("mpTest-flexible.py", rootPath,
"/DIRAC/tests/Utilities")[0]
except IndexError: # we are in Jenkins
self.exeScriptLocation = find_all("exe-script.py",
os.environ["WORKSPACE"],
integration_test_dir)[0]
self.helloWorld = find_all("helloWorld.py",
os.environ["WORKSPACE"],
integration_test_dir)[0]
self.mpExe = find_all("mpTest.py", os.environ["WORKSPACE"],
"/DIRAC/tests/Utilities")[0]
self.mpExeFlex = find_all("mpTest-flexible.py",
os.environ["WORKSPACE"],
"/DIRAC/tests/Utilities")[0]
gLogger.setLevel("DEBUG")
def helloWorldCloud():
"""simple hello world job to Cloud at Imperial College"""
J = baseToAllJobs("helloCloudWorld")
try:
J.setInputSandbox(
[find_all("exe-script.py", rootPath, "DIRAC/tests/Workflow")[0]])
except IndexError: # we are in Jenkins --- are we ?
J.setInputSandbox([
find_all("exe-script.py", os.environ["WORKSPACE"],
"DIRAC/tests/Workflow")[0]
])
J.setExecutable("exe-script.py", "", "helloWorldCloud.log")
J.setDestination("CLOUD.UKI-LT2-IC-HEP.uk")
return endOfAllJobs(J)
def helloWorldNCBJ():
"""simple hello world job to NCBJ"""
J = baseToAllJobs("helloWorldNCBJ")
try:
J.setInputSandbox(
[find_all("exe-script.py", rootPath, "DIRAC/tests/Workflow")[0]])
except IndexError: # we are in Jenkins
J.setInputSandbox([
find_all("exe-script.py", os.environ["WORKSPACE"],
"DIRAC/tests/Workflow")[0]
])
J.setExecutable("exe-script.py", "", "helloWorld.log")
J.setDestination("LCG.NCBJ.pl")
return endOfAllJobs(J)
def mpJob():
""" simple hello world job, with MultiProcessor tag
"""
J = baseToAllJobs('mpJob')
try:
J.setInputSandbox([find_all('mpTest.py', rootPath, 'DIRAC/tests/Utilities')[0]] +
[find_all('testMpJob.sh', rootPath, 'DIRAC/tests/Utilities')[0]])
except IndexError: # we are in Jenkins
J.setInputSandbox([find_all('mpTest.py', os.environ['WORKSPACE'], 'DIRAC/tests/Utilities')[0]] +
[find_all('testMpJob.sh', os.environ['WORKSPACE'], 'DIRAC/tests/Utilities')[0]])
J.setExecutable('testMpJob.sh mpTest.py')
J.setTag('MultiProcessor')
return endOfAllJobs(J)
def mpJob():
""" simple hello world job, with MultiProcessor tag
"""
J = baseToAllJobs('mpJob')
try:
J.setInputSandbox([find_all('mpTest.py', rootPath, 'DIRAC/tests/Utilities')[0]] +
[find_all('testMpJob.sh', rootPath, 'DIRAC/tests/Utilities')[0]])
except IndexError: # we are in Jenkins
J.setInputSandbox([find_all('mpTest.py', os.environ['WORKSPACE'], 'DIRAC/tests/Utilities')[0]] +
[find_all('testMpJob.sh', os.environ['WORKSPACE'], 'DIRAC/tests/Utilities')[0]])
J.setExecutable('testMpJob.sh mpTest.py')
J.setTag('MultiProcessor')
return endOfAllJobs(J)
def helloWorldJob():
job = Job()
job.setName("helloWorld")
exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
job.setInputSandbox(exeScriptLocation)
job.setExecutable(exeScriptLocation, "", "helloWorld.log")
return job
def helloWorldJob(): job = Job() job.setName( "helloWorld" ) exeScriptLocation = find_all( 'exe-script.py', '.', 'WorkloadManagementSystem' )[0] job.setInputSandbox( exeScriptLocation ) job.setExecutable( exeScriptLocation, "", "helloWorld.log" ) return job
def helloWorldJob():
job = Job()
job.setName("helloWorld")
exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
job.setInputSandbox(exeScriptLocation)
job.setExecutable(exeScriptLocation, "", "helloWorld.log")
return job
def helloWorldJob(): job = Job() job.setName( "helloWorld" ) exeScriptLocation = find_all( 'exe-script.py', '.', 'WorkloadManagementSystem' )[0] job.setInputSandbox( exeScriptLocation ) job.setExecutable( exeScriptLocation, "", "helloWorld.log" ) return job
def test_Gauss_Production_PR33857_1(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc48-opt'
This is taken from PR 33857 (and would fall back to SetupProject)
"""
gLogger.always("**** GAUSS v49r5")
ra = RunApplication()
ra.applicationName = 'Gauss'
ra.applicationVersion = 'v49r5'
ra.systemConfig = 'x86_64-slc6-gcc48-opt'
ra.commandOptions = ['$APPCONFIGOPTS/Gauss/Sim08-Beam4000GeV-mu100-2012-nu2.5.py',
'$APPCONFIGOPTS/Gauss/DataType-2012.py',
'$APPCONFIGOPTS/Gauss/RICHRandomHits.py',
'$APPCONFIGOPTS/Gauss/NoPacking.py',
'$DECFILESROOT/options/12877041.py',
'$LBPYTHIA8ROOT/options/Pythia8.py',
'$APPCONFIGOPTS/Gauss/G4PL_FTFP_BERT_EmNoCuts.py',
'$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py']
ra.extraPackages = [('AppConfig', 'v3r277'),
('Gen/DecFiles', 'v29r10'),
('ProdConf', '')
]
ra.prodConfFileName = find_all('test_prodConf_gauss_v49r5.py', '..')[0]
ra.applicationLog = '00033857_00000001_1_log.txt'
ra.stdError = '00033857_00000001_1_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def test_DaVinci_new_gcc62(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc62-opt'
This is taken from step 130847
"""
gLogger.always("**** DAVINCI v42r2")
ra = RunApplication()
ra.applicationName = 'DaVinci'
ra.applicationVersion = 'v42r2'
ra.systemConfig = 'x86_64-slc6-gcc62-opt'
ra.commandOptions = ['$APPCONFIGOPTS/DaVinci/DV-Stripping27-Stripping.py',
'$APPCONFIGOPTS/DaVinci/DataType-2016.py',
'$APPCONFIGOPTS/DaVinci/InputType-RDST.py',
'$APPCONFIGOPTS/DaVinci/DV-RawEventJuggler-0_3-to-4_2.py']
ra.extraPackages = [('AppConfig', 'v3r308'),
('Det/SQLDDDB', 'v7r11'),
('ProdConf', '')
]
ra.step_Number = 1
ra.prodConfFileName = find_all('test_prodConf_davinci_v42r2.py', '..')[0]
ra.applicationLog = '0daVinci_000v42r2_62_log.txt'
ra.stdError = '0daVinci_000v42r2_62_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def setUp( self ):
super( RegressionTestCase, self ).setUp()
self.dirac = Dirac()
exeScriptLoc = find_all( 'exe-script.py', '.', 'Regression' )[0]
helloWorldLoc = find_all( 'helloWorld.py', '.', 'Regression' )[0]
shutil.copyfile( exeScriptLoc, './exe-script.py' )
shutil.copyfile( helloWorldLoc, './helloWorld.py' )
helloWorldXMLLocation = find_all( 'helloWorld.xml', '.', 'Regression' )[0]
self.j_u_hello = Job( helloWorldXMLLocation )
helloWorldXMLFewMoreLocation = find_all( 'helloWorld.xml', '.', 'Regression' )[0]
self.j_u_helloPlus = Job( helloWorldXMLFewMoreLocation )
def test_DaVinci_1_Production_PR33857_6(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc5-gcc46-opt'
This is taken from PR 33857
"""
gLogger.always("**** DAVINCI v32r2p1")
ra = RunApplication()
ra.applicationName = 'DaVinci'
ra.applicationVersion = 'v32r2p1'
ra.systemConfig = 'x86_64-slc5-gcc46-opt'
ra.commandOptions = ['$CHARMCONFIGOPTS/MCFiltering/D02K3PiFromB2DstmunuXStripTrigFiltering_2012.py',
'$APPCONFIGOPTS/DaVinci/DataType-2012.py',
'$APPCONFIGOPTS/DaVinci/InputType-DST.py',
'$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py']
ra.extraPackages = [('AppConfig', 'v3r305'),
('CharmConfig', 'v3r30'),
('ProdConf', '')
]
ra.step_Number = 1
ra.prodConfFileName = find_all('test_prodConf_davinci_v32r2p1.py', '..')[0]
ra.applicationLog = '00033857_00000006_6_log.txt'
ra.stdError = '00033857_00000006_6_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def putAndDelete():
# Find the test data
fj = find_all("WMSHistory_testData.json", "../", "tests/Integration/Monitoring")[0]
with open(fj) as fp:
data = json.load(fp)
# hack to work on a test index, which is just the same as WMSHistory
mapping = {
"properties": {
"Status": {"type": "keyword"},
"Site": {"type": "keyword"},
"JobSplitType": {"type": "keyword"},
"ApplicationStatus": {"type": "keyword"},
"MinorStatus": {"type": "keyword"},
"User": {"type": "keyword"},
"JobGroup": {"type": "keyword"},
"UserGroup": {"type": "keyword"},
}
}
monitoringDB.documentTypes.setdefault(
"test", {"indexName": "test", "mapping": mapping, "monitoringFields": ["Jobs", "Reschedules"], "period": "day"}
)
# put
res = monitoringDB.put(data, "test")
assert res["OK"]
# Add a time delay to allow updating the modified index before querying it.
time.sleep(4)
yield putAndDelete
# from here on is teardown
# delete the index
monitoringDB.deleteIndex("test-*")
def test_execute( self ):
job = Job()
job.setName( "helloWorld-test" )
job.setExecutable( find_all( "helloWorld.py", '.', 'Integration' )[0],
arguments = "This is an argument",
logFile = "aLogFileForTest.txt" ,
parameters=[('executable', 'string', '', "Executable Script"),
('arguments', 'string', '', 'Arguments for executable Script'),
( 'applicationLog', 'string', '', "Log file name" ),
( 'someCustomOne', 'string', '', "boh" )],
paramValues = [( 'someCustomOne', 'aCustomValue' )] )
job.setBannedSites( ['LCG.SiteA.com', 'DIRAC.SiteB.org'] )
job.setOwner( 'ownerName' )
job.setOwnerGroup( 'ownerGroup' )
job.setName( 'jobName' )
job.setJobGroup( 'jobGroup' )
job.setType( 'jobType' )
job.setDestination( 'DIRAC.someSite.ch' )
job.setCPUTime( 12345 )
job.setLogLevel( 'DEBUG' )
res = job.runLocal( self.d )
self.assertTrue( res['OK'] )
def helloWorld():
J = baseToAllJobs('helloWorld')
J.setInputSandbox(
[find_all('exe-script.py', rootPath, 'DIRAC/tests/Workflow')[0]])
J.setExecutable("exe-script.py", "", "helloWorld.log")
return endOfAllJobs(J)
def test_DaVinci_2_Production_PR33857_7(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc49-opt'
This is taken from PR 33857
"""
gLogger.always("**** DAVINCI v41r3")
ra = RunApplication()
ra.applicationName = 'DaVinci'
ra.applicationVersion = 'v41r3'
ra.systemConfig = 'x86_64-slc6-gcc49-opt'
ra.commandOptions = ['$APPCONFIGOPTS/Merging/DVMergeDST.py',
'$APPCONFIGOPTS/DaVinci/DataType-2012.py',
'$APPCONFIGOPTS/Merging/WriteFSR.py',
'$APPCONFIGOPTS/Merging/MergeFSR.py']
ra.extraPackages = [('AppConfig', 'v3r305'),
('ProdConf', '')
]
ra.step_Number = 1
ra.prodConfFileName = find_all('test_prodConf_davinci_v41r3.py', '..')[0]
ra.applicationLog = '00033857_00000007_7_log.txt'
ra.stdError = '00033857_00000007_7_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def parametricJob():
job = Job()
job.setName("parametric_helloWorld_%n")
exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
job.setInputSandbox(exeScriptLocation)
job.setParameterSequence("args", ['one', 'two', 'three'])
job.setParameterSequence("iargs", [1, 2, 3])
job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log')
return job
def test_execute( self ):
""" this one tests that I can execute a job that requires multi-processing
"""
j = Job()
j.setName( "MP-test" )
j.setExecutable( self.mpExe )
j.setInputSandbox( find_all( 'mpTest.py', '.', 'Utilities' )[0] )
j.setTag( 'MultiProcessor' )
res = j.runLocal( self.d )
if multiprocessing.cpu_count() > 1:
self.assertTrue( res['OK'] )
else:
self.assertFalse( res['OK'] )
def test_SSCChain( self ):
""" full test of functionalities
"""
ssc = SandboxStoreClient()
smDB = SandboxMetadataDB()
exeScriptLocation = find_all( 'exe-script.py', '..', '/DIRAC/tests/Integration' )[0]
fileList = [exeScriptLocation]
res = ssc.uploadFilesAsSandbox( fileList )
self.assertTrue(res['OK'])
# SEPFN = res['Value'].split( '|' )[1]
res = ssc.uploadFilesAsSandboxForJob( fileList, 1, 'Input' )
self.assertTrue(res['OK'])
# res = ssc.downloadSandboxForJob( 1, 'Input' ) #to run this would need the RSS on
# self.assertTrue(res['OK'])
# only ones needing the DB
res = smDB.getUnusedSandboxes()
self.assertTrue(res['OK'])
def setUp(self):
super(UserJobTestCase, self).setUp()
self.d = Dirac()
try:
self.exeScriptLocation = find_all('exe-script.py', rootPath, '/DIRAC/tests/Workflow')[0]
self.helloWorld = find_all("helloWorld.py", rootPath, '/DIRAC/tests/Workflow')[0]
self.mpExe = find_all('testMpJob.sh', rootPath, '/DIRAC/tests/Utilities')[0]
except IndexError: # we are in Jenkins
self.exeScriptLocation = find_all('exe-script.py', os.environ['WORKSPACE'], '/DIRAC/tests/Workflow')[0]
self.helloWorld = find_all("helloWorld.py", os.environ['WORKSPACE'], '/DIRAC/tests/Workflow')[0]
self.mpExe = find_all('testMpJob.sh', os.environ['WORKSPACE'], '/DIRAC/tests/Utilities')[0]
gLogger.setLevel('DEBUG')
def helloWorld(): J = baseToAllJobs( 'helloWorld' ) J.setInputSandbox( [find_all( 'exe-script.py', '.', 'GridTestSubmission' )[0]] ) J.setExecutable( "exe-script.py", "", "helloWorld.log" ) return endOfAllJobs( J )
def setUp( self ):
super( UserJobTestCase, self ).setUp()
self.d = Dirac()
self.exeScriptLocation = find_all( 'exe-script.py', '.', 'Integration' )[0]
self.mpExe = find_all( 'testMpJob.sh', '.', 'Utilities' )[0]
from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac #from tests.Workflow.Integration.Test_UserJobs import createJob gLogger.setLevel( 'DEBUG' ) cwd = os.path.realpath( '.' ) dirac = Dirac() # Simple Hello Word job to DIRAC.Jenkins.ch gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch" ) helloJ = Job() helloJ.setName( "helloWorld-TEST-TO-Jenkins" ) helloJ.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJ.setExecutable( "exe-script.py", "", "helloWorld.log" ) helloJ.setCPUTime( 17800 ) helloJ.setDestination( 'DIRAC.Jenkins.ch' ) result = dirac.submit( helloJ ) gLogger.info( "Hello world job: ", result ) if not result['OK']: gLogger.error( "Problem submitting job", result['Message'] ) exit( 1 ) # Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN" ) helloJMP = Job() helloJMP.setName( "helloWorld-TEST-TO-Jenkins-MP" ) helloJMP.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJMP.setExecutable( "exe-script.py", "", "helloWorld.log" )