def get_brunel(custom_db=False):
# Add Brunel to cmtuser if needed
brunel_dir = join(os.environ['HOME'], 'cmtuser', 'BrunelDev_v51r1')
if not isdir(brunel_dir):
brunel = prepareGaudiExec('Brunel', 'v51r1')
# Configure Brunel
brunel = GaudiExec()
brunel.directory = brunel_dir
brunel.options = [
'assets/MC-WithTruth.py', 'assets/Brunel-Upgrade-Baseline-20150522.py',
'assets/xdst.py', 'assets/brunel_options.py'
]
if custom_db:
brunel.extraOpts = (
'from Configurables import CondDB\n'
'from Configurables import CondDBAccessSvc\n'
'CondDB().addLayer(dbFile="DDDB.db", dbName="DDDB")\n'
'CondDB().addLayer(dbFile="SIMCOND.db", dbName="SIMCOND")\n'
'alignment_conditions = CondDBAccessSvc("AlignmentConditions")\n'
'alignment_conditions.ConnectionString = "sqlite_file:Alignment_SIMCOND.db/SIMCOND"\n'
'CondDB().addLayer(alignment_conditions)\n')
else:
brunel.extraOpts = ('from Configurables import Brunel\n'
'from Configurables import LHCbApp\n'
'LHCbApp().DDDBtag = "dddb-20160304"\n'
'LHCbApp().CondDBtag = "sim-20150716-vc-md100"\n')
return brunel
def testPrepareJob(self):
from Ganga.GPI import Job, LocalFile, prepareGaudiExec
import os
if os.path.exists(TestExternalGaudiExec.tmpdir_release):
os.system("rm -rf %s/*" % TestExternalGaudiExec.tmpdir_release)
j = Job(application=prepareGaudiExec(
'DaVinci', latestDaVinci(), TestExternalGaudiExec.tmpdir_release))
myHelloOpts = path.join(TestExternalGaudiExec.tmpdir_release,
'hello.py')
FileBuffer('hello.py', 'print("Hello")').create(myHelloOpts)
assert path.isfile(myHelloOpts)
j.application.options = [LocalFile(myHelloOpts)]
j.prepare()
assert j.application.is_prepared.name
assert path.isdir(j.application.is_prepared.path())
def testConstruction(self):
"""
This tests that we can construct a GaudiExec object in a simple way
"""
from Ganga.GPI import Job, prepareGaudiExec
j = Job(application=prepareGaudiExec('DaVinci', latestDaVinci(), TestExternalGaudiExec.tmpdir_release))
assert j.application.directory == path.join(TestExternalGaudiExec.tmpdir_release, 'DaVinciDev_%s' % latestDaVinci())
assert path.isfile(path.join(TestExternalGaudiExec.tmpdir_release, 'DaVinciDev_%s' % latestDaVinci(), 'run'))
assert path.isfile(path.join(TestExternalGaudiExec.tmpdir_release, 'DaVinciDev_%s' % latestDaVinci(), 'Makefile'))
def testPrepareJob(self):
from Ganga.GPI import Job, LocalFile, prepareGaudiExec
j = Job(application=prepareGaudiExec('DaVinci', latestDaVinci(), TestExternalGaudiExec.tmpdir_release))
myHelloOpts = path.join(TestExternalGaudiExec.tmpdir_release, 'hello.py')
FileBuffer('hello.py', 'print("Hello")').create(myHelloOpts)
assert path.isfile(myHelloOpts)
j.application.options=[LocalFile(myHelloOpts)]
j.prepare()
def make_exec_app(application='DaVinci', version='v42r1'):
"""Function to create a temporary folder, creates a DaVinci folder
and recursively copies the content of cwd into that folder to be tarred
and uploaded as well."""
import tempfile
import os
from distutils.dir_util import copy_tree
from Ganga.GPI import prepareGaudiExec
tmp = tempfile.mktemp()
os.makedirs(tmp)
app = prepareGaudiExec(application, version, tmp)
location = app.directory
print('Copying content in {} to {}'.format(os.getcwd(), location))
copy_tree(os.getcwd(), location)
return app
def testConstruction(self):
"""
This tests that we can construct a GaudiExec object in a simple way
"""
from Ganga.GPI import Job, prepareGaudiExec
j = Job(application=prepareGaudiExec(
'DaVinci', latestDaVinci(), TestExternalGaudiExec.tmpdir_release))
assert j.application.directory == path.join(
TestExternalGaudiExec.tmpdir_release,
'DaVinciDev_%s' % latestDaVinci())
assert path.isfile(
path.join(TestExternalGaudiExec.tmpdir_release,
'DaVinciDev_%s' % latestDaVinci(), 'run'))
assert path.isfile(
path.join(TestExternalGaudiExec.tmpdir_release,
'DaVinciDev_%s' % latestDaVinci(), 'Makefile'))
def _constructJob():
"""
This is a helper method to construct a new GaudiExec job object for submission testing
This just helps reduce repeat code between tests
"""
import os
if os.path.exists(TestExternalGaudiExec.tmpdir_release):
os.system("rm -fr %s/" % TestExternalGaudiExec.tmpdir_release)
from Ganga.GPI import Job, LocalFile, prepareGaudiExec
j = Job(application=prepareGaudiExec('DaVinci', latestDaVinci(), TestExternalGaudiExec.tmpdir_release))
myOpts = path.join(TestExternalGaudiExec.tmpdir_release, 'testfile.py')
FileBuffer('testfile.py', 'print("ThisIsATest")').create(myOpts)
j.application.options=[LocalFile(myOpts)]
return j
def _constructJob():
"""
This is a helper method to construct a new GaudiExec job object for submission testing
This just helps reduce repeat code between tests
"""
import os
if os.path.exists(TestExternalGaudiExec.tmpdir_release):
os.system("rm -fr %s/" % TestExternalGaudiExec.tmpdir_release)
from Ganga.GPI import Job, LocalFile, prepareGaudiExec
j = Job(application=prepareGaudiExec(
'DaVinci', latestDaVinci(), TestExternalGaudiExec.tmpdir_release))
myOpts = path.join(TestExternalGaudiExec.tmpdir_release, 'testfile.py')
FileBuffer('testfile.py', 'print("ThisIsATest")').create(myOpts)
j.application.options = [LocalFile(myOpts)]
return j
def testSubmitJobComplete(self):
"""
Test that the job completes successfully
"""
from Ganga.GPI import jobs
from Ganga.GPI import Job, LocalFile, prepareGaudiExec
import os
if os.path.exists(TestExternalGaudiExec.tmpdir_release):
os.system("rm -rf %s/*" % TestExternalGaudiExec.tmpdir_release)
j = Job(application=prepareGaudiExec(
'DaVinci', latestDaVinci(), TestExternalGaudiExec.tmpdir_release))
myOpts = path.join(TestExternalGaudiExec.tmpdir_release, 'testfile.py')
FileBuffer('testfile.py', 'print("ThisIsATest")').create(myOpts)
j.application.options = [LocalFile(myOpts)]
j.submit()
run_until_completed(j)
assert j.status == 'completed'
outputfile = path.join(j.outputdir, 'stdout')
assert path.isfile(outputfile)
assert 'testfile.py' in open(outputfile).read()
assert 'data.py' in open(outputfile).read()
assert 'ThisIsATest' in open(outputfile).read()
assert j.application.platform in open(outputfile).read()
def testPrepareJob(self):
from Ganga.GPI import Job, LocalFile, prepareGaudiExec
import os
if os.path.exists(TestExternalGaudiExec.tmpdir_release):
os.system("rm -rf %s/*" % TestExternalGaudiExec.tmpdir_release)
j = Job(application=prepareGaudiExec('DaVinci', latestDaVinci(), TestExternalGaudiExec.tmpdir_release))
myHelloOpts = path.join(TestExternalGaudiExec.tmpdir_release, 'hello.py')
FileBuffer('hello.py', 'print("Hello")').create(myHelloOpts)
assert path.isfile(myHelloOpts)
j.application.options=[LocalFile(myHelloOpts)]
j.prepare()
assert j.application.is_prepared.name
assert path.isdir(j.application.is_prepared.path())
def submitDV(jname, tag, script, outfiles,
appName='DaVinci', appPath='.', appVer='v42r3', appArgs=[],
backend=None, infiles=None, gaudirun=False, test=False, maxFiles=-1, do_auto_resubmit = True):
'''
Function for submitting a generic GaudiExec job to the grid with Ganga.
If a local version of the application doesn't exist, prepareGaudiExec is
used to create it. Otherwise GaudiExec is used.
'''
#from GangaDirac.Lib.Backends import Dirac
import os
from Ganga.GPI import (Dirac,
DiracFile,
SplitByFiles,
BKQuery,
Job,
prepareGaudiExec,
GaudiExec
)
if test: maxFiles = 1
if infiles == None:
infiles = getInputList()
if backend == None:
backend = Dirac()
appDir = appPath + '/' + appName + 'Dev_' + appVer
appDir = os.path.expanduser(appDir)
if os.path.isdir(appDir):
app = GaudiExec()
app.directory = appDir
else:
app = prepareGaudiExec(appName,
appVer,
myPath=appPath)
app.options = [script]
app.useGaudiRun = gaudirun
nfiles = JobInfo[jname]['NFiles']
dpath = JobInfo[jname]['Input']
mySplit = SplitByFiles()
mySplit.filesPerJob = nfiles
mySplit.maxFiles = maxFiles
mySplit.ignoremissing = True
j = Job(
name = jname + '.' + tag,
application = app,
splitter = mySplit,
inputdata = BKQuery(path=dpath).getDataset(),
inputfiles = infiles,
outputfiles = [DiracFile(outfile) for outfile in outfiles],
do_auto_resubmit = do_auto_resubmit,
backend = backend
)
j.application.extraArgs += appArgs
j.submit()
