def setUp(self):
job = Job(application=Gauss())
job.application.platform = 'x86_64-slc6-gcc48-opt'
gauss = job.application._impl
gauss._getshell()
optsfiles = ['./TestGaudi/Gauss-Job.py']
self.extraopts = 'EventSelector(Input=[\"DATAFILE=\'LFN:dummy.dst\' TYP=\'POOL_ROOTTREE\' OPT=\'READ\'\"])'
self.parser = PythonOptionsParser(optsfiles, self.extraopts, gauss.env)
self.job = job
def readInputData(self, optsfiles, extraopts=False):
'''Returns a LHCbDataSet object from a list of options files. The
optional argument extraopts will decide if the extraopts string inside
the application is considered or not.
Usage examples:
# Create an LHCbDataset object with the data found in the optionsfile
l=DaVinci(version='v22r0p2').readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Get the data from an options file and assign it to the jobs inputdata
field
j.inputdata = j.application.readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Assuming you have data in your extraopts, you can use the extraopts.
# In this case your extraopts need to be fully parseable by gaudirun.py
# So you must make sure that you have the proper import statements.
# e.g.
from Gaudi.Configuration import *
# If you mix optionsfiles and extraopts, as usual extraopts may
# overwright your options
#
# Use this to create a new job with data from extraopts of an old job
j=Job(inputdata=jobs[-1].application.readInputData([],True))
'''
def dummyfile():
temp_fd, temp_filename = tempfile.mkstemp(text=True, suffix='.py')
os.write(temp_fd, "Dummy file to keep the Optionsparser happy")
os.close(temp_fd)
return temp_filename
if type(optsfiles) != type([]):
optsfiles = [optsfiles]
# use a dummy file to keep the parser happy
if len(optsfiles) == 0:
optsfiles.append(dummyfile())
if extraopts:
extraopts = self.extraopts
else:
extraopts = ""
# parser = check_inputs(optsfiles, extraopts, self.env)
try:
parser = PythonOptionsParser(optsfiles, extraopts,
self.getenv(False))
except Exception as err:
msg = 'Unable to parse the job options. Please check options ' \
'files and extraopts.'
logger.error("PythonOptionsParserError:\n%s" % str(err))
raise ApplicationConfigurationError(msg)
return GPIProxyObjectFactory(parser.get_input_data())
def readInputData(self, optsfiles, extraopts=False):
'''Returns a LHCbDataSet object from a list of options files. The
optional argument extraopts will decide if the extraopts string inside
the application is considered or not.
Usage examples:
# Create an LHCbDataset object with the data found in the optionsfile
l=DaVinci(version='v22r0p2').readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Get the data from an options file and assign it to the jobs inputdata
field
j.inputdata = j.application.readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Assuming you have data in your extraopts, you can use the extraopts.
# In this case your extraopts need to be fully parseable by gaudirun.py
# So you must make sure that you have the proper import statements.
# e.g.
from Gaudi.Configuration import *
# If you mix optionsfiles and extraopts, as usual extraopts may
# overwright your options
#
# Use this to create a new job with data from extraopts of an old job
j=Job(inputdata=jobs[-1].application.readInputData([],True))
'''
def dummyfile():
temp_fd, temp_filename = tempfile.mkstemp(text=True, suffix='.py')
os.write(temp_fd, "Dummy file to keep the Optionsparser happy")
os.close(temp_fd)
return temp_filename
if type(optsfiles) != type([]):
optsfiles = [optsfiles]
# use a dummy file to keep the parser happy
if len(optsfiles) == 0:
optsfiles.append(dummyfile())
if extraopts:
extraopts = self.extraopts
else:
extraopts = ""
# parser = check_inputs(optsfiles, extraopts, self.env)
try:
parser = PythonOptionsParser(
optsfiles, extraopts, self.getenv(False))
except Exception as err:
msg = 'Unable to parse the job options. Please check options ' \
'files and extraopts.'
logger.error("PythonOptionsParserError:\n%s" % str(err))
raise ApplicationConfigurationError(None, msg)
return GPIProxyObjectFactory(parser.get_input_data())
def _get_parser(self):
optsfiles = []
for fileitem in self.optsfile:
if type(fileitem) is str:
optsfiles.append(fileitem)
else:
optsfiles.append(fileitem.name)
#optsfiles = [fileitem.name for fileitem in self.optsfile]
# add on XML summary
extraopts = ''
if self.extraopts:
extraopts += self.extraopts
if self.extraopts.find('LHCbApp().XMLSummary') is -1:
extraopts += "\nfrom Gaudi.Configuration import *"
extraopts += "\nfrom Configurables import LHCbApp"
extraopts += "\nLHCbApp().XMLSummary='summary.xml'"
else:
extraopts += "\nfrom Gaudi.Configuration import *"
extraopts += "\nfrom Configurables import LHCbApp"
extraopts += "\nLHCbApp().XMLSummary='summary.xml'"
try:
parser = PythonOptionsParser(optsfiles, extraopts,
self.getenv(False))
except ApplicationConfigurationError as err:
logger.error("PythonOptionsParserError:\n%s" % str(err))
# fix this when preparing not attached to job
msg2 = ''
try:
debug_dir = self.getJobObject().getDebugWorkspace().getPath()
msg2 += 'You can also view this from within ganga '\
'by doing job.peek(\'../debug/gaudirun.<whatever>\').'
except Exception, err:
logger.debug("path Error:\n%s" % str(err))
debug_dir = tempfile.mkdtemp()
messages = err.message.split('###SPLIT###')
if len(messages) is 2:
stdout = open(debug_dir + '/gaudirun.stdout', 'w')
stderr = open(debug_dir + '/gaudirun.stderr', 'w')
stdout.write(messages[0])
stderr.write(messages[1])
stdout.close()
stderr.close()
msg = 'Unable to parse job options! Please check options ' \
'files and extraopts. The output and error streams from gaudirun.py can be ' \
'found in %s and %s respectively . ' % (stdout.name, stderr.name)
else:
f = open(debug_dir + '/gaudirun.out', 'w')
f.write(err.message)
f.close()
msg = 'Unable to parse job options! Please check options ' \
'files and extraopts. The output from gaudirun.py can be ' \
'found in %s . ' % f.name
msg += msg2
logger.debug(msg)
raise ApplicationConfigurationError(msg)
def setUp(self):
job = Job(application=Gauss())
job.application.platform = 'x86_64-slc6-gcc48-opt'
gauss = job.application._impl
gauss._getshell()
optsfiles = ['./TestGaudi/Gauss-Job.py']
self.extraopts = 'EventSelector(Input=[\"DATAFILE=\'LFN:dummy.dst\' TYP=\'POOL_ROOTTREE\' OPT=\'READ\'\"])'
self.parser = PythonOptionsParser(optsfiles, self.extraopts, gauss.env)
self.job = job
class TestPythonOptionsParser(GangaGPITestCase):
def setUp(self):
job = Job(application=Gauss())
job.application.platform = 'x86_64-slc6-gcc48-opt'
gauss = job.application._impl
gauss._getshell()
optsfiles = ['./TestGaudi/Gauss-Job.py']
self.extraopts = 'EventSelector(Input=[\"DATAFILE=\'LFN:dummy.dst\' TYP=\'POOL_ROOTTREE\' OPT=\'READ\'\"])'
self.parser = PythonOptionsParser(optsfiles, self.extraopts, gauss.env)
self.job = job
def test_PythonOptionsParser__get_opts_dict_and_pkl_string(self):
options, opts_pkl_string = self.parser._get_opts_dict_and_pkl_string()
assert options, 'failed to get options'
assert opts_pkl_string, 'failed to get pkl string'
def test_PythonOptionsParser__join_opts_files(self):
joined_str = self.parser._join_opts_files()
ok = joined_str.rfind(self.extraopts)
assert ok, 'extraopts not added to options'
def test_PythonOptionsParser_get_input_data(self):
input = self.parser.get_input_data()
assert len(input) == 1, 'problem collecting input data files'
def test_PythonOptionsParser_get_output_files(self):
config = Ganga.Utility.Config.getConfig('LHCb')
# start w/ default
config.setUserValue(
'outputsandbox_types',
['NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream'])
sandbox, data = self.parser.get_output_files()
ok = sandbox.count('GaussHistos.root') == 1 and \
data.count('Gauss.sim') == 1
assert ok, 'collecting/sorting of output files failed (default)'
# move the .sim file
config.setUserValue('outputsandbox_types', [
'NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream',
'GaussTape'
])
sandbox, data = self.parser.get_output_files()
ok = sandbox.count('GaussHistos.root') == 1 and \
sandbox.count('Gauss.sim') == 1 and \
data.count('Gauss.sim') == 0
assert ok, 'collecting/sorting of output files failed (.sim->sandbox)'
# move the .root files to data
config.setUserValue('outputsandbox_types', ['GaussTape'])
sandbox, data = self.parser.get_output_files()
ok = data.count('GaussHistos.root') == 1 and \
sandbox.count('Gauss.sim') == 1 and \
len(data) == 1 and len(sandbox) == 1
assert ok, 'collecting/sorting of output files failed (.root->data)'
def test_PythonOptionsParser_get_output(self):
config = Ganga.Utility.Config.getConfig('LHCb')
# previous method tests the configure stuff...let's just make sure we
# can properly override it in a job
config.setUserValue(
'outputsandbox_types',
['NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream'])
j = self.job
sandbox, data = self.parser.get_output(j)
ok = sandbox.count('GaussHistos.root') == 1 and \
data.count('Gauss.sim') == 1
assert ok, 'collecting/sorting of output files failed (default)'
j.outputfiles = []
j.outputfiles = ['Gauss.sim']
sandbox, data = self.parser.get_output(j)
ok = sandbox.count('Gauss.sim') == 1 and len(data) == 0
print type(sandbox)
assert ok, 'collecting/sorting of output files failed (.sim->sandbox) %s -> %s' % (
str(sandbox.count('Gauss.sim')), str(len(data)))
j.outputfiles = []
j.outputfiles = [DiracFile('*.root')]
sandbox, data = self.parser.get_output(j)
ok = len(sandbox) == 0 and data.count('GaussHistos.root') == 1
assert ok, 'collecting/sorting of output files failed (.root->data)'
# make sure if matches both goes to data
j.outputfiles = ['*.sim']
j.outputfiles = [DiracFile('*.sim')]
sandbox, data = self.parser.get_output(j)
ok = data.count('Gauss.sim') == 1 and sandbox.count('Gauss.sim') == 0
assert ok, 'collecting/sorting of output files failed (matches both)'
class TestPythonOptionsParser(GangaGPITestCase):
def setUp(self):
job = Job(application=Gauss())
job.application.platform = 'x86_64-slc6-gcc48-opt'
gauss = job.application._impl
gauss._getshell()
optsfiles = ['./TestGaudi/Gauss-Job.py']
self.extraopts = 'EventSelector(Input=[\"DATAFILE=\'LFN:dummy.dst\' TYP=\'POOL_ROOTTREE\' OPT=\'READ\'\"])'
self.parser = PythonOptionsParser(optsfiles, self.extraopts, gauss.env)
self.job = job
def test_PythonOptionsParser__get_opts_dict_and_pkl_string(self):
options, opts_pkl_string = self.parser._get_opts_dict_and_pkl_string()
assert options, 'failed to get options'
assert opts_pkl_string, 'failed to get pkl string'
def test_PythonOptionsParser__join_opts_files(self):
joined_str = self.parser._join_opts_files()
ok = joined_str.rfind(self.extraopts)
assert ok, 'extraopts not added to options'
def test_PythonOptionsParser_get_input_data(self):
input = self.parser.get_input_data()
assert len(input) == 1, 'problem collecting input data files'
def test_PythonOptionsParser_get_output_files(self):
config = Ganga.Utility.Config.getConfig('LHCb')
# start w/ default
config.setUserValue('outputsandbox_types',
['NTupleSvc', 'HistogramPersistencySvc',
'MicroDSTStream'])
sandbox, data = self.parser.get_output_files()
ok = sandbox.count('GaussHistos.root') == 1 and \
data.count('Gauss.sim') == 1
assert ok, 'collecting/sorting of output files failed (default)'
# move the .sim file
config.setUserValue('outputsandbox_types',
['NTupleSvc', 'HistogramPersistencySvc',
'MicroDSTStream', 'GaussTape'])
sandbox, data = self.parser.get_output_files()
ok = sandbox.count('GaussHistos.root') == 1 and \
sandbox.count('Gauss.sim') == 1 and \
data.count('Gauss.sim') == 0
assert ok, 'collecting/sorting of output files failed (.sim->sandbox)'
# move the .root files to data
config.setUserValue('outputsandbox_types', ['GaussTape'])
sandbox, data = self.parser.get_output_files()
ok = data.count('GaussHistos.root') == 1 and \
sandbox.count('Gauss.sim') == 1 and \
len(data) == 1 and len(sandbox) == 1
assert ok, 'collecting/sorting of output files failed (.root->data)'
def test_PythonOptionsParser_get_output(self):
config = Ganga.Utility.Config.getConfig('LHCb')
# previous method tests the configure stuff...let's just make sure we
# can properly override it in a job
config.setUserValue('outputsandbox_types',
['NTupleSvc', 'HistogramPersistencySvc',
'MicroDSTStream'])
j = self.job
sandbox, data = self.parser.get_output(j)
ok = sandbox.count('GaussHistos.root') == 1 and \
data.count('Gauss.sim') == 1
assert ok, 'collecting/sorting of output files failed (default)'
j.outputfiles = []
j.outputfiles = ['Gauss.sim']
sandbox, data = self.parser.get_output(j)
ok = sandbox.count('Gauss.sim') == 1 and len(data) == 0
print type(sandbox)
assert ok, 'collecting/sorting of output files failed (.sim->sandbox) %s -> %s' % (
str(sandbox.count('Gauss.sim')), str(len(data)))
j.outputfiles = []
j.outputfiles = [DiracFile('*.root')]
sandbox, data = self.parser.get_output(j)
ok = len(sandbox) == 0 and data.count('GaussHistos.root') == 1
assert ok, 'collecting/sorting of output files failed (.root->data)'
# make sure if matches both goes to data
j.outputfiles = ['*.sim']
j.outputfiles = [DiracFile('*.sim')]
sandbox, data = self.parser.get_output(j)
ok = data.count('Gauss.sim') == 1 and sandbox.count('Gauss.sim') == 0
assert ok, 'collecting/sorting of output files failed (matches both)'
