def readInputData(self, optsfiles, extraopts=False):
'''Returns a LHCbDataSet object from a list of options files. The
optional argument extraopts will decide if the extraopts string inside
the application is considered or not.
Usage examples:
# Create an LHCbDataset object with the data found in the optionsfile
l=DaVinci(version='v22r0p2').readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Get the data from an options file and assign it to the jobs inputdata
field
j.inputdata = j.application.readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Assuming you have data in your extraopts, you can use the extraopts.
# In this case your extraopts need to be fully parseable by gaudirun.py
# So you must make sure that you have the proper import statements.
# e.g.
from Gaudi.Configuration import *
# If you mix optionsfiles and extraopts, as usual extraopts may
# overwright your options
#
# Use this to create a new job with data from extraopts of an old job
j=Job(inputdata=jobs[-1].application.readInputData([],True))
'''
def dummyfile():
temp_fd, temp_filename = tempfile.mkstemp(text=True, suffix='.py')
os.write(temp_fd, "Dummy file to keep the Optionsparser happy")
os.close(temp_fd)
return temp_filename
if type(optsfiles) != type([]):
optsfiles = [optsfiles]
# use a dummy file to keep the parser happy
if len(optsfiles) == 0:
optsfiles.append(dummyfile())
if extraopts:
extraopts = self.extraopts
else:
extraopts = ""
# parser = check_inputs(optsfiles, extraopts, self.env)
try:
parser = PythonOptionsParser(optsfiles, extraopts,
self.getenv(False))
except Exception as err:
msg = 'Unable to parse the job options. Please check options ' \
'files and extraopts.'
logger.error("PythonOptionsParserError:\n%s" % str(err))
raise ApplicationConfigurationError(msg)
return GPIProxyObjectFactory(parser.get_input_data())
def readInputData(self, optsfiles, extraopts=False):
'''Returns a LHCbDataSet object from a list of options files. The
optional argument extraopts will decide if the extraopts string inside
the application is considered or not.
Usage examples:
# Create an LHCbDataset object with the data found in the optionsfile
l=DaVinci(version='v22r0p2').readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Get the data from an options file and assign it to the jobs inputdata
field
j.inputdata = j.application.readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Assuming you have data in your extraopts, you can use the extraopts.
# In this case your extraopts need to be fully parseable by gaudirun.py
# So you must make sure that you have the proper import statements.
# e.g.
from Gaudi.Configuration import *
# If you mix optionsfiles and extraopts, as usual extraopts may
# overwright your options
#
# Use this to create a new job with data from extraopts of an old job
j=Job(inputdata=jobs[-1].application.readInputData([],True))
'''
def dummyfile():
temp_fd, temp_filename = tempfile.mkstemp(text=True, suffix='.py')
os.write(temp_fd, "Dummy file to keep the Optionsparser happy")
os.close(temp_fd)
return temp_filename
if type(optsfiles) != type([]):
optsfiles = [optsfiles]
# use a dummy file to keep the parser happy
if len(optsfiles) == 0:
optsfiles.append(dummyfile())
if extraopts:
extraopts = self.extraopts
else:
extraopts = ""
# parser = check_inputs(optsfiles, extraopts, self.env)
try:
parser = PythonOptionsParser(
optsfiles, extraopts, self.getenv(False))
except Exception as err:
msg = 'Unable to parse the job options. Please check options ' \
'files and extraopts.'
logger.error("PythonOptionsParserError:\n%s" % str(err))
raise ApplicationConfigurationError(None, msg)
return GPIProxyObjectFactory(parser.get_input_data())
class TestPythonOptionsParser(GangaGPITestCase):
def setUp(self):
job = Job(application=Gauss())
job.application.platform = 'x86_64-slc6-gcc48-opt'
gauss = job.application._impl
gauss._getshell()
optsfiles = ['./TestGaudi/Gauss-Job.py']
self.extraopts = 'EventSelector(Input=[\"DATAFILE=\'LFN:dummy.dst\' TYP=\'POOL_ROOTTREE\' OPT=\'READ\'\"])'
self.parser = PythonOptionsParser(optsfiles, self.extraopts, gauss.env)
self.job = job
def test_PythonOptionsParser__get_opts_dict_and_pkl_string(self):
options, opts_pkl_string = self.parser._get_opts_dict_and_pkl_string()
assert options, 'failed to get options'
assert opts_pkl_string, 'failed to get pkl string'
def test_PythonOptionsParser__join_opts_files(self):
joined_str = self.parser._join_opts_files()
ok = joined_str.rfind(self.extraopts)
assert ok, 'extraopts not added to options'
def test_PythonOptionsParser_get_input_data(self):
input = self.parser.get_input_data()
assert len(input) == 1, 'problem collecting input data files'
def test_PythonOptionsParser_get_output_files(self):
config = Ganga.Utility.Config.getConfig('LHCb')
# start w/ default
config.setUserValue(
'outputsandbox_types',
['NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream'])
sandbox, data = self.parser.get_output_files()
ok = sandbox.count('GaussHistos.root') == 1 and \
data.count('Gauss.sim') == 1
assert ok, 'collecting/sorting of output files failed (default)'
# move the .sim file
config.setUserValue('outputsandbox_types', [
'NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream',
'GaussTape'
])
sandbox, data = self.parser.get_output_files()
ok = sandbox.count('GaussHistos.root') == 1 and \
sandbox.count('Gauss.sim') == 1 and \
data.count('Gauss.sim') == 0
assert ok, 'collecting/sorting of output files failed (.sim->sandbox)'
# move the .root files to data
config.setUserValue('outputsandbox_types', ['GaussTape'])
sandbox, data = self.parser.get_output_files()
ok = data.count('GaussHistos.root') == 1 and \
sandbox.count('Gauss.sim') == 1 and \
len(data) == 1 and len(sandbox) == 1
assert ok, 'collecting/sorting of output files failed (.root->data)'
def test_PythonOptionsParser_get_output(self):
config = Ganga.Utility.Config.getConfig('LHCb')
# previous method tests the configure stuff...let's just make sure we
# can properly override it in a job
config.setUserValue(
'outputsandbox_types',
['NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream'])
j = self.job
sandbox, data = self.parser.get_output(j)
ok = sandbox.count('GaussHistos.root') == 1 and \
data.count('Gauss.sim') == 1
assert ok, 'collecting/sorting of output files failed (default)'
j.outputfiles = []
j.outputfiles = ['Gauss.sim']
sandbox, data = self.parser.get_output(j)
ok = sandbox.count('Gauss.sim') == 1 and len(data) == 0
print type(sandbox)
assert ok, 'collecting/sorting of output files failed (.sim->sandbox) %s -> %s' % (
str(sandbox.count('Gauss.sim')), str(len(data)))
j.outputfiles = []
j.outputfiles = [DiracFile('*.root')]
sandbox, data = self.parser.get_output(j)
ok = len(sandbox) == 0 and data.count('GaussHistos.root') == 1
assert ok, 'collecting/sorting of output files failed (.root->data)'
# make sure if matches both goes to data
j.outputfiles = ['*.sim']
j.outputfiles = [DiracFile('*.sim')]
sandbox, data = self.parser.get_output(j)
ok = data.count('Gauss.sim') == 1 and sandbox.count('Gauss.sim') == 0
assert ok, 'collecting/sorting of output files failed (matches both)'
class TestPythonOptionsParser(GangaGPITestCase):
def setUp(self):
job = Job(application=Gauss())
job.application.platform = 'x86_64-slc6-gcc48-opt'
gauss = job.application._impl
gauss._getshell()
optsfiles = ['./TestGaudi/Gauss-Job.py']
self.extraopts = 'EventSelector(Input=[\"DATAFILE=\'LFN:dummy.dst\' TYP=\'POOL_ROOTTREE\' OPT=\'READ\'\"])'
self.parser = PythonOptionsParser(optsfiles, self.extraopts, gauss.env)
self.job = job
def test_PythonOptionsParser__get_opts_dict_and_pkl_string(self):
options, opts_pkl_string = self.parser._get_opts_dict_and_pkl_string()
assert options, 'failed to get options'
assert opts_pkl_string, 'failed to get pkl string'
def test_PythonOptionsParser__join_opts_files(self):
joined_str = self.parser._join_opts_files()
ok = joined_str.rfind(self.extraopts)
assert ok, 'extraopts not added to options'
def test_PythonOptionsParser_get_input_data(self):
input = self.parser.get_input_data()
assert len(input) == 1, 'problem collecting input data files'
def test_PythonOptionsParser_get_output_files(self):
config = Ganga.Utility.Config.getConfig('LHCb')
# start w/ default
config.setUserValue('outputsandbox_types',
['NTupleSvc', 'HistogramPersistencySvc',
'MicroDSTStream'])
sandbox, data = self.parser.get_output_files()
ok = sandbox.count('GaussHistos.root') == 1 and \
data.count('Gauss.sim') == 1
assert ok, 'collecting/sorting of output files failed (default)'
# move the .sim file
config.setUserValue('outputsandbox_types',
['NTupleSvc', 'HistogramPersistencySvc',
'MicroDSTStream', 'GaussTape'])
sandbox, data = self.parser.get_output_files()
ok = sandbox.count('GaussHistos.root') == 1 and \
sandbox.count('Gauss.sim') == 1 and \
data.count('Gauss.sim') == 0
assert ok, 'collecting/sorting of output files failed (.sim->sandbox)'
# move the .root files to data
config.setUserValue('outputsandbox_types', ['GaussTape'])
sandbox, data = self.parser.get_output_files()
ok = data.count('GaussHistos.root') == 1 and \
sandbox.count('Gauss.sim') == 1 and \
len(data) == 1 and len(sandbox) == 1
assert ok, 'collecting/sorting of output files failed (.root->data)'
def test_PythonOptionsParser_get_output(self):
config = Ganga.Utility.Config.getConfig('LHCb')
# previous method tests the configure stuff...let's just make sure we
# can properly override it in a job
config.setUserValue('outputsandbox_types',
['NTupleSvc', 'HistogramPersistencySvc',
'MicroDSTStream'])
j = self.job
sandbox, data = self.parser.get_output(j)
ok = sandbox.count('GaussHistos.root') == 1 and \
data.count('Gauss.sim') == 1
assert ok, 'collecting/sorting of output files failed (default)'
j.outputfiles = []
j.outputfiles = ['Gauss.sim']
sandbox, data = self.parser.get_output(j)
ok = sandbox.count('Gauss.sim') == 1 and len(data) == 0
print type(sandbox)
assert ok, 'collecting/sorting of output files failed (.sim->sandbox) %s -> %s' % (
str(sandbox.count('Gauss.sim')), str(len(data)))
j.outputfiles = []
j.outputfiles = [DiracFile('*.root')]
sandbox, data = self.parser.get_output(j)
ok = len(sandbox) == 0 and data.count('GaussHistos.root') == 1
assert ok, 'collecting/sorting of output files failed (.root->data)'
# make sure if matches both goes to data
j.outputfiles = ['*.sim']
j.outputfiles = [DiracFile('*.sim')]
sandbox, data = self.parser.get_output(j)
ok = data.count('Gauss.sim') == 1 and sandbox.count('Gauss.sim') == 0
assert ok, 'collecting/sorting of output files failed (matches both)'
