def DataframeParser():
dataframeParser = ArgumentParser( add_help = False )
dataframeGroup = dataframeParser.add_argument_group("TuningTools DATA framework configuration" , "")
dataframeGroup.add_argument( '--data-framework',
type = Dataframe, action = RetrieveDataFramework, default=NotSet,
help = """Specify which data framework should be used in the job.""" )
return dataframeParser
def CoreFrameworkParser():
coreFrameworkParser = ArgumentParser( add_help = False)
coreFrameworkGroup = coreFrameworkParser.add_argument_group("TuningTools CORE configuration" , "")
coreFrameworkGroup.add_argument( '-core', '--core-framework',
type = AvailableTuningToolCores, action = RetrieveCoreFramework,
help = """ Specify which core framework should be used in the job.""" \
+ ( " Current default is: " + AvailableTuningToolCores.tostring( coreConf.default() ) ))
return coreFrameworkParser
def CrossValStatsMonParser():
crossValStatsMonParser = ArgumentParser(
add_help=False,
description=
'Retrieve cross-validation-monitoring information performance.',
conflict_handler='resolve')
reqArgs = crossValStatsMonParser.add_argument_group(
"required arguments", "")
reqArgs.add_argument(
'-f',
'--file',
action='store',
required=True,
help=
"""The crossvalidation data files or folders that will be used to run the
analysis.""")
reqArgs.add_argument(
'-d',
'--dataPath',
default=None,
required=True,
help="""The tuning data file to retrieve the patterns.""")
optArgs = crossValStatsMonParser.add_argument_group(
"optional arguments", "")
optArgs.add_argument('--debug',
default=False,
type=BooleanStr,
help="Debug mode")
optArgs.add_argument('--grid',
default=False,
type=BooleanStr,
help="Enable the grid filter tag.")
optArgs.add_argument('--doBeamer',
default=False,
type=BooleanStr,
help="Enable the beamer creation.")
optArgs.add_argument('--doShortSlides',
default=False,
type=BooleanStr,
help="Enable the beamer short slides.")
optArgs.add_argument('--reference',
default=None,
help="The reference string to be used.")
optArgs.add_argument('--output',
'-o',
default="report",
help="the output file path to the data")
optArgs.add_argument('--choicesfile',
'-c',
default=None,
help="the .mat file with the neuron choices ")
return crossValStatsMonParser
def TuningExpertParser():
tuningExpertParser = ArgumentParser(
add_help=False,
description='Tune expert discriminator for a specific TuningTool data.',
conflict_handler='resolve',
parents=[TuningJobParser()])
tuningExpertParser.delete_arguments('data')
tuningExpertParser.suppress_arguments(core='keras')
tuneExpDataArgs = tuningExpertParser.add_argument_group(
"required arguments", "")
tuneExpDataArgs.add_argument(
'-dc',
'--data-calo',
action='store',
metavar='data_calo',
required=True,
help=
"The calorimeter data file that will be used to tune the discriminators"
)
tuneExpDataArgs.add_argument(
'-dt',
'--data-track',
action='store',
metavar='data_track',
required=True,
help=
"The tracking data file that will be used to tune the discriminators")
tuneExpDataArgs.add_argument(
'-nc',
'--network-calo',
action='store',
metavar='nn_calo',
required=True,
help=
"""List of files of the calorimeter neural networks performance analysis.
There must be one file per bin and they must be ordered from the first
et bin to the last, and for each et the eta bins must also be ordered.
Example:
For et-bins 0 and 1 and eta-bins 0, 1 and 2 the files must be in the
order: Et0 Eta0, Et0 Eta1, Et0 Eta2, Et1 Eta0, Et1 Eta1, Et1 Eta2.
In order to obtain such files, it is necessary to run the executable
crossValidStatAnalysis.py.""")
tuneExpDataArgs.add_argument(
'-nt',
'--network-track',
action='store',
metavar='nn_track',
required=True,
help=
"""List of files of the tracking neural networks performance analysis.
For more information see explanation of network_calo argument."""
)
return tuningExpertParser
#!/usr/bin/env python
import subprocess as sp, json, pprint, pickle, requests, tempfile, os
try:
from RingerCore import ArgumentParser, loggerParser
from RingerCore import emptyArgumentsPrintHelp, Logger, printArgs
except ImportError:
from argparse import ArgumentParser
loggerParser = None
pass
parentParser = ArgumentParser(add_help=False)
parentReqParser = parentParser.add_argument_group("required arguments", '')
parentReqParser.add_argument(
'-u',
'--url',
required=True,
metavar='URL',
action='store',
nargs='+',
help="Bigpanda urls to retrieve the job configuration.")
parents = [parentParser, loggerParser] if loggerParser else [parentParser]
parser = ArgumentParser(
description=
'Retrieve configuration from panda job and create a file to run the job locally',
parents=parents,
conflict_handler='resolve')
if loggerParser:
parser.make_adjustments()
# for et = 1:nEtBin
# for eta = 1:nEtaBin
# for sort = 1:nSort
# projCollection{et,eta,sort} = A % your matrix projection here!
# end
# end
# end
# save('your pp file name', 'nEtaBin', 'nEtBin', 'projCollection')
#
from RingerCore import Logger, LoggingLevel, save, load, ArgumentParser, emptyArgumentsPrintHelp
from TuningTools import fixPPCol
import numpy as np
import sys, os
mainParser = ArgumentParser()
mainParser.add_argument('-i','--input', action='store', required = True,
help = "The input files [.mat] that will be used to generate a extract the preproc proj matrix.\
Your matlab file must be inside: projCollection = [[[]]], nEtaBin = int, nEtBin = int, \
nSort = int")
mainParser.add_argument('-o','--output', action='store', required = False, default = 'ppCol',
help = "The output file")
emptyArgumentsPrintHelp( mainParser )
mainLogger = Logger.getModuleLogger( __name__, LoggingLevel.INFO )
args=mainParser.parse_args()
import scipy.io
__all__ = ['tuningJobParser', 'tuningExpertParser']
from RingerCore import NotSet, ArgumentParser, BooleanStr
from TuningTools.dataframe.EnumCollection import RingerOperation
from TuningTools.TuningJob import BatchSizeMethod
from TuningTools.coreDef import hasFastnet, hasExmachina
from TuningTools.parsers.BaseModuleParser import coreFrameworkParser
from TuningTools.CrossValid import CrossValidMethod
################################################################################
# Create tuningJob file related objects
################################################################################
tuningJobParser = ArgumentParser(
add_help=False,
description='Tune discriminator for a specific TuningTool data.',
conflict_handler='resolve',
parents=[coreFrameworkParser])
tuningDataArgs = tuningJobParser.add_argument_group("required arguments", "")
tuningDataArgs.add_argument(
'-d',
'--data',
action='store',
metavar='data',
required=True,
help="The data file that will be used to tune the discriminators")
tuningOptArgs = tuningJobParser.add_argument_group("optional arguments", "")
tuningOptArgs.add_argument(
'--outputFileBase',
action='store',
default=NotSet,
#!/usr/bin/env python
from RingerCore import Logger, LoggerNamespace, ArgumentParser, emptyArgumentsPrintHelp
parser = ArgumentParser()
parser.add_argument('-o', '--output',action='store', default="merge.root",
help='output merged file')
parser.add_argument('-n', '--nFilesPerMerge', type=int, default=2,
help='Number of files per merge')
parser.add_argument('-i', '--input' ,action='store', nargs='+',required=True,
help='input file')
mainLogger = Logger.getModuleLogger(__name__)
#***************************** Main ******************************
emptyArgumentsPrintHelp( parser )
args = parser.parse_args( namespace = LoggerNamespace() )
mainLogger.info('Starting merging files...')
poutput = args.output.replace('.root','')
import numpy as np
mainLogger.info(('Trying to merge %d files')%(len(args.input)))
files = args.input
files.sort()
imerge=0
all_merged_files=str()
while len(files) > 0:
#!/usr/bin/env python
from RingerCore import (csvStr2List, emptyArgumentsPrintHelp, expandFolders,
Logger, progressbar, LoggingLevel, ArgumentParser)
from TuningTools.parsers import loggerParser, LoggerNamespace
mainParser = ArgumentParser(description='Export files pdfs into unique file.',
add_help=False)
mainMergeParser = mainParser.add_argument_group("Required arguments", "")
mainMergeParser.add_argument(
'-i',
'--inputFiles',
action='store',
metavar='InputFiles',
required=True,
nargs='+',
help="The input files that will be used to generate a matlab file")
mainMergeParser.add_argument('-f',
'--filter-keys',
action='store',
required=False,
default=[''],
nargs='+',
help="Filter histogram keys to be exported.")
mainMergeParser.add_argument('-e',
'--exclude-keys',
action='store',
required=False,
default=[],
nargs='+',
__all__ = ['crossValStatsJobParser']
from RingerCore import ArgumentParser, BooleanStr, NotSet
from TuningTools.dataframe.EnumCollection import RingerOperation
from TuningTools.TuningJob import ChooseOPMethod
################################################################################
# Create cross valid stats job parser file related objects
################################################################################
crossValStatsJobParser = ArgumentParser(add_help = False,
description = 'Retrieve cross-validation information and tuned discriminators performance.',
conflict_handler = 'resolve')
################################################################################
reqArgs = crossValStatsJobParser.add_argument_group( "required arguments", "")
reqArgs.add_argument('-d', '--discrFiles', action='store',
metavar='data', required = True,
help = """The tuned discriminator data files or folders that will be used to run the
cross-validation analysis.""")
################################################################################
optArgs = crossValStatsJobParser.add_argument_group( "optional arguments", "")
# TODO Reset this when running on the Grid to the GridJobFilter
optArgs.add_argument('--binFilters', action='store', default = NotSet,
help = """This option filter the files types from each job. It can be a string
with the name of a class defined on python/CrossValidStat dedicated to automatically
separate the files or a comma separated list of patterns that identify unique group
of files for each bin. A python list can also be speficied.
E.g.: You can specify 'group001,group002' if you have file001.group001.pic,
file002.group001, file001.group002, file002.group002 available and group001
specifies one binning limit, and group002 another, both of them with 2 files
available in this case.
return False
del cobject
# Everything is good
return True
def save(self):
self._logger.info(('Saving file %s') % (self._outputDS))
self._file.Close()
######################### __main__ ############################
from RingerCore import expandFolders, csvStr2List, ArgumentParser
from pprint import pprint
mainFilterParser = ArgumentParser()
mainFilterParser.add_argument(
'-i',
'--inputFiles',
action='store',
metavar='InputFiles',
required=True,
nargs='+',
help="The input files that will be used to generate a extract file")
mainFilterParser.add_argument(
'-t',
'--trigger',
action='store',
default='e0_perf_L1EM15',
def TuningJobParser():
tuningJobParser = ArgumentParser(
add_help=False,
description='Tune discriminator for a specific TuningTool data.',
conflict_handler='resolve',
parents=[CoreFrameworkParser(),
DataCurationParser()])
tuningOptArgs = tuningJobParser.add_argument_group("optional arguments",
"")
tuningOptArgs.add_argument(
'--outputFileBase',
action='store',
default=NotSet,
help=
"""Base name for the output file, e.g. 'nn-tuned', 'tunedDiscr' etc."""
)
tuningOptArgs.add_argument(
'-odir',
'--outputDir',
action='store',
default=NotSet,
help=
"""Output directory path. When not specified, output will be created in PWD."""
)
tuningOptArgs.add_argument('--overwrite',
type=BooleanStr,
default=NotSet,
help="""Whether to overwrite output files.""")
tuningLoopVars = tuningJobParser.add_argument_group(
"Looping configuration", "")
tuningLoopVars.add_argument(
'-c',
'--confFileList',
nargs='+',
default=None,
help="""A python list or a comma separated list of the
root files containing the configuration to run the jobs. The files can
be generated using a CreateConfFiles instance which can be accessed via
command line using the createTuningJobFiles.py script.""")
tuningLoopVars.add_argument('--neuronBounds',
nargs='+',
type=int,
default=None,
help="""
Input a sequential bounded list to be used as the
neuron job range, the arguments should have the
same format from the seq unix command or as the
Matlab format. If not specified, the range will
start from 1. I.e 5 2 9 leads to [5 7 9] and 50
leads to 1:50
""")
tuningLoopVars.add_argument('--sortBounds',
nargs='+',
type=int,
default=None,
help="""
Input a sequential bounded list using seq format to
be used as the sort job range, but the last bound
will be opened just as happens when using python
range function. If not specified, the range will
start from 0. I.e. 5 2 9 leads to [5 7] and 50 leads
to range(50)
""")
tuningLoopVars.add_argument('--initBounds',
nargs='+',
type=int,
default=None,
help="""
Input a sequential bounded list using seq format to
be used as the inits job range, but the last bound
will be opened just as happens when using python
range function. If not specified, the range will
start from 0. I.e. 5 2 9 leads to [5 7] and 50 leads
to range(50)
""")
tuningDepVars = tuningJobParser.add_argument_group("Binning configuration",
"")
addBinningOptions(tuningDepVars)
tuningOptArgs.add_argument('--compress',
type=BooleanStr,
default=NotSet,
help="""Whether to compress output files.""")
tuningArgs = tuningJobParser.add_argument_group(
"Tuning CORE configuration", "")
tuningArgs.add_argument(
'--show-evo',
type=int,
default=NotSet,
help="""The number of iterations where performance is shown.""")
tuningArgs.add_argument(
'--max-fail',
type=int,
default=NotSet,
help="""Maximum number of failures to imrpove performance over
validation dataset that is tolerated.""")
tuningArgs.add_argument(
'--epochs',
type=int,
default=NotSet,
help="""Number of iterations where the tuning algorithm can run the
optimization.""")
tuningArgs.add_argument('--do-perf',
type=int,
default=NotSet,
help="""Whether we should run performance
testing under convergence conditions, using test/validation dataset
and also estimate operation condition.""")
tuningArgs.add_argument('--batch-size',
type=int,
default=NotSet,
help="""Set the batch size used during tuning.""")
tuningArgs.add_argument(
'--batch-method',
type=BatchSizeMethod,
default=NotSet,
help="""Set the batch size method to be used during tuning.
If batch size is set this will be overwritten by Manual
method. """)
tuningArgs.add_argument(
'--expert-networks',
nargs='+',
default=NotSet,
help="""The Cross-Valid summary data file with the expert networks.""")
if hasExmachina:
exMachinaArgs = tuningJobParser.add_argument_group(
"ExMachina CORE configuration", "")
exMachinaArgs.add_argument('--algorithm-name',
default=NotSet,
help="""The tuning method to use.""")
exMachinaArgs.add_argument(
'--network-arch',
default=NotSet,
help="""The neural network architeture to use.""")
exMachinaArgs.add_argument(
'--cost-function',
default=NotSet,
help="""The cost function used by ExMachina.""")
exMachinaArgs.add_argument(
'--shuffle',
default=NotSet,
help="""Whether to shuffle datasets while training.""")
else:
tuningJobParser.set_defaults(algorithm_name=NotSet,
network_arch=NotSet,
cost_function=NotSet,
shuffle=NotSet)
if hasFastnet:
fastNetArgs = tuningJobParser.add_argument_group(
"FastNet CORE configuration", "")
fastNetArgs.add_argument(
'--seed',
default=NotSet,
help="""The seed to be used by the tuning algorithm.""")
fastNetArgs.add_argument('--do-multi-stop',
default=NotSet,
type=BooleanStr,
help="""Tune classifier using P_D, P_F and
SP when set to True. Uses only SP when set to False.""")
fastNetArgs.add_argument(
'--add-pileup',
default=None,
type=BooleanStr,
help="""Propagate pile-up to the network hidden layer.""")
else:
tuningJobParser.set_defaults(seed=NotSet, do_multi_stop=NotSet)
return tuningJobParser
def CrossValStatsJobParser():
extraDescr = """NOTE: Only valid when --redo-decision-making flag to be set."""
crossValStatsJobParser = ArgumentParser(add_help = False,
description = 'Retrieve cross-validation information and tuned discriminators performance.',
conflict_handler = 'resolve',
parents=[DataCurationParser( False, extraDescr)])
################################################################################
reqArgs = crossValStatsJobParser.add_argument_group( "required arguments", "")
reqArgs.add_argument('-d', '--discrFiles', action='store',
metavar='files', required = True,
help = """The tuned discriminator data files or folders that will be used to run the
cross-validation analysis.""")
################################################################################
decisionMakingArgs = crossValStatsJobParser.add_argument_group( "Decision making configuration", extraDescr)
addBinningOptions( decisionMakingArgs, 'thres' )
decisionMakingArgs.add_argument('--redo-decision-making', default = NotSet, type = BooleanStr
, help = """Whether to overwrite decision making process using
specified method and data. When NotSet, its value will be set
accordingly to --data.""" )
decisionMakingArgs.add_argument('-dmm', '--decision-making-method', default = NotSet, type = DecisionMakingMethod
, help = """Specify which method should be used in the decision making process.""" )
decisionMakingArgs.add_argument('-pup', '--pile-up-ref', default = NotSet, type = PileupReference
, help = """Specify the pileup reference to be used. This will be used
to determine the label in the plots using pile-up and the limits if not specified.""" )
decisionMakingArgs.add_argument('--fr-margin', default = NotSet, type = float, nargs = '+'
, help = """Specify the fake rate margin
multipliers to calculate the correction range
estimators.""")
decisionMakingArgs.add_argument('--max-corr', default = NotSet, type = float
, help = """Specify the maximum correction limit.""" )
decisionMakingArgs.add_argument('--pile-up-limits', default = NotSet, nargs = '+', type = float
, help = """Specify the limits for calculating
the linear correction method. It must have
size equal exactly to 3.""" )
################################################################################
optArgs = crossValStatsJobParser.add_argument_group( "optional arguments", "")
# TODO Reset this when running on the Grid to the GridJobFilter
optArgs.add_argument('--binFilters', action='store', default = NotSet,
help = """This option filter the files types from each job. It can be a string
with the name of a class defined on python/CrossValidStat dedicated to automatically
separate the files or a comma separated list of patterns that identify unique group
of files for each bin. A python list can also be speficied.
E.g.: You can specify 'group001,group002' if you have file001.group001.pic,
file002.group001, file001.group002, file002.group002 available and group001
specifies one binning limit, and group002 another, both of them with 2 files
available in this case.
When not set, all files are considered to be from the same binning.
""")
optArgs.add_argument('-idx','--binFilterIdx', default = None, nargs='+',type = int,
help = """The index of the bin job to run. e.g. two bins, idx will be: 0 and 1"""
)
optArgs.add_argument('--doMonitoring', default=NotSet,
help = "Enable or disable monitoring file creation.", type=BooleanStr,
)
optArgs.add_argument('--doMatlab', default=NotSet,
help = "Enable or disable matlab file creation.", type=BooleanStr,
)
optArgs.add_argument('--doCompress', default=NotSet, type=BooleanStr,
help = "Enable or disable raw output file compression."
)
optArgs.add_argument('-r','--refFile', default = None,
help = """The performance reference file to retrieve the operation points.""")
optArgs.add_argument('-op','--operation', default = None, type=RingerOperation,
help = """The Ringer operation determining in each Trigger
level or what is (are) the offline operation point reference(s)."""
)
optArgs.add_argument('-rocm','--roc-method', nargs='+', default = NotSet, type=ChooseOPMethod,
help = """The method to be used to choose the operation
point on the ROC. Usually it will come in the following order:
SP Pd Pf."""
)
optArgs.add_argument('-eps','--epsilon', nargs='+', default = NotSet, type=float,
help = """The value is used to calculate the limits in
which a point is accept as valid. Usually it will come in the following order:
SP Pd Pf."""
)
optArgs.add_argument('-modelm','--model-method', nargs='+', default = NotSet, type=ChooseOPMethod,
help = """The method to be used to choose the best model to operate from
all models available operating at the chosen operation point by the roc method.
Usually it will come in the following order: SP Pd Pf.
"""
)
optArgs.add_argument('-imodelm','--init-model-method', default = NotSet, #ChooseOPMethod.MSE, type=ChooseOPMethod,
help = """Whether to overwrite, for all operation points,
the initialization model choice method by the one specified
here. If not set, it will use the same value as modelm.
"""
)
optArgs.add_argument('-aeps','--AUC-epsilon', nargs='+', default = NotSet, type=float,
help = """The Area Under the ROC Curve epsilon value. This
value is used as a delta from the reference in which the value is calculated.
Usually it will come in the following order: SP Pd Pf.
""")
optArgs.add_argument('--expandOP', default = NotSet, type=BooleanStr,
help = """If the tune was done using --do-multi-stop set to false, then
this option will try to expand the operation point to derive the best
models for each OP case, using the target Pd/Pf/SP. If no target Pd/Pf
is available, then it will print a warning and choose only one model
via SP maximization.
""")
optArgs.add_argument('--always-use-SP-network', default = NotSet, type=BooleanStr,
help = """If the tune was done using --do-multi-stop set to true, then
this option will force all operation points to be derived using the bestSP
neural network epoch.
""")
optArgs.add_argument('--outputFileBase', action='store', default = NotSet,
help = """Base name for the output file.""")
optArgs.add_argument('--fullDumpNeurons', nargs='+', type=int, default = NotSet,
help = """
For debuging purporses, input a sequential bounded
list to be used as the neurons to be fully dumped
by the monitoring.
""")
optArgs.add_argument('--overwrite', default = NotSet, type=BooleanStr,
help = """
Whether to overwrite files or just skip those which
already exist.
"""
)
optArgs.add_argument('--tmpFolder', '--tmpDir', default = None,
help = """
Specify tmp folder for tar files extraction.
Consider using this option if you are facing disk
space issues. Please also note that failing jobs
won't delete the created temporary files, so the
user might need to take action by doing (assuming
that the default tmp folder is /tmp):
rm $(find /tmp -name "*.pic")
"""
)
optArgs.add_argument('--test', type=BooleanStr,
help = "Set debug mode.")
return crossValStatsJobParser
def CreateDataParser():
createDataParser = ArgumentParser(
add_help=False,
description='Create TuningTool data from PhysVal.',
parents=[CoreFrameworkParser(),
DataframeParser()])
from TuningTools.dataframe.EnumCollection import Reference, Detector, PileupReference
mainCreateData = createDataParser.add_argument_group(
"Required arguments", "")
mainCreateData.add_argument(
'-s',
'--sgnInputFiles',
action='store',
metavar='SignalInputFiles',
required=True,
nargs='+',
help="The signal files that will be used to tune the discriminators")
mainCreateData.add_argument(
'-b',
'--bkgInputFiles',
action='store',
metavar='BackgroundInputFiles',
required=True,
nargs='+',
help="The background files that will be used to tune the discriminators"
)
mainCreateData.add_argument(
'-op',
'--operation',
default=NotSet,
type=RingerOperation,
help="""The Ringer operation determining in each Trigger
level or what is the offline operation point reference."""
)
mainCreateData.add_argument(
'-t',
'--treePath',
metavar='TreePath',
action='store',
default=NotSet,
type=str,
nargs='+',
help="""The Tree path to be filtered on the files. It can be a value for
each dataset.""")
optCreateData = createDataParser.add_argument_group(
"Configuration extra arguments", "")
optCreateData.add_argument('--reference',
action='store',
nargs='+',
default=NotSet,
type=Reference,
help="""
The reference used for filtering datasets. It needs to be set
to a value on the Reference enumeration on ReadData file.
You can set only one value to be used for both datasets, or one
value first for the Signal dataset and the second for the Background
dataset.""")
optCreateData.add_argument('-tEff',
'--efficiencyTreePath',
metavar='EfficienciyTreePath',
action='store',
default=NotSet,
type=str,
nargs='+',
help="""The Tree path to calculate efficiency.
If not specified, efficiency is calculated upon treePath.""")
optCreateData.add_argument('-l1',
'--l1EmClusCut',
default=NotSet,
type=float,
help="The L1 cut threshold")
optCreateData.add_argument('-l2',
'--l2EtCut',
default=NotSet,
type=float,
help="The L2 Et cut threshold")
optCreateData.add_argument('-ef',
'--efEtCut',
default=NotSet,
type=float,
help="The EF Et cut threshold")
optCreateData.add_argument('-off',
'--offEtCut',
default=NotSet,
type=float,
help="The Offline Et cut threshold")
optCreateData.add_argument(
'--getRatesOnly',
default=NotSet,
action='store_true',
help="""Don't save output file, just print benchmark
algorithm operation reference.""")
optCreateData.add_argument(
'--etBins',
action='store',
nargs='+',
default=NotSet,
type=float,
help="E_T bins (GeV) where the data should be segmented.")
optCreateData.add_argument(
'--etaBins',
action='store',
nargs='+',
default=NotSet,
type=float,
help="eta bins where the data should be segmented.")
optCreateData.add_argument(
'--ringConfig',
action='store',
nargs='+',
type=int,
default=NotSet,
help="Number of rings for each eta bin segmentation.")
optCreateData.add_argument(
'-nC',
'--nClusters',
default=NotSet,
type=int,
help="Maximum number of events to add to each dataset.")
optCreateData.add_argument(
'-o',
'--pattern-output-file',
default=NotSet,
help=
"The pickle intermediate file that will be used to train the datasets. It also contains the efficiency targets."
)
optCreateData.add_argument(
'-of',
'--efficiency-output-file',
default=NotSet,
help="File only containing the dumped efficiencies for posterior use.")
optCreateData.add_argument(
'--crossFile',
default=NotSet,
type=str,
help="""Cross-Validation file which will be used to tune the Ringer
Discriminators.""")
optCreateData.add_argument(
'--extractDet',
action='store',
default=NotSet,
type=Detector,
help=""" Which detector to export data from. """)
optCreateData.add_argument(
'--standardCaloVariables',
default=NotSet,
type=BooleanStr,
help=
"Whether to use standard calorimeter variables or rings information.")
optCreateData.add_argument(
'--useTRT',
default=NotSet,
type=BooleanStr,
help="Enable or disable TRT usage when exporting tracking information."
)
optCreateData.add_argument('--toMatlab',
default=NotSet,
type=BooleanStr,
help="Also save data on matlab format.")
optCreateData.add_argument('--plotMeans',
default=NotSet,
type=BooleanStr,
help="Plot pattern mean values.")
optCreateData.add_argument('--plotProfiles',
default=NotSet,
type=BooleanStr,
help="Plot pattern profiles.")
optCreateData.add_argument('--pileupRef',
action='store',
type=BooleanStr,
default=NotSet,
help="Luminosity reference branch.")
optCreateData.add_argument(
'--supportTriggers',
default=NotSet,
type=BooleanStr,
help="Whether reading data comes from support triggers.")
optCreateData.add_argument(
'-l',
'--label',
default=NotSet,
help="The label for tagging what has been proccessed.")
################################################################################
return createDataParser
#!/usr/bin/env python
from RingerCore import (ArgumentParser, BooleanStr, OMP_NUM_THREADS, grouper,
Logger, LoggingLevel)
from itertools import repeat
import sys, os, tempfile
from shutil import rmtree
from subprocess import call, PIPE
hadd_chunk = ArgumentParser(add_help=False, description='')
hadd_chunk.add_argument('target', action='store', help="Target file")
hadd_chunk.add_argument('files', action='store', nargs='+', help="Input files")
hadd_chunk.add_argument('--chunk-size',
action='store',
type=int,
default=50,
help="Chunk size of the hadd jobs")
hadd_chunk.add_argument('--divide-by-run',
action='store_true',
help="""Try to divide using run counts equally.""")
args, argv = hadd_chunk.parse_known_args()
argv = list(argv)
mainLogger = Logger.getModuleLogger(__name__, LoggingLevel.INFO)
def lcall(inputs):
target = inputs[0]
files = list(filter(lambda x: x is not None, inputs[1]))
with open(os.devnull, "w") as w:
if not call(['hadd'] + argv + [target] + files, stdout=w):
defaultTrigList = [
'e24_medium_L1EM18VH',
'e24_lhmedium_L1EM18VH',
'e24_tight_L1EM20VH',
#'e24_vloose_L1EM20VH',
#'e5_loose_idperf',
#'e5_lhloose_idperf',
#'e5_tight_idperf',
#'e5_lhtight_idperf',
'e24_medium_idperf_L1EM20VH',
'e24_lhmedium_idperf_L1EM20VH'
]
parser = ArgumentParser()
parser.add_argument('--inFolderList',
nargs='+',
required=True,
help="Input container to retrieve data")
parser.add_argument('--signalDS',
action='store_true',
help="Whether the dataset contains TPNtuple")
parser.add_argument('--outfile',
action='store',
default="mergedOutput.root",
help="Name of the output file")
parser.add_argument('--triggerList',
nargs='+',
default=defaultTrigList,
help="Trigger list to keep on the filtered file.")
__all__ = ['createDataParser']
from RingerCore import ArgumentParser, get_attributes, BooleanStr, NotSet
from TuningTools.dataframe.EnumCollection import RingerOperation
from TuningTools.parsers.BaseModuleParser import coreFrameworkParser, dataframeParser
###############################################################################
# Create data related objects
###############################################################################
createDataParser = ArgumentParser(
add_help=False,
description='Create TuningTool data from PhysVal.',
parents=[coreFrameworkParser, dataframeParser])
from TuningTools.dataframe.EnumCollection import Reference, Detector, PileupReference
mainCreateData = createDataParser.add_argument_group("Required arguments", "")
mainCreateData.add_argument(
'-s',
'--sgnInputFiles',
action='store',
metavar='SignalInputFiles',
required=True,
nargs='+',
help="The signal files that will be used to tune the discriminators")
mainCreateData.add_argument(
'-b',
'--bkgInputFiles',
action='store',
metavar='BackgroundInputFiles',
required=True,
nargs='+',
def DataCurationParser(tuningDataRequired=True, extraDescr=''):
dataCurationParser = ArgumentParser(
add_help=False,
description='Apply transformation and cross-validation to tuning data.'
)
tuningDataArgs = dataCurationParser.add_argument_group(
"Data curation required arguments", extraDescr)
tuningDataArgs.add_argument(
'-d',
'--data',
action='store',
nargs='+',
metavar='data',
required=tuningDataRequired,
default=NotSet,
help="The data file that will be used to tune the discriminators")
tuningOptArgs = dataCurationParser.add_argument_group(
"Data curation optional arguments", extraDescr)
tuningOptArgs.add_argument(
'-op',
'--operation',
default=None,
type=RingerOperation,
required=False,
help="""The Ringer operation determining in each Trigger
level or what is the offline operation point reference."""
)
tuningOptArgs.add_argument(
'-r',
'--refFile',
default=None,
help="""The Ringer references to set the discriminator point.""")
tuningOptArgs.add_argument(
'--saveMatPP',
action='store',
default=NotSet,
type=BooleanStr,
help="""Whether to save pre-processings to a matlab file.""")
#tuningOptArgs.add_argument('--savePPFile', action='store', default = NotSet, type=BooleanStr,
# help = """Whether to save pre-processings to pre-processed version of the tuning data.""")
crossValArgs = dataCurationParser.add_argument_group(
"Cross-validation configuration", extraDescr)
# TODO Make these options mutually exclusive
crossValArgs.add_argument(
'-x',
'--crossFile',
action='store',
default=NotSet,
help="""The cross-validation file path, pointing to a file
created with the create tuning job files""")
crossValArgs.add_argument(
'-xc',
'--clusterFile',
action='store',
default=NotSet,
help="""The subset cross-validation file path, pointing to a file
created with the create tuning job files""")
crossValArgs.add_argument(
'-xm',
'--crossValidMethod',
type=CrossValidMethod,
default=NotSet,
help="""Which cross validation method to use when no cross-validation
object was specified.""")
crossValArgs.add_argument(
'-xs',
'--crossValidShuffle',
type=BooleanStr,
default=NotSet,
help="""Which cross validation method to use when no cross-validation
object was specified.""")
ppArgs = dataCurationParser.add_argument_group(
"Pre-processing configuration", extraDescr)
ppArgs.add_argument(
'-pp',
'--ppFile',
default=NotSet,
help=""" The file containing the pre-processing collection to apply. """
)
return dataCurationParser
################################################################################
# Create tuningJob file related objects
################################################################################
class JobFileTypeCreation(EnumStringification):
"""
The possible file creation options
"""
all = 0,
ConfigFiles = 1,
CrossValidFile = 2,
ppFile = 3
tuningJobFileParser = ArgumentParser(
add_help=False, description='Create files used by TuningJob.')
tuningJobFileParser.add_argument(
'fileType',
choices=get_attributes(JobFileTypeCreation,
onlyVars=True,
getProtected=False),
nargs='+',
help="""Which kind of files to create. You can choose one
or more of the available choices, just don't use all with
the other available choices.""")
tuningJobFileParser.add_argument('--compress',
type=BooleanStr,
help="Whether to compress files or not.")
################################################################################
jobConfig = tuningJobFileParser.add_argument_group(
"JobConfig Files Creation Options", """Change configuration for
#!/usr/bin/env python
from RingerCore import ( csvStr2List, str_to_class, NotSet, BooleanStr, WriteMethod
, get_attributes, expandFolders, Logger, getFilters, select
, appendToFileName, ensureExtension, progressbar, LoggingLevel
, printArgs, conditionalOption, emptyArgumentsPrintHelp, ArgumentParser)
from TuningTools.parsers import loggerParser
from TuningTools import GridJobFilter
mainParser = ArgumentParser( add_help = False)
mainGroup = mainParser.add_argument_group( "Required arguments", "")
mainGroup.add_argument('--inDS','-i', action='store',
required = True, dest = 'grid_inDS',
help = "The input Dataset ID (DID)")
mainGroup.add_argument('--outDS','-o', action='store',
required = True, dest = 'grid_outDS',
help = "The output Dataset ID (DID)")
parser = ArgumentParser(description = 'Attach the input container files to the output dataset, creating it if necessary.',
parents = [mainParser, loggerParser],
conflict_handler = 'resolve')
parser.make_adjustments()
emptyArgumentsPrintHelp(parser)
## Retrieve parser args:
args = parser.parse_args()
mainLogger = Logger.getModuleLogger(__name__, args.output_level)
def TuningJobFileParser():
tuningJobFileParser = ArgumentParser(
add_help=False, description='Create files used by TuningJob.')
tuningJobFileParser.add_argument(
'fileType',
choices=get_attributes(JobFileTypeCreation,
onlyVars=True,
getProtected=False),
nargs='+',
help="""Which kind of files to create. You can choose one
or more of the available choices, just don't use all with
the other available choices.""")
tuningJobFileParser.add_argument('--compress',
type=BooleanStr,
help="Whether to compress files or not.")
################################################################################
jobConfig = tuningJobFileParser.add_argument_group(
"JobConfig Files Creation Options", """Change configuration for
job config files creation.""")
jobConfig.add_argument('-oJConf',
'--jobConfiFilesOutputFolder',
default=NotSet,
help="The job config files output folder.")
jobConfig.add_argument('--neuronBounds',
nargs='+',
type=int,
default=NotSet,
help="""
Input a sequential bounded list to be used as the
neuron job range, the arguments should have the
same format from the seq unix command or as the
Matlab format. If not specified, the range will
start from 1. I.e. 5 2 9 leads to [5 7 9] and 50
leads to 1:50
""")
jobConfig.add_argument('--sortBounds',
nargs='+',
type=int,
default=NotSet,
help="""
Input a sequential bounded list using seq format to
be used as the sort job range, but the last bound
will be opened just as happens when using python
range function. If not specified, the range will
start from 0. I.e. 5 2 9 leads to [5 7] and 50 leads
to range(50)
""")
jobConfig.add_argument(
'--nInits',
nargs='?',
type=int,
default=NotSet,
help="The number of initilizations of the discriminator.")
jobConfig.add_argument('--nNeuronsPerJob',
type=int,
default=NotSet,
help="The number of hidden layer neurons per job.")
jobConfig.add_argument('--nSortsPerJob',
type=int,
default=NotSet,
help="The number of sorts per job.")
jobConfig.add_argument('--nInitsPerJob',
type=int,
default=NotSet,
help="The number of initializations per job.")
################################################################################
crossConfig = tuningJobFileParser.add_argument_group(
"CrossValid File Creation Options",
"""Change configuration for CrossValid
file creation.""")
crossConfig.add_argument('-outCross',
'--crossValidOutputFile',
default='crossValid',
help="The cross validation output file.")
crossConfig.add_argument('-m',
'--method',
default=NotSet,
type=CrossValidMethod,
help="The Cross-Validation method.")
crossConfig.add_argument(
'-ns',
'--nSorts',
type=int,
default=NotSet,
help="""The number of sort used by cross validation
configuration.""")
crossConfig.add_argument(
'-nb',
'--nBoxes',
type=int,
default=NotSet,
help="""The number of boxes used by cross validation
configuration.""")
crossConfig.add_argument('-ntr',
'--nTrain',
type=int,
default=NotSet,
help="""The number of train boxes used by cross
validation.""")
crossConfig.add_argument('-nval',
'--nValid',
type=int,
default=NotSet,
help="""The number of valid boxes used by cross
validation.""")
crossConfig.add_argument('-ntst',
'--nTest',
type=int,
default=NotSet,
help="""The number of test boxes used by cross
validation.""")
crossConfig.add_argument(
'-seed',
type=int,
default=NotSet,
help="The seed value for generating CrossValid object.")
################################################################################
ppConfig = tuningJobFileParser.add_argument_group(
"PreProc File Creation Options",
"""Change configuration for pre-processing
file creation. These options will only
be taken into account if job fileType is
set to "ppFile" or "all".""")
ppConfig.add_argument('-outPP',
'--preProcOutputFile',
default='ppFile',
help="The pre-processing validation output file")
ppConfig.add_argument('-ppCol',
type=str,
default='[[Norm1()]]',
help="""The pre-processing collection to apply. The
string will be parsed by python and created using
the available pre-processings on
TuningTools.PreProc.py file.
This string can have classes from the PreProc
module initialized with determined values. E.g.:
-ppCol "[[[Norm1(),MapStd()],[RingerRp(2.,1.3)],[MapStd]],[[Norm1(),MapStd],[Norm1],[MapStd]],[[Norm1,MapStd],[Norm1({'level' : 'VERBOSE'})],[MapStd({'d' : {'level' : 'VERBOSE'}})]]]"
The explicit usage of () or not will make no
difference resulting in the class instance
initialization.
Also, a special syntax need to be used when
passing keyword arguments as specified in:
MapStd({'level' : 'VERBOSE'}) (equivalent in python) => MapStd( level = VERBOSE )
MapStd({'d' : {'level' : 'VERBOSE'}}) => MapStd( d = { level : VERBOSE } )
""")
ppConfig.add_argument('-pp_ns',
'--pp_nSorts',
default=NotSet,
type=int,
help="""The number of sort used by cross validation
configuration. Import from nSorts if not set."""
)
ppConfig.add_argument('-pp_nEt',
'--pp_nEtBins',
default=NotSet,
type=int,
help="""The number of et bins.""")
ppConfig.add_argument('-pp_nEta',
'--pp_nEtaBins',
default=NotSet,
type=int,
help="""The number of eta bins.""")
################################################################################
return tuningJobFileParser
__all__ = [
'RetrieveCoreFramework', 'coreFrameworkParser', 'RetrieveDataFramework',
'dataframeParser'
]
from RingerCore import argparse, ArgumentParser, NotSet
from TuningTools.coreDef import coreConf, AvailableTuningToolCores, dataframeConf
class RetrieveCoreFramework(argparse.Action):
def __call__(self, parser, namespace, value, option_string=None):
coreConf.set(value)
coreFrameworkParser = ArgumentParser(add_help=False)
coreFrameworkGroup = coreFrameworkParser.add_argument_group(
"TuningTools CORE configuration", "")
coreFrameworkGroup.add_argument( '-core', '--core-framework',
type = AvailableTuningToolCores, action = RetrieveCoreFramework,
help = """ Specify which core framework should be used in the job.""" \
+ ( " Current default is: " + AvailableTuningToolCores.tostring( coreConf.default() ) ))
if not hasattr(argparse.Namespace, 'core_framework'):
# Decorate Namespace with the TuningTools core properties.
# We do this on the original class to simplify usage, as there will be
# no need to specify a different namespace for parsing the arguments.
def _getCoreFramework(self):
if coreConf: return coreConf()
else: return None
def _setCoreFramework(self, val):
#!/usr/bin/env python
from RingerCore import emptyArgumentsPrintHelp, ArgumentParser
from TuningTools.parsers import createDataParser, loggerParser, devParser
parser = ArgumentParser(description='Create TuningTool data from PhysVal.',
parents=[createDataParser, loggerParser, devParser])
parser.make_adjustments()
emptyArgumentsPrintHelp(parser)
# Retrieve parser args:
args = parser.parse_args()
# Treat special argument
if len(args.reference) > 2:
raise ValueError("--reference set to multiple values: %r", args.reference)
if len(args.reference) is 1:
args.reference.append(args.reference[0])
from RingerCore import Logger, LoggingLevel, printArgs, NotSet
logger = Logger.getModuleLogger(__name__, args.output_level)
from TuningTools import RingerOperation
if RingerOperation.retrieve(args.operation) < 0 and not args.treePath:
ValueError(
"If operation is not set to Offline, it is needed to set the TreePath manually."
)
printArgs(args, logger.debug)
crossVal = NotSet
defaultTrigList = [
'e24_medium_L1EM18VH',
'e24_lhmedium_L1EM18VH',
'e24_tight_L1EM20VH',
#'e24_vloose_L1EM20VH',
#'e5_loose_idperf',
#'e5_lhloose_idperf',
#'e5_tight_idperf',
#'e5_lhtight_idperf',
'e24_medium_idperf_L1EM20VH',
'e24_lhmedium_idperf_L1EM20VH'
]
parser = ArgumentParser()
parser.add_argument('--inDS',
action='store',
required=True,
help="Input container to retrieve data")
parser.add_argument('--outFolder', action='store', default="dumpOutput")
parser.add_argument('--triggerList', nargs='+', default=defaultTrigList)
parser.add_argument('--numberOfSamplesPerPackage', type=int, default=50)
args = parser.parse_args()
mainLogger = Logger.getModuleLogger(__name__, LoggingLevel.INFO)
if os.path.exists('dq2_ls.txt'):
os.system('rm dq2_ls.txt')
if args.inDS[-1] != '/':