for xname in paths:
if jobID in xname and checkExtension(xname, '.root'):
oDict[jobID]['root'] = xname
if jobID in xname and checkExtension(
xname, '.pic|.pic.gz|.pic.tgz'):
oDict[jobID]['pic'] = xname
return oDict
from RingerCore import csvStr2List, str_to_class, NotSet, BooleanStr, emptyArgumentsPrintHelp
from TuningTools.parsers import ArgumentParser, loggerParser, crossValStatsMonParser, LoggerNamespace
from TuningTools import GridJobFilter
parser = ArgumentParser(
description=
'Retrieve performance information from the Cross-Validation method.',
parents=[crossValStatsMonParser, loggerParser])
parser.make_adjustments()
emptyArgumentsPrintHelp(parser)
# Retrieve parser args:
args = parser.parse_args(namespace=LoggerNamespace())
from RingerCore import Logger, LoggingLevel, printArgs
logger = Logger.getModuleLogger(__name__, args.output_level)
printArgs(args, logger.debug)
#Find files
from RingerCore import expandFolders, ensureExtension, keyboard
grid__match=False,
grid__writeInputToTxt='IN:input.csv',
grid__allowTaskDuplication=True,
grid__nFiles=None,
grid__nFilesPerJob=1,
grid__nJobs=1,
grid__maxNFilesPerJob=1,
grid__forceStaged=True,
grid__forceStagedSecondary=True,
grid__crossSite=1,
grid__secondaryDS=SecondaryDatasetCollection())
## Create our paser
# Add base parser options (this is just a wrapper so that we can have this as
# the first options to show, as they are important options)
parentParser = ArgumentParser(add_help=False)
parentReqParser = parentParser.add_argument_group("required arguments", '')
parentReqParser.add_argument('-d',
'--discrFilesDS',
required=True,
metavar='DATA',
action='store',
dest='grid__inDS',
help="The dataset with the tuned discriminators.")
parentOptParser = parentParser.add_argument_group("optional arguments", '')
parentOptParser.add_argument(
'-r',
'--refFileDS',
metavar='REF_FILE',
required=False,
default=None,
from TuningTools import GridJobFilter
ioGridParser.suppress_arguments( grid_CSV__outputs = GridOutputCollection(GridOutput('td','merge.TunedDiscr.tgz'))
, grid__forceStaged = True
, grid__forceStagedSecondary = True
, grid__mergeOutput = True
, grid__allowTaskDuplication = True
, grid__nFiles = None
, grid__nFilesPerJob = None
, grid__maxNFilesPerJob = None
, grid__match = None
, grid__antiMatch = None
)
parser = ArgumentParser(description = 'Merge files into unique file on the GRID.',
parents = [ioGridParser, loggerParser],
conflict_handler = 'resolve')
parser.make_adjustments()
emptyArgumentsPrintHelp(parser)
args = parser.parse_args( namespace = TuningToolGridNamespace('prun') )
mainLogger = Logger.getModuleLogger( __name__, args.output_level )
printArgs( args, mainLogger.debug )
# Set primary dataset number of files:
import os.path
user_scope = 'user.%s' % os.path.expandvars('$USER')
try:
# The input files can be send via a text file to avoid very large command lines?
else:
f = preInitLogger.info
s = ''
f('%susing clean %s: %s', s, moduleType, configureObj.tag)
printVersion(ProjectGit, moduleType='project')
printVersion(RingerCoreGit)
printVersion(TuningToolsGit)
TuningToolsGit.ensure_clean()
RingerCoreGit.ensure_clean()
# This parser is dedicated to have the specific options which should be added
# to the parent parsers for this job
parentParser = ArgumentParser(add_help=False)
parentReqParser = parentParser.add_argument_group("required arguments", '')
if clusterManagerConf() is ClusterManager.Panda:
# Suppress/delete the following options in the main-job parser:
tuningJobParser.delete_arguments('outputFileBase', 'data', 'crossFile',
'confFileList', 'neuronBounds',
'sortBounds', 'initBounds', 'ppFile',
'refFile', 'outputDir',
'crossValidShuffle', 'expert_networks')
tuningJobParser.suppress_arguments(compress='False')
# Suppress/delete the following options in the grid parser:
ioGridParser.delete_arguments('grid__inDS', 'grid__nJobs')
ioGridParser.suppress_arguments(
grid__mergeOutput=False # We disabled it since late 2017, where GRID
#!/usr/bin/env python
from TuningTools.parsers import ArgumentParser, loggerParser, tuningJobParser
from RingerCore import emptyArgumentsPrintHelp
parser = ArgumentParser(description='Tune discriminators using input data.',
parents=[tuningJobParser, loggerParser])
parser.make_adjustments()
emptyArgumentsPrintHelp(parser)
# Retrieve parser args:
args = parser.parse_args()
## Treating special args:
# Configuration
conf_kw = {}
if args.neuronBounds is not None:
conf_kw['neuronBoundsCol'] = args.neuronBounds
if args.sortBounds is not None: conf_kw['sortBoundsCol'] = args.sortBounds
if args.initBounds is not None: conf_kw['initBoundsCol'] = args.initBounds
if args.confFileList is not None: conf_kw['confFileList'] = args.confFileList
# Binning
from RingerCore import printArgs, NotSet, Logger, LoggingLevel
if not (args.et_bins is NotSet) and len(args.et_bins) == 1:
args.et_bins = args.et_bins[0]
if not (args.eta_bins is NotSet) and len(args.eta_bins) == 1:
args.eta_bins = args.eta_bins[0]
logger = Logger.getModuleLogger(__name__, args.output_level)
printArgs(args, logger.debug)
#!/usr/bin/env python
from RingerCore import (csvStr2List, emptyArgumentsPrintHelp, expandFolders,
Logger, progressbar, LoggingLevel)
from TuningTools.parsers import ArgumentParser, loggerParser, LoggerNamespace
mainParser = ArgumentParser(description='Merge files into unique file.',
add_help=False)
mainMergeParser = mainParser.add_argument_group("Required arguments", "")
mainMergeParser.add_argument(
'-i',
'--inputFiles',
action='store',
metavar='InputFiles',
required=True,
nargs='+',
help="The input files that will be used to generate a matlab file")
mainMergeParser.add_argument(
'-c',
'--change-output-folder',
action='store',
required=False,
default=None,
help=
"Change output folder to be in the specified path instead using the same input dir as input file."
)
mainLogger = Logger.getModuleLogger(__name__)
parser = ArgumentParser(description='Save files on matlab format.',
parents=[mainParser, loggerParser],
conflict_handler='resolve')
#!/usr/bin/env python
from TuningTools.parsers import ArgumentParser, argparse, createDataParser, TuningToolGridNamespace
from RingerCore import emptyArgumentsPrintHelp, ioGridParser, loggerParser
## Create our paser
# Add base parser options (this is just a wrapper so that we can have this as
# the first options to show, as they are important options)
parentParser = ArgumentParser( add_help = False )
parentParser.add_argument('-s','--inDS-SGN', action='store',
metavar='inDS_SGN', required = True, nargs='+', dest = 'grid_inDS',
help = "The signal files that will be used to tune the discriminators")
parentParser.add_argument('-b','--inDS-BKG', action='store',
metavar='inDS_BKG', required = True, nargs='+',
help = "The background files that will be used to tune the discriminators")
## The main parser
parser = ArgumentParser(description = 'Generate input file for TuningTool on GRID',
parents = [createDataParser, parentParser, ioGridParser, loggerParser],
conflict_handler = 'resolve')
## Change parent options
# Hide sgnInputFiles
parser.add_argument('--sgnInputFiles', action='store_const',
metavar='SignalInputFiles', required = False, const = '', default = '',
help = argparse.SUPPRESS)
# Hide bkgInputFiles
parser.add_argument('--bkgInputFiles', action='store_const',
metavar='BackgroundInputFiles', required = False, const = '', default = '',
help = argparse.SUPPRESS)
# Hide output
parser.add_argument('--output', action='store_const',
#!/usr/bin/env python
import time
from TuningTools.parsers import argparse, ArgumentParser
from RingerCore import emptyArgumentsPrintHelp
parser = ArgumentParser(description='Retry failed jobs on bsub')
parser.add_argument('-l',
'--logFile',
action='store',
required=True,
help="The log file containing the commands submitted.")
parser.add_argument(
'-perm-op',
'--permanentOutputPlace',
action='store',
required=True,
help=
"The permanent output place where the tuned classifiers should be found.")
parser.add_argument('--queue', default=None, help="Change queue if defined.")
parser.add_argument('--pause',
default=5,
type=int,
help="Time to wait between each submission.")
parser.add_argument('--checkForMissingJobs',
action='store_true',
help="Check if there are any missing job, and submit it.")
parser.add_argument(
'--overrideOutputPlace',
#!/usr/bin/env python
from TuningTools.parsers import ArgumentParser, loggerParser, tuningExpertParser
from RingerCore import emptyArgumentsPrintHelp
parser = ArgumentParser(
description=
'Tune expert discriminator based on calorimeter and trackin data.',
parents=[tuningExpertParser, loggerParser])
parser.make_adjustments()
emptyArgumentsPrintHelp(parser)
# Retrieve parser args:
args = parser.parse_args()
# FIXME: The core configuration is not being automaticaly set to keras
## Treating special args:
# Configuration
conf_kw = {}
if args.neuronBounds is not None:
conf_kw['neuronBoundsCol'] = args.neuronBounds
if args.sortBounds is not None: conf_kw['sortBoundsCol'] = args.sortBounds
if args.initBounds is not None: conf_kw['initBoundsCol'] = args.initBounds
if args.confFileList is not None: conf_kw['confFileList'] = args.confFileList
# Binning
from RingerCore import printArgs, NotSet, Logger, LoggingLevel
if not (args.et_bins is NotSet) and len(args.et_bins) == 1:
args.et_bins = args.et_bins[0]
if not (args.eta_bins is NotSet) and len(args.eta_bins) == 1:
#!/usr/bin/env python
from TuningTools.parsers import (ArgumentParser, outGridParser, loggerParser,
tuningJobFileParser, TuningToolGridNamespace)
from RingerCore import Logger, printArgs, conditionalOption, emptyArgumentsPrintHelp
outGridParser.delete_arguments('outputs')
outGridParser.suppress_arguments(grid__nJobs=1)
tuningJobFileParser.suppress_arguments(compress=0)
## The main parser
parser = ArgumentParser(
description='Generate input file for TuningTool on GRID',
parents=[tuningJobFileParser, outGridParser, loggerParser],
conflict_handler='resolve')
parser.make_adjustments()
## Now the job really starts
emptyArgumentsPrintHelp(parser)
args = parser.parse_args(namespace=TuningToolGridNamespace('prun'))
mainLogger = Logger.getModuleLogger(__name__, args.output_level)
# Retrieve outputs containers
outputs = []
if any(val in args.fileType for val in ("all", "ConfigFiles")):
outputs.append('Config:job.*')
if any(val in args.fileType for val in ("all", "CrossValidFile")):
outputs.append('CrossValid:crossValid*')
if any(val in args.fileType for val in ("all", "ppFile")):
#!/usr/bin/env python
from TuningTools.parsers import argparse, ArgumentParser
from RingerCore import emptyArgumentsPrintHelp
parser = ArgumentParser(description='Run training job on grid')
parser.add_argument('-d',
'--data',
action='store',
required=True,
help="The file containing data for discriminator tunning")
parser.add_argument('-o',
'--output',
action='store',
required=True,
help="The output base string for the discriminator file.")
parser.add_argument('-op',
'--outputPlace',
action='store',
required=True,
help="The output place to a lxplus tmp.")
parser.add_argument(
'-i',
'--inputConfig',
metavar='InputFolder',
help="Folder to loop upon files to retrieve configuration.")
parser.add_argument('--ppFile', default=None, help="Pre-processing file.")
parser.add_argument('--crossValidFile',
default=None,
help="Cross-validation file.")
#!/usr/bin/env python
from RingerCore import (csvStr2List, str_to_class, NotSet, BooleanStr,
WriteMethod, expandFolders, Logger, getFilters, select,
appendToFileName, ensureExtension, progressbar,
LoggingLevel, printArgs, conditionalOption,
emptyArgumentsPrintHelp)
from TuningTools.parsers import argparse, ArgumentParser, ioGridParser, loggerParser, \
TuningToolGridNamespace
parser = ArgumentParser(
description='Extract ntuple files into unique file on the GRID.',
parents=[ioGridParser, loggerParser],
conflict_handler='resolve')
parser.add_argument('-t',
'--trigger',
action='store',
required=True,
help="Trigger required to filter.")
parser.add_argument('--basepath',
action='store',
required=False,
default='HLT/Egamma/Expert',
help="the tree location inside of the file.")
parser.add_argument('--treename',
action='store',
required=False,
default='trigger',
help="the tree ntuple name")
#!/usr/bin/env python
from TuningTools.parsers import ArgumentParser, loggerParser, LoggerNamespace
from RingerCore import emptyArgumentsPrintHelp
parser = ArgumentParser(description="""Change data memory representation
without changing its dimensions.""",
parents=[loggerParser])
parser.add_argument('inputs',
action='store',
metavar='INPUT',
nargs='+',
help="Files to change representation")
emptyArgumentsPrintHelp(parser)
args = parser.parse_args(namespace=LoggerNamespace())
from RingerCore import Logger, LoggingLevel, save, load, expandFolders, traverse
import numpy as np
from TuningTools.coreDef import npCurrent
npCurrent.level = args.output_level
logger = Logger.getModuleLogger(__name__, args.output_level)
files = expandFolders(args.inputs) # FIXME *.npz
from zipfile import BadZipfile
for f in files:
logger.info("Changing representation of file '%s'...", f)
try:
data = dict(load(f))
