GridOutput, GridOutputCollection, SecondaryDataset,
SecondaryDatasetCollection, emptyArgumentsPrintHelp)
crossValStatsJobParser.suppress_arguments(doMatlab=True,
test=False,
doCompress=False)
crossValStatsJobParser.delete_arguments('binFilters', 'discrFiles', 'refFile',
'outputs')
ioGridParser.delete_arguments('grid__inDS', 'grid__reusableSecondary',
'grid__nFiles', 'grid__antiMatch')
ioGridParser.suppress_arguments(grid_CSV__outputs=GridOutputCollection(),
grid__match=False,
grid__writeInputToTxt='IN:input.csv',
grid__allowTaskDuplication=True,
grid__nFiles=None,
grid__nFilesPerJob=1,
grid__nJobs=1,
grid__maxNFilesPerJob=1,
grid__forceStaged=True,
grid__forceStagedSecondary=True,
grid__crossSite=1,
grid__secondaryDS=SecondaryDatasetCollection())
## Create our paser
# Add base parser options (this is just a wrapper so that we can have this as
# the first options to show, as they are important options)
parentParser = ArgumentParser(add_help=False)
parentReqParser = parentParser.add_argument_group("required arguments", '')
parentReqParser.add_argument('-d',
'--discrFilesDS',
required=True,
metavar='DATA',
expandFolders, Logger, getFilters, select,
appendToFileName, ensureExtension, progressbar, LoggingLevel,
printArgs, conditionalOption, GridOutputCollection, GridOutput,
emptyArgumentsPrintHelp )
from TuningTools.parsers import (ArgumentParser, ioGridParser, loggerParser,
TuningToolGridNamespace )
from TuningTools import GridJobFilter
ioGridParser.suppress_arguments( grid_CSV__outputs = GridOutputCollection(GridOutput('td','merge.TunedDiscr.tgz'))
, grid__forceStaged = True
, grid__forceStagedSecondary = True
, grid__mergeOutput = True
, grid__allowTaskDuplication = True
, grid__nFiles = None
, grid__nFilesPerJob = None
, grid__maxNFilesPerJob = None
, grid__match = None
, grid__antiMatch = None
)
parser = ArgumentParser(description = 'Merge files into unique file on the GRID.',
parents = [ioGridParser, loggerParser],
conflict_handler = 'resolve')
parser.make_adjustments()
emptyArgumentsPrintHelp(parser)
args = parser.parse_args( namespace = TuningToolGridNamespace('prun') )
mainLogger = Logger.getModuleLogger( __name__, args.output_level )
# Suppress/delete the following options in the main-job parser:
tuningJobParser.delete_arguments('outputFileBase', 'data', 'crossFile',
'confFileList', 'neuronBounds',
'sortBounds', 'initBounds', 'ppFile',
'refFile', 'outputDir',
'crossValidShuffle', 'expert_networks')
tuningJobParser.suppress_arguments(compress='False')
# Suppress/delete the following options in the grid parser:
ioGridParser.delete_arguments('grid__inDS', 'grid__nJobs')
ioGridParser.suppress_arguments(
grid__mergeOutput=False # We disabled it since late 2017, where GRID
# added a limited to the total memory and processing time for merging jobs.
,
grid_CSV__outputs=GridOutputCollection(
[GridOutput('td', 'tunedDiscr*.pic')]),
grid__nFiles=None,
grid__nFilesPerJob=1,
grid__forceStaged=True,
grid__forceStagedSecondary=True,
grid__nCore=None)
## Create dedicated arguments for the panda job:
# WARNING: Groups can be used to replace conflicting options -o/-d and so on
parentReqParser.add_argument(
'-d',
'--dataDS',
required=True,
metavar='DATA',
action='store',
nargs='+',
parentParser = ArgumentParser(add_help = False)
parentReqParser = parentParser.add_argument_group("required arguments", '')
if clusterManagerConf() is ClusterManager.Panda:
# Suppress/delete the following options in the main-job parser:
tuningJobParser.delete_arguments( 'outputFileBase', 'data', 'crossFile', 'confFileList'
, 'neuronBounds', 'sortBounds', 'initBounds', 'ppFile'
, 'ppFile', 'outputDir' )
tuningJobParser.suppress_arguments(compress = 'False')
# Suppress/delete the following options in the grid parser:
ioGridParser.delete_arguments('grid__inDS', 'grid__nJobs')
ioGridParser.suppress_arguments( grid__mergeOutput = True
, grid_CSV__outputs = GridOutputCollection( [ GridOutput('td','tunedDiscr*.pic') ] )
, grid__nFiles = None
, grid__nFilesPerJob = 1
, grid__forceStaged = True
, grid__forceStagedSecondary = True
, grid__nCore = None
)
## Create dedicated arguments for the panda job:
# WARNING: Groups can be used to replace conflicting options -o/-d and so on
parentReqParser.add_argument('-d','--dataDS', required = True, metavar='DATA',
action='store', nargs='+',
help = "The dataset with the data for discriminator tuning.")
parentReqParser.add_argument('-r','--refDS', required = False, metavar='REF',
action='store', nargs='+', default = None,
help = "The reference values used to tuning all discriminators.")
parentLoopParser = parentParser.add_argument_group("Looping configuration", '')
parentLoopParser.add_argument('-c','--configFileDS', metavar='Config_DS',