GridOutput, GridOutputCollection, SecondaryDataset, SecondaryDatasetCollection, emptyArgumentsPrintHelp) crossValStatsJobParser.suppress_arguments(doMatlab=True, test=False, doCompress=False) crossValStatsJobParser.delete_arguments('binFilters', 'discrFiles', 'refFile', 'outputs') ioGridParser.delete_arguments('grid__inDS', 'grid__reusableSecondary', 'grid__nFiles', 'grid__antiMatch') ioGridParser.suppress_arguments(grid_CSV__outputs=GridOutputCollection(), grid__match=False, grid__writeInputToTxt='IN:input.csv', grid__allowTaskDuplication=True, grid__nFiles=None, grid__nFilesPerJob=1, grid__nJobs=1, grid__maxNFilesPerJob=1, grid__forceStaged=True, grid__forceStagedSecondary=True, grid__crossSite=1, grid__secondaryDS=SecondaryDatasetCollection()) ## Create our paser # Add base parser options (this is just a wrapper so that we can have this as # the first options to show, as they are important options) parentParser = ArgumentParser(add_help=False) parentReqParser = parentParser.add_argument_group("required arguments", '') parentReqParser.add_argument('-d', '--discrFilesDS', required=True, metavar='DATA',
expandFolders, Logger, getFilters, select, appendToFileName, ensureExtension, progressbar, LoggingLevel, printArgs, conditionalOption, GridOutputCollection, GridOutput, emptyArgumentsPrintHelp ) from TuningTools.parsers import (ArgumentParser, ioGridParser, loggerParser, TuningToolGridNamespace ) from TuningTools import GridJobFilter ioGridParser.suppress_arguments( grid_CSV__outputs = GridOutputCollection(GridOutput('td','merge.TunedDiscr.tgz')) , grid__forceStaged = True , grid__forceStagedSecondary = True , grid__mergeOutput = True , grid__allowTaskDuplication = True , grid__nFiles = None , grid__nFilesPerJob = None , grid__maxNFilesPerJob = None , grid__match = None , grid__antiMatch = None ) parser = ArgumentParser(description = 'Merge files into unique file on the GRID.', parents = [ioGridParser, loggerParser], conflict_handler = 'resolve') parser.make_adjustments() emptyArgumentsPrintHelp(parser) args = parser.parse_args( namespace = TuningToolGridNamespace('prun') ) mainLogger = Logger.getModuleLogger( __name__, args.output_level )
# Suppress/delete the following options in the main-job parser: tuningJobParser.delete_arguments('outputFileBase', 'data', 'crossFile', 'confFileList', 'neuronBounds', 'sortBounds', 'initBounds', 'ppFile', 'refFile', 'outputDir', 'crossValidShuffle', 'expert_networks') tuningJobParser.suppress_arguments(compress='False') # Suppress/delete the following options in the grid parser: ioGridParser.delete_arguments('grid__inDS', 'grid__nJobs') ioGridParser.suppress_arguments( grid__mergeOutput=False # We disabled it since late 2017, where GRID # added a limited to the total memory and processing time for merging jobs. , grid_CSV__outputs=GridOutputCollection( [GridOutput('td', 'tunedDiscr*.pic')]), grid__nFiles=None, grid__nFilesPerJob=1, grid__forceStaged=True, grid__forceStagedSecondary=True, grid__nCore=None) ## Create dedicated arguments for the panda job: # WARNING: Groups can be used to replace conflicting options -o/-d and so on parentReqParser.add_argument( '-d', '--dataDS', required=True, metavar='DATA', action='store', nargs='+',
parentParser = ArgumentParser(add_help = False) parentReqParser = parentParser.add_argument_group("required arguments", '') if clusterManagerConf() is ClusterManager.Panda: # Suppress/delete the following options in the main-job parser: tuningJobParser.delete_arguments( 'outputFileBase', 'data', 'crossFile', 'confFileList' , 'neuronBounds', 'sortBounds', 'initBounds', 'ppFile' , 'ppFile', 'outputDir' ) tuningJobParser.suppress_arguments(compress = 'False') # Suppress/delete the following options in the grid parser: ioGridParser.delete_arguments('grid__inDS', 'grid__nJobs') ioGridParser.suppress_arguments( grid__mergeOutput = True , grid_CSV__outputs = GridOutputCollection( [ GridOutput('td','tunedDiscr*.pic') ] ) , grid__nFiles = None , grid__nFilesPerJob = 1 , grid__forceStaged = True , grid__forceStagedSecondary = True , grid__nCore = None ) ## Create dedicated arguments for the panda job: # WARNING: Groups can be used to replace conflicting options -o/-d and so on parentReqParser.add_argument('-d','--dataDS', required = True, metavar='DATA', action='store', nargs='+', help = "The dataset with the data for discriminator tuning.") parentReqParser.add_argument('-r','--refDS', required = False, metavar='REF', action='store', nargs='+', default = None, help = "The reference values used to tuning all discriminators.") parentLoopParser = parentParser.add_argument_group("Looping configuration", '') parentLoopParser.add_argument('-c','--configFileDS', metavar='Config_DS',