def _mk_registration_parser(p: ArgParser) -> ArgParser:
g = p.add_argument_group("General registration options",
"....")
# p = ArgParser(add_help=False)
g.add_argument("--input-space", dest="input_space",
type=lambda x: InputSpace[x], # type: ignore # mypy/issues/741
default=InputSpace.native,
# choices=[x for x, _ in InputSpace.__members__.items()],
help="Option to specify space of input-files. Can be native (default), lsq6, lsq12. "
"Native means that there is no prior formal alignment between the input files "
"yet. lsq6 means that the input files have been aligned using translations "
"and rotations; the code will continue with a 12 parameter alignment. lsq12 "
"means that the input files are fully linearly aligned. Only non-linear "
"registrations are performed. [Default=%(default)s]")
g.add_argument("--resolution", dest="resolution",
type=float, default=None,
help="Specify the resolution at which you want the registration to be run. "
"If not specified, the resolution of the target of your pipeline will "
"be used. [Default=%(default)s]")
g.add_argument("--subject-matter", dest="subject_matter",
type=str, default=None,
help="Specify the subject matter for the pipeline. This will set the parameters "
"for multiple programs based on the overall size of the subject matter. Instead "
"of using the resolution of the files. Currently supported option is: \"mousebrain\" "
"[Default=%(default)s]")
return p # g?
def _mk_common_space_parser(parser : ArgParser):
group = parser.add_argument_group("Common space options", "Options for registration/resampling to common (db) space.")
group.add_argument("--common-space-model", dest="common_space_model",
type=str, help="Run MAGeT segmentation on the images.")
group.add_argument("--no-common-space-registration", dest="do_common_space_registration",
default=False, action="store_false", help="Skip registration to common (db) space.")
return parser
def _mk_maget_parser(parser : ArgParser):
group = parser.add_argument_group("MAGeT options", "Options for running MAGeT.")
group.add_argument("--atlas-library", dest="atlas_lib", # can't make required=True since may not be using MAGeT :|
type=str, # TODO: check existence of this dir?
help="Directory of existing atlas/label pairs")
group.add_argument("--pairwise", dest="pairwise",
action="store_true",
help="""If specified, register inputs to each other pairwise. [Default]""")
group.add_argument("--no-pairwise", dest="pairwise",
action="store_false",
help="""If specified, only register inputs to atlases in library.""")
parser.set_defaults(pairwise=True)
group.add_argument("--mask", dest="mask",
action="store_true", default=False,
help="Create a mask for all images prior to handling labels. [Default = %(default)s]")
group.add_argument("--mask-only", dest="mask_only",
action="store_true", default=False,
help="Create a mask for all images only, do not run full algorithm. [Default = %(default)s]")
group.add_argument("--max-templates", dest="max_templates",
default=25, type=int,
help="Maximum number of templates to generate. [Default = %(default)s]")
group.add_argument("--masking-method", dest="mask_method",
default="minctracc", type=str,
help="Specify whether to use minctracc or mincANTS for masking. [Default = %(default)s].")
group.add_argument("--masking-nlin-protocol", dest="masking_nlin_protocol",
# TODO basically copied from nlin parser
type=str, default=None,
help="Can optionally specify a registration protocol that is different from nlin protocol. "
"Parameters must be specified as in either or the following examples: \n"
"applications_testing/test_data/minctracc_example_nlin_protocol.csv \n"
"applications_testing/test_data/mincANTS_example_nlin_protocol.csv \n"
"[Default = %(default)s]")
return parser
def _mk_thickness_parser(parser: ArgParser):
group = parser.add_argument_group("Thickness",
"Thickness calculation options.")
group.add_argument("--run-thickness",
action='store_true',
dest="run_thickness",
help="Run thickness computation.")
group.add_argument("--no-run-thickness",
action='store_false',
dest="run_thickness",
help="Don't run thickness computation.")
parser.set_defaults(run_thickness=True)
group.add_argument(
"--label-mapping",
type=str,
dest="label_mapping",
help="path to CSV file mapping; see minclaplace/wiki/LaplaceGrid")
group.add_argument(
"--atlas-fwhm",
dest="atlas_fwhm",
type=float, # default ?!
help="Blurring kernel (mm) for atlas")
group.add_argument(
"--thickness-fwhm",
dest="thickness_fwhm",
type=float, # default??
help="Blurring kernel (mm) for cortical surfaces")
return parser
def _mk_staging_parser(parser : ArgParser):
group = parser.add_argument_group("Embryo staging options", "Options for staging embryos in a 4D atlas.")
group.add_argument("--csv-4D", dest="csv_4D", type=str,
help="CSV containing information about the 4D altas. Should contain "
"the following fields: `volume`, `timepoint`, `file`, "
"`mask_file`.")
return parser
def _mk_model_building_parser(parser: ArgParser):
group = parser.add_argument_group(
"Model building options",
"Options specific to consensus model building")
#group.add_argument("--registration-strategy", dest="reg_strategy",
# default="build_model", choices=['build_model', 'pairwise', 'tournament'],
# help="Process used for model construction [Default = %(default)s")
#group.add_argument("--preliminary-registration-strategy", dest="preliminary_reg_strategy",
# default=None, choices=['pairwise', 'tournament'],
# help="Process used to construct a preliminary target for nonlinear model building "
# "(use with '--registration-strategy=build_model' only!)")
#group.add_argument("--preliminary-registration-protocol", dest="preliminary_reg_protocol",
# default=None, type=str,
# help="Protocol file for the optional preliminary model building")
group.add_argument(
"--pairwise-nlin-max-pairs",
default=None,
type=int,
dest="prelim_nlin_max_pairs",
help="Maximum number of nonlinear registrations per input file "
"for preliminary pairwise nonlinear model construction [default = %(default)s"
)
group.add_argument(
"--pairwise-nlin-max-images",
default=25,
type=int,
dest="prelim_nlin_max_images",
help="Maximum number of images to use "
"for preliminary pairwise nonlinear model construction [default = %(default)s"
)
# TODO prelim_tournament_max_depth # continue as a build model afterwards??
return parser
def _mk_application_parser(p: ArgParser) -> ArgParser:
"""
The arguments that all applications share:
--pipeline-name
--restart
--no-restart
--output-dir
--create-graph
--execute
--no-execute
--version
--verbose
--no-verbose
files - leftover arguments (0 or more are allowed)
"""
# p = ArgParser(add_help=False)
g = p.add_argument_group("General application options",
"General options for all pydpiper applications.")
g.add_argument("--restart", dest="restart",
action="store_false", default=True,
help="Restart pipeline using backup files. [default = %(default)s]")
g.add_argument("--pipeline-name", dest="pipeline_name", type=str,
default=time.strftime("pipeline-%d-%m-%Y-at-%H-%m-%S"),
help="Name of pipeline and prefix for models.")
g.add_argument("--no-restart", dest="restart",
action="store_false", help="Opposite of --restart")
# TODO instead of prefixing all subdirectories (logs, backups, processed, ...)
# with the pipeline name/date, we could create one identifying directory
# and put these other directories inside
g.add_argument("--output-dir", dest="output_directory",
type=str,
default='',
help="Directory where output data and backups will be saved.")
g.add_argument("--create-graph", dest="create_graph",
action="store_true", default=False,
help="Create a .dot file with graphical representation of pipeline relationships [default = %(default)s]")
g.set_defaults(execute=True)
g.set_defaults(verbose=True)
g.add_argument("--execute", dest="execute",
action="store_true",
help="Actually execute the planned commands [default = %(default)s]")
g.add_argument("--no-execute", dest="execute",
action="store_false",
help="Opposite of --execute")
g.add_argument("--version", action="version",
version="%(prog)s (" + get_distribution("pydpiper").version + ")") # pylint: disable=E1101
g.add_argument("--verbose", dest="verbose",
action="store_true",
help="Be verbose in what is printed to the screen [default = %(default)s]")
g.add_argument("--no-verbose", dest="verbose",
action="store_false",
help="Opposite of --verbose")
g.add_argument("--files", type=str, nargs='*', metavar='file',
help='Files to process')
g.add_argument("--csv-file", dest="csv_file",
type=str, default=None,
help="CSV file containing application-specific columns. [Default=%(default)s]")
return p
def _mk_segmentation_parser(parser : ArgParser, default : bool):
group = parser.add_argument_group("Segmentation", "Segmentation options.")
group.add_argument("--run-maget", action='store_true', dest="run_maget",
help="Run MAGeT segmentation. [default = %(default)s]")
group.add_argument("--no-run-maget", dest="run_maget",
action='store_false', help="Don't run MAGeT segmentation")
parser.set_defaults(run_maget=True)
return parser
def _mk_segmentation_parser(parser: ArgParser, default: bool):
group = parser.add_argument_group("Segmentation", "Segmentation options.")
group.add_argument("--run-maget",
action='store_true',
dest="run_maget",
help="Run MAGeT segmentation. [default = %(default)s]")
group.add_argument("--no-run-maget",
dest="run_maget",
action='store_false',
help="Don't run MAGeT segmentation")
parser.set_defaults(run_maget=True)
return parser
def parse():
configpath = resource_filename(__name__, '../data/config.ini')
p = ArgParser(default_config_files=[configpath])
# GENERAL
general = p.add_argument_group('General')
general.add('-c', '--config', is_config_file=True, help='config file path')
general.add('-v', action='store_true', help='verbose')
general.add('-m', '--mode', help='program mode')
general.add('--multiprocessing', type=string2bool, nargs='?', const=True, default=False)
general.add('--cores', type=int)
# CLUSTERS
clusters = p.add_argument_group('Clusters')
clusters.add('--add_threshold', type=float)
clusters.add('--merge_threshold', type=float)
clusters.add('--mature_threshold', type=int)
clusters.add('--merge_frequency', type=int)
clusters.add('--remove_frequency', type=int)
clusters.add('--summary_frequency', type=int)
# EMBEDDING MODEL
model = p.add_argument_group('Embedding Model')
model.add('--type', type=str)
model.add('--source', type=str)
# DATASETS
datasets = p.add_argument_group('Datasets')
datasets.add('--background', type=str)
datasets.add('--background_weight', type=float)
datasets.add('--filter', type=string2bool, nargs='?', const=True, default=False)
datasets.add('--merge', type=string2bool, nargs='?', const=True, default=False)
# p.add('-i', '--input', help='input file name')
# p.add('-o', '--output', help='output file name')
# make a setup function to download nltk resources:
# stopwords, punkt, wordnet
return p.parse_args()
def _mk_thickness_parser(parser : ArgParser):
group = parser.add_argument_group("Thickness", "Thickness calculation options.")
group.add_argument("--run-thickness", action='store_true', dest="run_thickness",
help="Run thickness computation.")
group.add_argument("--no-run-thickness", action='store_false', dest="run_thickness",
help="Don't run thickness computation.")
parser.set_defaults(run_thickness=True)
group.add_argument("--label-mapping", type=str, dest="label_mapping",
help="path to CSV file mapping; see minclaplace/wiki/LaplaceGrid")
group.add_argument("--atlas-fwhm", dest="atlas_fwhm", type=float, # default ?!
help="Blurring kernel (mm) for atlas")
group.add_argument("--thickness-fwhm", dest="thickness_fwhm", type=float, # default??
help="Blurring kernel (mm) for cortical surfaces")
return parser
def _mk_nlin_parser(p: ArgParser):
group = p.add_argument_group("Nonlinear registration options",
"Options for performing a non-linear registration")
group.add_argument("--registration-method", dest="reg_method",
default="ANTS", choices=["ANTS", "minctracc"],
help="Specify whether to use minctracc or ANTS for non-linear registrations. "
"[Default = %(default)s]")
group.add_argument("--nlin-protocol", dest="nlin_protocol",
type=str, default=None,
help="Can optionally specify a registration protocol that is different from defaults. "
"Parameters must be specified as in either or the following examples: \n"
"applications_testing/test_data/minctracc_example_nlin_protocol.csv \n"
"applications_testing/test_data/mincANTS_example_nlin_protocol.csv \n"
"[Default = %(default)s]")
return p
def _mk_nlin_parser(p: ArgParser):
group = p.add_argument_group("Nonlinear registration options",
"Options for performing a non-linear registration")
group.add_argument("--registration-method", dest="reg_method",
default="ANTS", choices=["ANTS", "antsRegistration", "demons",
"DRAMMS", "elastix", "minctracc"],
help="Specify algorithm used for non-linear registrations. "
"[Default = %(default)s]")
# TODO wire up the choices here in reg_method and reg_strategy to the actual ones ...
group.add_argument("--registration-strategy", dest="reg_strategy",
default="build_model", choices=['build_model', 'pairwise', 'tournament',
'tournament_and_build_model', 'pairwise_and_build_model'],
help="Process used for model construction [Default = %(default)s")
group.add_argument("--nlin-protocol", dest="nlin_protocol",
type=str, default=None,
help="Can optionally specify a registration protocol that is different from defaults. "
"Parameters must be specified as in either or the following examples: \n"
"applications_testing/test_data/minctracc_example_nlin_protocol.csv \n"
"applications_testing/test_data/mincANTS_example_nlin_protocol.csv \n"
"[Default = %(default)s]")
return p
def _mk_nlin_parser(p: ArgParser):
group = p.add_argument_group(
"Nonlinear registration options",
"Options for performing a non-linear registration")
group.add_argument(
"--registration-method",
dest="reg_method",
default="ANTS",
choices=["ANTS", "antsRegistration", "minctracc"],
help="Specify algorithm used for non-linear registrations. "
"[Default = %(default)s]")
# TODO wire up the choices here in reg_method and reg_strategy to the actual ones ...
group.add_argument(
"--registration-strategy",
dest="reg_strategy",
default="build_model",
choices=[
'build_model', 'pairwise', 'tournament',
'tournament_and_build_model', 'pairwise_and_build_model'
],
help="Process used for model construction [Default = %(default)s")
group.add_argument(
"--nlin-protocol",
dest="nlin_protocol",
type=str,
default=None,
help=
"Can optionally specify a registration protocol that is different from defaults. "
"Parameters must be specified as in either or the following examples: \n"
"applications_testing/test_data/minctracc_example_nlin_protocol.csv \n"
"applications_testing/test_data/mincANTS_example_nlin_protocol.csv \n"
"[Default = %(default)s]")
group.add_argument("--use-robust-averaging",
dest="use_robust_averaging",
action='store_true',
help="use robust intensity averaging if possible")
p.set_defaults(use_robust_averaging=False)
return p
def _mk_registration_parser(p: ArgParser) -> ArgParser:
g = p.add_argument_group("General registration options", "....")
# p = ArgParser(add_help=False)
g.add_argument(
"--input-space",
dest="input_space",
type=lambda x: InputSpace[x], # type: ignore # mypy/issues/741
default=InputSpace.native,
# choices=[x for x, _ in InputSpace.__members__.items()],
help=
"Option to specify space of input-files. Can be native (default), lsq6, lsq12. "
"Native means that there is no prior formal alignment between the input files "
"yet. lsq6 means that the input files have been aligned using translations "
"and rotations; the code will continue with a 12 parameter alignment. lsq12 "
"means that the input files are fully linearly aligned. Only non-linear "
"registrations are performed. [Default=%(default)s]")
g.add_argument(
"--resolution",
dest="resolution",
type=float,
default=None,
help=
"Specify the resolution at which you want the registration to be run. "
"If not specified, the resolution of the target of your pipeline will "
"be used. [Default=%(default)s]")
g.add_argument(
"--subject-matter",
dest="subject_matter",
type=str,
default=None,
help=
"Specify the subject matter for the pipeline. This will set the parameters "
"for multiple programs based on the overall size of the subject matter. Instead "
"of using the resolution of the files. Currently supported option is: \"mousebrain\" "
"[Default=%(default)s]")
return p # g?
def _mk_common_space_parser(parser: ArgParser):
group = parser.add_argument_group(
"Common space options",
"Options for registration/resampling to common (db) space.")
group.add_argument("--common-space-model",
dest="common_space_model",
type=str,
help="Model image to which to align consensus average")
group.add_argument("--common-space-mask",
dest="common_space_mask",
type=str,
help="Mask for common space model")
group.set_defaults(do_common_space_registration=False)
group.add_argument("--common-space-registration",
dest="do_common_space_registration",
action="store_true",
help="Do registration to common (db) space.")
group.add_argument(
"--no-common-space-registration",
dest="do_common_space_registration",
default=True,
action="store_false",
help="Skip registration to common (db) space [default].")
return parser
def parse_args():
parser = ArgParser(default_config_files=[os.getcwd() + '/src/initial_configurations/default'])
# Core setting
core_parse = parser.add_argument_group('Core setting')
core_parse.add_argument('-s', '--start_date', dest='start_date', default='now', type=str, help='Training start date')
core_parse.add_argument('-p', '--train_period', dest='train_period', default=-1, type=int, help='Time period of training file is used')
core_parse.add_argument('-n', '--new_run', dest='new_run', default=0, type=int, help='If the model checkpoint is erased to run new model')
core_parse.add_argument('-l', '--local_run', dest='local_run', default=0, type=int, help='If the parameter JSON file is kept locally insteat to redis')
core_parse.add_argument('-nni', '--tuning', dest='tuning', default=0, type=int, help='Whether or not to peform NNI hyper parameter tuning')
# Model
model_parse = parser.add_argument_group('Model')
model_parse.add_argument('-m', '--model', dest='model', default='DNN', type=str, help='Select the model to train e.g. DNN')
model_parse.add_argument('--loss', dest='loss', default=30, type=int, help="Setting of loss function '10','11','12','20','21','22','30','31','32'" )
model_parse.add_argument('--hidden_units', dest='hidden_units', default=[128, 64], type=int, nargs='+', help='List containing the number of hidden units to use for each hidden layer')
model_parse.add_argument('--dropout_rate', dest='dropout_rate', default=0.5, type=float, help='List containing the number of dropout rate to use for each hidden layer')
model_parse.add_argument('--one_hot_units', dest='one_hot_units', default=[2, 35, 359, 3, 2], type=int, nargs='+', help='List containing the number of embedding units to use for features (in order): [weekday, region, city, adexchange, slotformat]; this replaces the one hot encoding')
model_parse.add_argument('--multi_hot_units', dest='multi_hot_units', default=[45], type=int, nargs='+', help='List containing the number of embedding units to use for features: [usertag]')
model_parse.add_argument('--learning_rate', dest='learning_rate', default=0.002, type=float, help='Learning rate of updating gradient')
model_parse.add_argument('--decay_step', dest='decay_step', default=100, type=int, help='Decay step')
model_parse.add_argument('--decay_rate', dest='decay_rate', default=0.98, type=float, help='Decay rate for exponential decay of learning rate')
model_parse.add_argument('--class_ratio', dest='class_ratio', default=0.5, type=float, help='Ratio of 2 classes for imbalanced data')
model_parse.add_argument('--alpha', dest='alpha', default=1., type=float, help='Alpha for Focal loss regularization in DNN')
model_parse.add_argument('--beta', dest='beta', default=1., type=float, help='Beta for regularization')
model_parse.add_argument('--gamma', dest='gamma', default=1., type=float, help='Gamma for Focal loss regularization in DNN')
# Training
train_parse = parser.add_argument_group('Training hyperparameters')
train_parse.add_argument('--save_summary_steps', dest='save_summary_steps', default=100, type=int, help='save summary steps')
train_parse.add_argument('--log_step_count_steps', dest='log_step_count_steps', default=100, type=int, help='logging step count steps')
train_parse.add_argument('--checkpoints_steps', dest='save_checkpoints_steps',default=500, type=int, help='checkpoints steps')
train_parse.add_argument('--has_gpu', dest='has_gpu', default=0, type=int, help='1 if GPU is present, else 0')
train_parse.add_argument('--oversample', dest='oversample', default=0, type=int, help='1 if will oversample training dataset, else 0')
train_parse.add_argument('--is_test', dest='is_test', default=0, type=int, help='1 if the trained model will be evaluated, else 0')
train_parse.add_argument('--num_epochs', dest='num_epochs', default=1.0, type=float, help='Number of total epochs')
train_parse.add_argument('--start_delay_secs', dest='start_delay_secs', default=10, type=int, help='Start evaluating after 10 secs')
train_parse.add_argument('--throttle_secs', dest='throttle_secs', default=10, type=int, help='Evaluate only every 30 secs')
train_parse.add_argument('--batch_size', dest='batch_size', default=128, type=int, help='Number of examples per batch')
# Directory paths
dir_parse = parser.add_argument_group('Directory paths')
dir_parse.add_argument('--train_data_path', dest='train_data_path', default='./data/', type=str, help='Directory where the training files are located')
dir_parse.add_argument('--save_dir', dest='save_dir', default='./Outputs/', type=str, help='Directory to save model directories')
dir_parse.add_argument('--load_dir', dest='load_dir', default='latest', type=str, help='Directory to load old model,default "new" as the latest model')
dir_parse.add_argument('--store_dir', dest='store_dir', default='latest', type=str, help='Directory to store current model, default "latest" to save in timestamp')
dir_parse.add_argument('--builder_save_dir', dest='builder_save_dir', default='builder_save', type=str, help='Directory to store current model for tfjs predictor')
_args, _ = parser.parse_known_args()
_params = vars(_args)
_params['train_data_path'] = os.getcwd() + _params['train_data_path']
# Identify whether it's using NNI tuning mode
if _params['tuning'] == 1:
import nni
tuner_params = nni.get_next_parameter()
try:
_params.update(tuner_params)
except Exception as err:
tf.logging.error('Error args updated: %s', err)
tf.logging.error('Failed with params: %s', str(_params))
_params['num_features'] = len(INT_NUM_FEAT) + sum(_params['one_hot_units']) + sum(_params['multi_hot_units'])
_params['model_name'] = _params['model']
# Adjust filename to restore/save by config settings
if _params['store_dir'] == 'latest':
_params['store_dir'] = _params['model_name'] + '_' + parse_date('now').strftime(FILE_DATE_FORMAT)
if _params['load_dir'] == 'latest':
_params['load_dir'] = find_latest_model_dir(_params['save_dir'], _params['store_dir'], _params['model_name'])
if _params['new_run'] == 1:
_params['load_dir'] = _params['store_dir']
return _params
def _mk_application_parser(p: ArgParser) -> ArgParser:
# p = ArgParser(add_help=False)
g = p.add_argument_group("General application options",
"General options for all pydpiper applications.")
#TODO is this broken? Shouldn't this action be "store_true"?
g.add_argument(
"--restart",
dest="restart",
action="store_false",
default=True,
help="Restart pipeline using backup files. [default = %(default)s]")
g.add_argument(
"--smart-restart",
dest="smart_restart",
action="store_true",
default=False,
help=
"Restart pipeline using backup files accounting for modifications. [default = %(default)s]"
)
g.add_argument("--pipeline-name",
dest="pipeline_name",
type=str,
default=time.strftime("pipeline-%d-%m-%Y-at-%H-%m-%S"),
help="Name of pipeline and prefix for models.")
g.add_argument("--no-restart",
dest="restart",
action="store_false",
help="Opposite of --restart")
# TODO instead of prefixing all subdirectories (logs, backups, processed, ...)
# with the pipeline name/date, we could create one identifying directory
# and put these other directories inside
g.add_argument(
"--output-dir",
dest="output_directory",
type=str,
default='',
help="Directory where output data and backups will be saved.")
g.add_argument(
"--create-graph",
dest="create_graph",
action="store_true",
default=False,
help=
"Create a .dot file with graphical representation of pipeline relationships [default = %(default)s]"
)
g.set_defaults(execute=True)
g.set_defaults(verbose=True)
g.add_argument(
"--execute",
dest="execute",
action="store_true",
help="Actually execute the planned commands [default = %(default)s]")
g.add_argument("--no-execute",
dest="execute",
action="store_false",
help="Opposite of --execute")
g.add_argument("--version",
action="version",
version="%(prog)s (" +
get_distribution("pydpiper").version + ")") # pylint: disable=E1101
g.add_argument(
"--verbose",
dest="verbose",
action="store_true",
help=
"Be verbose in what is printed to the screen [default = %(default)s]")
g.add_argument("--no-verbose",
dest="verbose",
action="store_false",
help="Opposite of --verbose")
g.add_argument("--files",
type=str,
nargs='*',
metavar='file',
help='Files to process')
g.add_argument(
"--csv-file",
dest="csv_file",
type=str,
default=None,
help=
"CSV file containing application-specific columns. [Default=%(default)s]"
)
# TODO also allow relative to pipeline output dir??
g.add_argument(
"--csv-paths-relative-to-wd",
dest="csv_paths_relative_to_wd",
default=False,
action="store_true",
help="CSV paths are relative to the working directory, not the CSV file"
)
return p
def _mk_execution_parser(p: ArgParser) -> ArgParser:
# parser = ArgParser(add_help=False)
group = p.add_argument_group("Executor options",
"Options controlling how and where the code is run.")
group.add_argument("--uri-file", dest="urifile",
type=str, default=None,
help="Location for uri file if NameServer is not used. If not specified, default is current working directory.")
group.add_argument("--use-ns", dest="use_ns",
action="store_true",
help="Use the Pyro NameServer to store object locations. Currently a Pyro nameserver must be started separately for this to work.")
group.add_argument("--latency-tolerance", dest="latency_tolerance",
type=float, default=600.0,
help="Allowed grace period by which an executor may miss a heartbeat tick before being considered failed [Default = %(default)s.")
group.add_argument("--num-executors", dest="num_exec",
type=int, default=-1,
help="Number of independent executors to launch. [Default = %(default)s. Code will not run without an explicit number specified.]")
group.add_argument("--max-failed-executors", dest="max_failed_executors",
type=int, default=10,
help="Maximum number of failed executors before we stop relaunching. [Default = %(default)s]")
# TODO: add corresponding --monitor-heartbeats
group.add_argument("--no-monitor-heartbeats", dest="monitor_heartbeats",
action="store_false",
help="Don't assume executors have died if they don't check in with the server (NOTE: this can hang your pipeline if an executor crashes).")
group.add_argument("--time", dest="time",
type=str, default="23:59:59",
help="Wall time to request for each server/executor in the format hh:mm:ss. Required only if --queue-type=pbs. Current default on PBS is %(default)s.")
group.add_argument("--proc", dest="proc",
type=int, default=1,
help="Number of processes per executor. Also sets max value for processor use per executor. [Default = %(default)s]")
group.add_argument("--mem", dest="mem",
type=float, default=6,
help="Total amount of requested memory (in GB) for all processes the executor runs. [Default = %(default)s].")
group.add_argument("--pe", dest="pe",
type=str, default=None,
help="Name of the SGE pe, if any. [Default = %(default)s]")
group.add_argument("--mem-request-attribute", dest="mem_request_attribute",
type=str, default=None,
help="Name of the resource attribute to request for managing memory limits. [Default = %(default)s]")
group.add_argument("--greedy", dest="greedy",
action="store_true",
help="Request the full amount of RAM specified by --mem rather than the (lesser) amount needed by runnable jobs. Always use this if your executor is assigned a full node.")
group.add_argument("--ppn", dest="ppn",
type=int, default=8,
help="Number of processes per node. Used when --queue-type=pbs. [Default = %(default)s].")
group.add_argument("--queue-name", dest="queue_name", type=str, default=None,
help="Name of the queue, e.g., all.q (MICe) or batch (SciNet)")
group.add_argument("--queue-type", dest="queue_type", type=str, default=None,
help="""Queue type to submit jobs, i.e., "sge" or "pbs". [Default = %(default)s]""")
group.add_argument("--queue-opts", dest="queue_opts",
type=str, default="",
help="A string of extra arguments/flags to pass to qsub. [Default = %(default)s]")
group.add_argument("--executor-start-delay", dest="executor_start_delay", type=int, default=180,
help="Seconds before starting remote executors when running the server on the grid")
group.add_argument("--submit-server", dest="submit_server", action="store_true",
help="Submit the server to the grid. Currently works only with PBS/Torque systems.")
group.add_argument("--no-submit-server", dest="submit_server", action="store_false",
help="Opposite of --submit-server. [default]")
group.add_argument("--max-idle-time", dest="max_idle_time",
type=int, default=1,
help="The number of minutes an executor is allowed to continuously sleep, i.e. wait for an available job, while active on a compute node/farm before it kills itself due to resource hogging. [Default = %(default)s]")
group.add_argument("--time-to-accept-jobs", dest="time_to_accept_jobs",
type=int,
help="The number of minutes after which an executor will not accept new jobs anymore. This can be useful when running executors on a batch system where other (competing) jobs run for a limited amount of time. The executors can behave in a similar way by given them a rough end time. [Default = %(default)s]")
group.add_argument('--local', dest="local", action='store_true',
help="Don't submit anything to any specified queueing system but instead run as a server/executor")
group.add_argument("--config-file", type=str, metavar='config_file', is_config_file=True,
required=False, help='Config file location')
group.add_argument("--prologue-file", type=str, metavar='file',
help="Location of a shell script to inline into PBS submit script to set paths, load modules, etc.")
group.add_argument("--min-walltime", dest="min_walltime", type=int, default=0,
help="Min walltime (s) allowed by the queuing system [Default = %(default)s]")
group.add_argument("--max-walltime", dest="max_walltime", type=int, default=None,
help="Max walltime (s) allowed for jobs on the queuing system, or infinite if None [Default = %(default)s]")
group.add_argument("--default-job-mem", dest="default_job_mem",
type=float, default=1.75,
help="Memory (in GB) to allocate to jobs which don't make a request. [Default=%(default)s]")
group.add_argument("--memory-factor", dest="memory_factor",
type=float, default=1,
help="Overall factor by which to scale all memory estimates/requests (including default job memory, "
"but not executor totals (--mem)), say due to system differences or overcommitted nodes. "
"[Default=%(default)s]")
group.add_argument("--cmd-wrapper", dest="cmd_wrapper",
type=str, default="",
help="Wrapper inside of which to run the command, e.g., '/usr/bin/time -v'. [Default='%(default)s']")
group.add_argument("--check-input-files", dest="check_input_files", action="store_true",
help="Check overall pipeline inputs exist and, when applicable, "
"are valid MINC files [Default=%(default)s]")
group.add_argument("--no-check-input-files", dest="check_input_files", action="store_false",
help="Opposite of --check-input-files")
group.set_defaults(check_inputs=True)
group.add_argument("--check-outputs", dest="check_outputs",
action="store_true",
help="Check output files exist and error if not [Default=%(default)s]")
group.add_argument("--no-check-outputs", dest="check_outputs",
action="store_false",
help="Opposite of --check-outputs.")
group.set_defaults(check_outputs=False)
group.add_argument("--fs-delay", dest="fs_delay",
type=float, default=5,
help="Time (sec) to allow for NFS to become consistent after stage completion [Default=%(default)s]")
group.add_argument("--executor_wrapper", dest="executor_wrapper",
type=str, default="",
help="Command inside of which to run the executor. [Default='%(default)s']")
group.add_argument("--defer-directory-creation", default=False,
action="store_true", dest="defer_directory_creation",
help="Create relevant directories when a stage is run instead of at startup [Default=%(default)s]")
return p
def _mk_execution_parser(p: ArgParser) -> ArgParser:
# parser = ArgParser(add_help=False)
group = p.add_argument_group(
"Executor options",
"Options controlling how and where the code is run.")
group.add_argument(
"--uri-file",
dest="urifile",
type=str,
default=None,
help=
"Location for uri file if NameServer is not used. If not specified, default is current working directory."
)
group.add_argument(
"--use-ns",
dest="use_ns",
action="store_true",
help=
"Use the Pyro NameServer to store object locations. Currently a Pyro nameserver must be started separately for this to work."
)
group.add_argument(
"--latency-tolerance",
dest="latency_tolerance",
type=float,
default=600.0,
help=
"Allowed grace period by which an executor may miss a heartbeat tick before being considered failed [Default = %(default)s."
)
group.add_argument(
"--num-executors",
dest="num_exec",
type=int,
default=-1,
help=
"Number of independent executors to launch. [Default = %(default)s. Code will not run without an explicit number specified.]"
)
group.add_argument(
"--max-failed-executors",
dest="max_failed_executors",
type=int,
default=10,
help=
"Maximum number of failed executors before we stop relaunching. [Default = %(default)s]"
)
# TODO: add corresponding --monitor-heartbeats
group.add_argument(
"--no-monitor-heartbeats",
dest="monitor_heartbeats",
action="store_false",
help=
"Don't assume executors have died if they don't check in with the server (NOTE: this can hang your pipeline if an executor crashes)."
)
group.add_argument(
"--time",
dest="time",
type=str,
default="23:59:59",
help=
"Wall time to request for each server/executor in the format hh:mm:ss. Required only if --queue-type=pbs. Current default on PBS is %(default)s."
)
group.add_argument(
"--proc",
dest="proc",
type=int,
default=1,
help=
"Number of processes per executor. Also sets max value for processor use per executor. [Default = %(default)s]"
)
group.add_argument(
"--mem",
dest="mem",
type=float,
default=6,
help=
"Total amount of requested memory (in GB) for all processes the executor runs. [Default = %(default)s]."
)
group.add_argument(
"--pe",
dest="pe",
type=str,
default=None,
help="Name of the SGE pe, if any. [Default = %(default)s]")
group.add_argument(
"--mem-request-attribute",
dest="mem_request_attribute",
type=str,
default=None,
help=
"Name of the resource attribute to request for managing memory limits. [Default = %(default)s]"
)
group.add_argument(
"--greedy",
dest="greedy",
action="store_true",
help=
"Request the full amount of RAM specified by --mem rather than the (lesser) amount needed by runnable jobs. Always use this if your executor is assigned a full node."
)
group.add_argument(
"--ppn",
dest="ppn",
type=int,
default=8,
help=
"Number of processes per node. Used when --queue-type=pbs. [Default = %(default)s]."
)
group.add_argument(
"--queue-name",
dest="queue_name",
type=str,
default=None,
help="Name of the queue, e.g., all.q (MICe) or batch (SciNet)")
group.add_argument(
"--queue-type",
dest="queue_type",
type=str,
default=None,
help=
"""Queue type to submit jobs, i.e., "sge" or "pbs". [Default = %(default)s]"""
)
group.add_argument(
"--queue-opts",
dest="queue_opts",
type=str,
default="",
help=
"A string of extra arguments/flags to pass to qsub. [Default = %(default)s]"
)
group.add_argument(
"--executor-start-delay",
dest="executor_start_delay",
type=int,
default=180,
help=
"Seconds before starting remote executors when running the server on the grid"
)
group.add_argument(
"--submit-server",
dest="submit_server",
action="store_true",
help=
"Submit the server to the grid. Currently works only with PBS/Torque systems."
)
group.add_argument("--no-submit-server",
dest="submit_server",
action="store_false",
help="Opposite of --submit-server. [default]")
group.add_argument(
"--max-idle-time",
dest="max_idle_time",
type=int,
default=1,
help=
"The number of minutes an executor is allowed to continuously sleep, i.e. wait for an available job, while active on a compute node/farm before it kills itself due to resource hogging. [Default = %(default)s]"
)
group.add_argument(
"--time-to-accept-jobs",
dest="time_to_accept_jobs",
type=int,
help=
"The number of minutes after which an executor will not accept new jobs anymore. This can be useful when running executors on a batch system where other (competing) jobs run for a limited amount of time. The executors can behave in a similar way by given them a rough end time. [Default = %(default)s]"
)
group.add_argument(
'--local',
dest="local",
action='store_true',
help=
"Don't submit anything to any specified queueing system but instead run as a server/executor"
)
group.add_argument("--config-file",
type=str,
metavar='config_file',
is_config_file=True,
required=False,
help='Config file location')
group.add_argument(
"--prologue-file",
type=str,
metavar='file',
help=
"Location of a shell script to inline into PBS submit script to set paths, load modules, etc."
)
group.add_argument(
"--min-walltime",
dest="min_walltime",
type=int,
default=0,
help=
"Min walltime (s) allowed by the queuing system [Default = %(default)s]"
)
group.add_argument(
"--max-walltime",
dest="max_walltime",
type=int,
default=None,
help=
"Max walltime (s) allowed for jobs on the queuing system, or infinite if None [Default = %(default)s]"
)
group.add_argument(
"--default-job-mem",
dest="default_job_mem",
type=float,
default=1.75,
help=
"Memory (in GB) to allocate to jobs which don't make a request. [Default=%(default)s]"
)
group.add_argument(
"--memory-factor",
dest="memory_factor",
type=float,
default=1,
help=
"Overall factor by which to scale all memory estimates/requests (including default job memory, "
"but not executor totals (--mem)), say due to system differences or overcommitted nodes. "
"[Default=%(default)s]")
group.add_argument(
"--cmd-wrapper",
dest="cmd_wrapper",
type=str,
default="",
help=
"Wrapper inside of which to run the command, e.g., '/usr/bin/time -v'. [Default='%(default)s']"
)
group.add_argument(
"--check-input-files",
dest="check_input_files",
action="store_true",
help="Check overall pipeline inputs exist and, when applicable, "
"are valid MINC files [Default=%(default)s]")
group.add_argument("--no-check-input-files",
dest="check_input_files",
action="store_false",
help="Opposite of --check-input-files")
group.set_defaults(check_inputs=True)
group.add_argument(
"--check-outputs",
dest="check_outputs",
action="store_true",
help="Check output files exist and error if not [Default=%(default)s]")
group.add_argument("--no-check-outputs",
dest="check_outputs",
action="store_false",
help="Opposite of --check-outputs.")
group.set_defaults(check_outputs=False)
group.add_argument(
"--fs-delay",
dest="fs_delay",
type=float,
default=5,
help=
"Time (sec) to allow for NFS to become consistent after stage completion [Default=%(default)s]"
)
group.add_argument(
"--executor_wrapper",
dest="executor_wrapper",
type=str,
default="",
help=
"Command inside of which to run the executor. [Default='%(default)s']")
group.add_argument(
"--defer-directory-creation",
default=False,
action="store_true",
dest="defer_directory_creation",
help=
"Create relevant directories when a stage is run instead of at startup [Default=%(default)s]"
)
return p
def program_options():
"""Create argument parser for the CLI.
"""
parser = ArgParser()
dp_group = parser.add_argument_group(title="Required DP Parameters")
io_group = parser.add_argument_group(
title="Required Train Data Parameters")
# File IO
io_group.add(
"--train-normal",
metavar="TRAIN-NORMAL-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the NORMAL classification label.",
type=str,
required=True
)
io_group.add(
"--train-tumor",
metavar="TRAIN-TUMOR-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the TUMOR classification label.",
type=str,
required=True
)
# DP Req. Options
dp_group.add(
"--epsilon",
nargs="+",
help="Epsilon value(s) for differentially-private training. "\
"One or many epsilon values can be specified. If multiple epsilons are "\
"specified, then independent experiments will be run for each specified "\
"epislon value. The results of each of these runs will be stored in "\
"separate, named result files. "\
"Epsilons can be specified as decimal values. Some examples of valid "\
"epsilon arguments are "\
"`--epsilon 3`, "\
"`--epsilon 5.32341`, "\
"`--epsilon 3 3.5 4 4.5 20`.",
type=float,
required=True
)
dp_group.add(
"--delta",
nargs="+",
help= "Delta value(s) for differentially-private training. "\
"One or many delta values can be specified. If multiple deltas are"\
"specified, then independent experiments will be run for each delta"\
"value in combination with each epsilon value."\
"The reuslts of these runs are stored in separate, named result files."\
"To use (eps)-DP for privacy calculations, pass use the option\n"\
"`--delta 0`.",
type=float,
required=True
)
parser.add("--participant",
metavar="PARTICPANT",
help="Server or client.",
choices=["server", "client"],
required=True)
parser.add("--training-seed",
help="Seed used for the training.",
type=int,
default=42)
parser.add(
"--output-dir",
help="Directory to store output result files to. If the directory does not "\
"exist, it will be created.",
type=str,
default="."
)
parser.add("--host",
help="Specifies the server address.",
default="localhost",
type=str)
parser.add(
"--port",
help=
"Specifies the port through which the two workers should communicate on the host machine.",
default="8081",
type=int)
parser.add("--mode",
help="Specifies the launching mode.",
default="subprocess",
choices=["subprocess", "docker"],
type=str)
args = parser.parse_args()
# Some post processing, for lists etc.
if not isinstance(args.epsilon, list):
args.epsilon = [
args.epsilon,
]
if not isinstance(args.delta, list):
args.delta = [
args.delta,
]
# Convert all relative file paths into absolute paths
resolve_path = lambda p: str(pathlib.Path(p).resolve())
args.train_normal = resolve_path(args.train_normal)
args.train_tumor = resolve_path(args.train_tumor)
if args.output_dir is not None:
args.output_dir = resolve_path(args.output_dir)
return args
def program_options():
"""Create argument parser for the CLI.
"""
parser = ArgParser()
dp_group = parser.add_argument_group(title="Required DP Parameters")
io_group = parser.add_argument_group(
title="Required Train Data Parameters")
comm_group = parser.add_argument_group(
title="Flags for Communication (no touch)")
# File IO
io_group.add(
"--train-normal-alice",
metavar="TRAIN-NORMAL-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the NORMAL classification label.",
type=str,
required=True
)
io_group.add(
"--train-tumor-alice",
metavar="TRAIN-TUMOR-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the TUMOR classification label.",
type=str,
required=True
)
io_group.add(
"--train-normal-bob",
metavar="TRAIN-NORMAL-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the NORMAL classification label.",
type=str,
required=True
)
io_group.add(
"--train-tumor-bob",
metavar="TRAIN-TUMOR-DATA-FILE",
help="Path to training data file consisting of data samples corresponding "\
"to the TUMOR classification label.",
type=str,
required=True
)
# DP Req. Options
dp_group.add(
"--epsilon",
nargs="+",
help="Epsilon value(s) for differentially-private training. "\
"One or many epsilon values can be specified. If multiple epsilons are "\
"specified, then independent experiments will be run for each specified "\
"epislon value. The results of each of these runs will be stored in "\
"separate, named result files. "\
"Epsilons can be specified as decimal values. Some examples of valid "\
"epsilon arguments are "\
"`--epsilon 3`, "\
"`--epsilon 5.32341`, "\
"`--epsilon 3 3.5 4 4.5 20`.",
type=float,
required=True
)
dp_group.add(
"--delta",
nargs="+",
help= "Delta value(s) for differentially-private training. "\
"One or many delta values can be specified. If multiple deltas are"\
"specified, then independent experiments will be run for each delta"\
"value in combination with each epsilon value."\
"The reuslts of these runs are stored in separate, named result files."\
"To use (eps)-DP for privacy calculations, pass use the option\n"\
"`--delta 0`.",
type=float,
required=True
)
# Optional arguments
parser.add(
"--output-dir",
help="Directory to store output trained model to. If the directory does not "\
"exist, it will be created.",
type=str,
default="owkin-results"
)
# Next arguments
comm_group.add(
"--port",
help=
"Specifies the port through which the two workers should communicate on the host machine.",
default="8081",
type=int)
comm_group.add(
"--subprocess",
help="If set, the training will be performed between two subprocesses. If unset (default), "\
"the training will be performed with Docker containers.",
default=False,
action="store_true"
)
args = parser.parse_args()
# Some post processing, for lists etc.
if not isinstance(args.epsilon, list):
args.epsilon = [
args.epsilon,
]
if not isinstance(args.delta, list):
args.delta = [
args.delta,
]
return args
def _mk_maget_parser(parser: ArgParser):
group = parser.add_argument_group("MAGeT options",
"Options for running MAGeT.")
group.add_argument(
"--atlas-library",
dest=
"atlas_lib", # can't make required=True since may not be using MAGeT :|
type=str, # TODO: check existence of this dir?
help=
"Directory of existing atlas/label pairs (alternative to --atlas-csv)")
group.add_argument(
"--atlas-csv",
dest="atlas_csv",
type=str,
help=
"CSV containing (at least) `file`, `mask_file`, `label_file` columns "
"(alternative to --atlas-library)")
group.add_argument(
"--pairwise",
dest="pairwise",
action="store_true",
help=
"""If specified, register inputs to each other pairwise. [Default]""")
group.add_argument(
"--no-pairwise",
dest="pairwise",
action="store_false",
help="""If specified, only register inputs to atlases in library.""")
parser.set_defaults(pairwise=True)
group.add_argument(
"--mask",
dest="mask",
action="store_true",
default=True,
help=
"Create a mask for all images prior to handling labels. [Default = %(default)s]"
)
group.add_argument("--no-mask",
dest="mask",
action="store_false",
default=True,
help="Opposite of --mask.")
group.add_argument(
"--mask-only",
dest="mask_only",
action="store_true",
default=False,
help=
"Create a mask for all images only, do not run full algorithm. [Default = %(default)s]"
)
group.add_argument(
"--max-templates",
dest="max_templates",
default=25,
type=int,
help="Maximum number of templates to generate. [Default = %(default)s]"
)
group.add_argument(
"--masking-method",
dest="mask_method",
default="minctracc",
type=str,
help=
"Specify whether to use minctracc or ANTS for masking. [Default = %(default)s]."
)
group.add_argument(
"--masking-nlin-protocol",
dest="masking_nlin_protocol",
# TODO basically copied from nlin parser
type=str,
default=None,
help=
"Can optionally specify a registration protocol that is different from nlin protocol. "
"Parameters must be specified as in either or the following examples: \n"
"applications_testing/test_data/minctracc_example_nlin_protocol.csv \n"
"applications_testing/test_data/mincANTS_example_nlin_protocol.csv \n"
"[Default = %(default)s]")
return parser
def ParseArgs():
parser = ArgParser(default_config_files=[
os.getcwd() + '/src/initial_configurations/default'])
# Core settings
core_parse = parser.add_argument_group('Core setting')
core_parse.add_argument('-s', '--start_date', dest='start_date',
default=START_DATE, type=str, help='Training start date')
core_parse.add_argument('-p','--train_period', dest='train_period',
default=TRAIN_PERIOD, type=int, help='Time period of training file is used')
core_parse.add_argument('-n','--new_run', dest='new_run', default=1,
type=int, help='If the model checkpoint is erased to run new model')
core_parse.add_argument('-l','--local_run', dest='local_run', default=1,
type=int, help='If the parameter JSON file is kept locally insteat to redis')
core_parse.add_argument('-nni','--tuning', dest='tuning', default=0,
type=int, help='Whether or not to peform hyper parameter tuning')
# Data
data_parse = parser.add_argument_group('Data setting')
data_parse.add_argument('--train_data_path', dest='train_data_path',
default=DATA_PATH, type=str, help='Directory where the training files are located')
data_parse.add_argument('--random_seed', dest='random_seed', default=8888,
type=int, help='Random seed used for shuffling the list of training files')
data_parse.add_argument('--num_cores', dest='num_cores',
default=24, type=int, help='Number of CPU cores')
data_parse.add_argument('--train_ratio', dest='train_ratio', default=0.7,
type=float, help='Fraction of data to be used for training')
data_parse.add_argument('--valid_ratio', dest='valid_ratio', default=0.15, type=float,
help='Fraction of data to be used for validation (only matters when there is a third dataset to be created for testing)')
data_parse.add_argument('--batch_size', dest='batch_size',
default=32, type=int, help='Number of examples per batch')
data_parse.add_argument('--prefetch_size', dest='prefetch_size', default=1,
type=int, help='Number of batches to be prepared in queue')
# Model
model_parse = parser.add_argument_group('Model setting')
model_parse.add_argument('--model', dest='model', default='DNN',
type=str, help='Select the model to train e.g. DNN; note that this version only has DNN')
model_parse.add_argument('--loss', dest='loss', default=40,
type=int, help="Setting of loss function '10','11','12','20','21','22','30','31','32','40'")
model_parse.add_argument('--hidden_units', dest='hidden_units', default=[
128, 64], type=int, nargs='+', help='List containing the number of hidden units to use for each hidden layer')
model_parse.add_argument('--learning_rate', dest='learning_rate',
default=0.001, type=float, help='Learning rate of updating gradient')
model_parse.add_argument('--decay_step', dest='decay_step',
default=100, type=int, help='Decay step')
model_parse.add_argument('--decay_rate', dest='decay_rate', default=0.98,
type=float, help='Decay rate for exponential decay of learning rate')
model_parse.add_argument('--Lambda', dest='Lambda', default=0.25,
type=float, help='Lambda for L2,L1 regularization; alpha for focal loss')
model_parse.add_argument('--gamma', dest='gamma',
default=2., type=float, help='parameter for focal loss')
model_parse.add_argument('--beta', dest='beta',
default=1., type=float, help='Regularization parameter')
model_parse.add_argument('--drop_rate', dest='drop_rate',
default=0.5, type=float, help='dropout rate')
model_parse.add_argument('--embedding_units', dest='embedding_units', default=[1, 35, 359, 3, 2], type=int, nargs='+',
help='List containing the number of embedding units to use for features (in order): [weekday, region, city, adexchange, slotformat]; this replaces the one hot encoding')
model_parse.add_argument('--embedding_units_ohe', dest='embedding_units_ohe', default=[
45], type=int, nargs='+', help='List containing the number of embedding units to use for OHE features (in order): usertag')
# Training
train_parse = parser.add_argument_group('Training hyperparameters')
train_parse.add_argument('--has_gpu', dest='has_gpu',
default=0, type=int, help='1 if GPU is present, else 0')
train_parse.add_argument('--is_test', dest='is_test', default=0,
type=int, help='1 if the trained model will be evaluated, else 0')
train_parse.add_argument('--num_epochs_min', dest='num_epochs_min',
default=100, type=float, help='Minimum number of training epochs')
train_parse.add_argument('--num_epochs', dest='num_epochs',
default=101, type=float, help='Number of total training epochs')
train_parse.add_argument('--validation_length', dest='validation_length', default=100,
type=int, help='In one validation, how many number of batches to use')
train_parse.add_argument('--test_length', dest='test_length',
default=100, type=int, help='In one test, how many number of batches to use')
train_parse.add_argument('--earlystop_check_frequency', dest='earlystop_check_frequency',
default=10, type=int, help='earlystop_check_frequency')
train_parse.add_argument('--earlystop_duration', dest='earlystop_duration',
default=10, type=int, help='earlystop_duration')
train_parse.add_argument('--valid_loss_delta', dest='valid_loss_delta',
default=0.0001, type=float, help='valid_loss_delta')
train_parse.add_argument('--num_threshold_buffer', dest='num_threshold_buffer',
default=3, type=int, help='num_threshold_buffer')
train_parse.add_argument('--percentile_threshold', dest='percentile_threshold',
default=8, type=int, help='percentile_threshold')
# Directory paths
dir_parse = parser.add_argument_group('Directory paths')
dir_parse.add_argument('--save_dir', dest='save_dir',
default='./Outputs/', type=str, help='Directory to save model directories')
dir_parse.add_argument('--load_dir', dest='load_dir', default='latest',
type=str, help='Directory to load old model,default "new" as the latest model')
dir_parse.add_argument('--store_dir', dest='store_dir', default='latest',
type=str, help='Directory to store current model, default "latest" to save in timestamp')
dir_parse.add_argument('--result_dir', dest='result_dir', default='result.csv',
type=str, help='Directory to store (history) performance result')
dir_parse.add_argument('--builder_save_dir', dest='builder_save_dir', default='builder_save',
type=str, help='Directory to store current model for tfjs predictor')
_args, _ = parser.parse_known_args()
return vars(_args)
def add_param_arguments(
ap=None,
arg_default=None
): # arguments with possible format-specific parameter values
def add_argument(a, *args, **kwargs):
return a.add_argument(*args, **kwargs)
def add(a, *args, default=None, func=add_argument, **kwargs):
arg = func(a,
*args,
default=default if arg_default is None else arg_default,
**kwargs)
try:
RESTORED_ARGS.add(arg.dest)
except AttributeError:
RESTORED_ARGS.update(get_group_arg_names(arg))
def add_boolean(a, *args, **kwargs):
add(a, *args, func=add_boolean_option, **kwargs)
if not ap:
ap = ArgParser()
group = ap.add_argument_group(title="Node labels")
add(group,
"--max-node-labels",
type=int,
default=0,
help="max number of node labels to allow")
add(group,
"--max-node-categories",
type=int,
default=0,
help="max node categories to allow")
add(group,
"--min-node-label-count",
type=int,
default=2,
help="min number of occurrences for a label")
add_boolean(group, "use-gold-node-labels", "gold node labels when parsing")
add_boolean(group, "wikification",
"use Spotlight to wikify any named node")
add_boolean(group,
"node-labels",
"prediction of node labels, if supported by format",
default=True)
group = ap.add_argument_group(title="Structural constraints")
add_boolean(group, "linkage", "linkage nodes and edges")
add_boolean(group, "implicit", "implicit nodes and edges")
add_boolean(group, "remote", "remote edges", default=True)
add_boolean(group, "constraints", "scheme-specific rules", default=True)
add_boolean(group, "require-connected",
"constraint that output graph must be connected")
add(group,
"--orphan-label",
default="orphan",
help="edge label to use for nodes without parents")
add(group,
"--max-action-ratio",
type=float,
default=100,
help="max action/terminal ratio")
add(group,
"--max-node-ratio",
type=float,
default=10,
help="max node/terminal ratio")
add(group, "--max-height", type=int, default=20, help="max graph height")
group = ap.add_mutually_exclusive_group()
add(group,
"--swap",
choices=(REGULAR, COMPOUND),
default=REGULAR,
help="swap transitions")
add(group,
"--no-swap",
action="store_false",
dest="swap",
help="exclude swap transitions")
add(ap,
"--max-swap",
type=int,
default=15,
help="if compound swap enabled, maximum swap size")
group = ap.add_argument_group(
title="General classifier training parameters")
add(group,
"--learning-rate",
type=float,
help="rate for model weight updates (default: by trainer/1)")
add(group,
"--learning-rate-decay",
type=float,
default=0,
help="learning rate decay per iteration")
add(group,
"--swap-importance",
type=float,
default=1,
help="learning rate factor for Swap")
add(group,
"--max-training-per-format",
type=int,
help="max number of training passages per format per iteration")
add_boolean(group,
"missing-node-features",
"allow node features to be missing if not available",
default=True)
add(group,
"--omit-features",
help="string of feature properties to omit, out of " +
FEATURE_PROPERTIES)
add_boolean(
group, "curriculum",
"sort training passages by action prediction accuracy in previous epoch"
)
group = ap.add_argument_group(title="Perceptron parameters")
add(group,
"--min-update",
type=int,
default=5,
help="minimum #updates for using a feature")
SPARSE_ARG_NAMES.update(get_group_arg_names(group))
group = ap.add_argument_group(title="Neural network parameters")
add(group,
"--word-dim-external",
type=int,
default=300,
help="dimension for external word embeddings")
add(group,
"--word-vectors",
help="file to load external word embeddings from (default: GloVe)")
add(group,
"--vocab",
help=
"file mapping integer ID to word form (to avoid loading spaCy), or '-' to use word form"
)
add_boolean(group,
"update-word-vectors",
"external word vectors in training parameters",
default=True)
add(group,
"--word-dim",
type=int,
default=0,
help="dimension for learned word embeddings")
add(group,
"--lemma-dim",
type=int,
default=200,
help="dimension for lemma embeddings")
add(group,
"--tag-dim",
type=int,
default=20,
help="dimension for fine POS tag embeddings")
add(group,
"--pos-dim",
type=int,
default=20,
help="dimension for coarse/universal POS tag embeddings")
add(group,
"--dep-dim",
type=int,
default=10,
help="dimension for dependency relation embeddings")
add(group,
"--edge-label-dim",
type=int,
default=20,
help="dimension for edge label embeddings")
add(group,
"--node-label-dim",
type=int,
default=0,
help="dimension for node label embeddings")
add(group,
"--node-category-dim",
type=int,
default=0,
help="dimension for node category embeddings")
add(group,
"--punct-dim",
type=int,
default=1,
help="dimension for separator punctuation embeddings")
add(group,
"--action-dim",
type=int,
default=3,
help="dimension for input action type embeddings")
add(group,
"--ner-dim",
type=int,
default=3,
help="dimension for input entity type embeddings")
add(group,
"--shape-dim",
type=int,
default=3,
help="dimension for word shape embeddings")
add(group,
"--prefix-dim",
type=int,
default=2,
help="dimension for word prefix embeddings")
add(group,
"--suffix-dim",
type=int,
default=3,
help="dimension for word suffix embeddings")
add(group,
"--output-dim",
type=int,
default=50,
help="dimension for output action embeddings")
add(group,
"--layer-dim",
type=int,
default=50,
help="dimension for hidden layers")
add(group, "--layers", type=int, default=2, help="number of hidden layers")
add(group,
"--lstm-layer-dim",
type=int,
default=500,
help="dimension for LSTM hidden layers")
add(group,
"--lstm-layers",
type=int,
default=0,
help="number of LSTM hidden layers")
add(group,
"--embedding-layer-dim",
type=int,
default=500,
help="dimension for layers before LSTM")
add(group,
"--embedding-layers",
type=int,
default=1,
help="number of layers before LSTM")
add(group,
"--activation",
choices=ACTIVATIONS,
default=DEFAULT_ACTIVATION,
help="activation function")
add(group,
"--init",
choices=INITIALIZERS,
default=DEFAULT_INITIALIZER,
help="weight initialization")
add(group,
"--minibatch-size",
type=int,
default=200,
help="mini-batch size for optimization")
add(group,
"--optimizer",
choices=TRAINERS,
default=DEFAULT_TRAINER,
help="algorithm for optimization")
add(group,
"--loss",
choices=LOSSES,
default=DEFAULT_LOSS,
help="loss function for training")
add(group,
"--max-words-external",
type=int,
default=250000,
help="max external word vectors to use")
add(group,
"--max-words",
type=int,
default=10000,
help="max number of words to keep embeddings for")
add(group,
"--max-lemmas",
type=int,
default=3000,
help="max number of lemmas to keep embeddings for")
add(group,
"--max-tags",
type=int,
default=100,
help="max number of fine POS tags to keep embeddings for")
add(group,
"--max-pos",
type=int,
default=100,
help="max number of coarse POS tags to keep embeddings for")
add(group,
"--max-deps",
type=int,
default=100,
help="max number of dep labels to keep embeddings for")
add(group,
"--max-edge-labels",
type=int,
default=15,
help="max number of edge labels for embeddings")
add(group,
"--max-puncts",
type=int,
default=5,
help="max number of punctuations for embeddings")
add(group,
"--max-action-types",
type=int,
default=10,
help="max number of action types for embeddings")
add(group,
"--max-action-labels",
type=int,
default=100,
help="max number of action labels to allow")
add(group,
"--max-ner-types",
type=int,
default=18,
help="max number of entity types to allow")
add(group,
"--max-shapes",
type=int,
default=30,
help="max number of word shapes to allow")
add(group,
"--max-prefixes",
type=int,
default=30,
help="max number of 1-character word prefixes to allow")
add(group,
"--max-suffixes",
type=int,
default=500,
help="max number of 3-character word suffixes to allow")
add(group,
"--word-dropout",
type=float,
default=0.2,
help="word dropout parameter")
add(group,
"--word-dropout-external",
type=float,
default=0,
help="word dropout for word vectors")
add(group,
"--lemma-dropout",
type=float,
default=0.2,
help="lemma dropout parameter")
add(group,
"--tag-dropout",
type=float,
default=0.2,
help="fine POS tag dropout parameter")
add(group,
"--pos-dropout",
type=float,
default=0.2,
help="coarse POS tag dropout parameter")
add(group,
"--dep-dropout",
type=float,
default=0.5,
help="dependency label dropout parameter")
add(group,
"--node-label-dropout",
type=float,
default=0.2,
help="node label dropout parameter")
add(group,
"--node-dropout",
type=float,
default=0.1,
help="probability to drop features for a whole node")
add(group,
"--dropout",
type=float,
default=0.4,
help="dropout parameter between layers")
add(group,
"--max-length",
type=int,
default=120,
help="maximum length of input sentence")
add(group,
"--rnn",
choices=["None"] + list(RNNS),
default=DEFAULT_RNN,
help="type of recurrent neural network")
add(group,
"--gated",
type=int,
nargs="?",
default=2,
help="gated input to BiRNN and MLP")
NN_ARG_NAMES.update(get_group_arg_names(group))
return ap
def add_param_arguments(ap=None, arg_default=None): # arguments with possible format-specific parameter values
def add_argument(a, *args, **kwargs):
return a.add_argument(*args, **kwargs)
def add(a, *args, default=None, func=add_argument, **kwargs):
arg = func(a, *args, default=default if arg_default is None else arg_default, **kwargs)
try:
RESTORED_ARGS.add(arg.dest)
except AttributeError:
RESTORED_ARGS.update(get_group_arg_names(arg))
def add_boolean(a, *args, **kwargs):
add(a, *args, func=add_boolean_option, **kwargs)
if not ap:
ap = ArgParser()
group = ap.add_argument_group(title="Node labels")
add(group, "--max-node-labels", type=int, default=0, help="max number of node labels to allow")
add(group, "--max-node-categories", type=int, default=0, help="max node categories to allow")
add(group, "--min-node-label-count", type=int, default=2, help="min number of occurrences for a label")
add_boolean(group, "use-gold-node-labels", "gold node labels when parsing")
add_boolean(group, "wikification", "use Spotlight to wikify any named node")
add_boolean(group, "node-labels", "prediction of node labels, if supported by format", default=True)
group = ap.add_argument_group(title="Structural constraints")
add_boolean(group, "linkage", "linkage nodes and edges")
add_boolean(group, "implicit", "implicit nodes and edges")
add_boolean(group, "remote", "remote edges", default=True)
add_boolean(group, "constraints", "scheme-specific rules", default=True)
add_boolean(group, "require-connected", "constraint that output graph must be connected")
add(group, "--orphan-label", default="orphan", help="edge label to use for nodes without parents")
add(group, "--max-action-ratio", type=float, default=100, help="max action/terminal ratio")
add(group, "--max-node-ratio", type=float, default=10, help="max node/terminal ratio")
add(group, "--max-height", type=int, default=20, help="max graph height")
group = ap.add_mutually_exclusive_group()
add(group, "--swap", choices=(REGULAR, COMPOUND), default=REGULAR, help="swap transitions")
add(group, "--no-swap", action="store_false", dest="swap", help="exclude swap transitions")
add(ap, "--max-swap", type=int, default=15, help="if compound swap enabled, maximum swap size")
group = ap.add_argument_group(title="General classifier training parameters")
add(group, "--learning-rate", type=float, help="rate for model weight updates (default: by trainer/1)")
add(group, "--learning-rate-decay", type=float, default=0, help="learning rate decay per iteration")
add(group, "--swap-importance", type=float, default=1, help="learning rate factor for Swap")
add(group, "--max-training-per-format", type=int, help="max number of training passages per format per iteration")
add_boolean(group, "missing-node-features", "allow node features to be missing if not available", default=True)
add(group, "--omit-features", help="string of feature properties to omit, out of " + FEATURE_PROPERTIES)
add_boolean(group, "curriculum", "sort training passages by action prediction accuracy in previous epoch")
group = ap.add_argument_group(title="Perceptron parameters")
add(group, "--min-update", type=int, default=5, help="minimum #updates for using a feature")
SPARSE_ARG_NAMES.update(get_group_arg_names(group))
group = ap.add_argument_group(title="Neural network parameters")
add(group, "--word-dim-external", type=int, default=300, help="dimension for external word embeddings")
add(group, "--word-vectors", help="file to load external word embeddings from (default: GloVe)")
add(group, "--vocab", help="file mapping integer ID to word form (to avoid loading spaCy), or '-' to use word form")
add_boolean(group, "update-word-vectors", "external word vectors in training parameters", default=True)
add(group, "--word-dim", type=int, default=0, help="dimension for learned word embeddings")
add(group, "--lemma-dim", type=int, default=200, help="dimension for lemma embeddings")
add(group, "--tag-dim", type=int, default=20, help="dimension for fine POS tag embeddings")
add(group, "--pos-dim", type=int, default=20, help="dimension for coarse/universal POS tag embeddings")
add(group, "--dep-dim", type=int, default=10, help="dimension for dependency relation embeddings")
add(group, "--edge-label-dim", type=int, default=20, help="dimension for edge label embeddings")
add(group, "--node-label-dim", type=int, default=0, help="dimension for node label embeddings")
add(group, "--node-category-dim", type=int, default=0, help="dimension for node category embeddings")
add(group, "--punct-dim", type=int, default=1, help="dimension for separator punctuation embeddings")
add(group, "--action-dim", type=int, default=3, help="dimension for input action type embeddings")
add(group, "--ner-dim", type=int, default=3, help="dimension for input entity type embeddings")
add(group, "--shape-dim", type=int, default=3, help="dimension for word shape embeddings")
add(group, "--prefix-dim", type=int, default=2, help="dimension for word prefix embeddings")
add(group, "--suffix-dim", type=int, default=3, help="dimension for word suffix embeddings")
add(group, "--output-dim", type=int, default=50, help="dimension for output action embeddings")
add(group, "--layer-dim", type=int, default=50, help="dimension for hidden layers")
add(group, "--layers", type=int, default=2, help="number of hidden layers")
add(group, "--lstm-layer-dim", type=int, default=500, help="dimension for LSTM hidden layers")
add(group, "--lstm-layers", type=int, default=0, help="number of LSTM hidden layers")
add(group, "--embedding-layer-dim", type=int, default=500, help="dimension for layers before LSTM")
add(group, "--embedding-layers", type=int, default=1, help="number of layers before LSTM")
add(group, "--activation", choices=ACTIVATIONS, default=DEFAULT_ACTIVATION, help="activation function")
add(group, "--init", choices=INITIALIZERS, default=DEFAULT_INITIALIZER, help="weight initialization")
add(group, "--minibatch-size", type=int, default=200, help="mini-batch size for optimization")
add(group, "--optimizer", choices=TRAINERS, default=DEFAULT_TRAINER, help="algorithm for optimization")
add(group, "--loss", choices=LOSSES, default=DEFAULT_LOSS, help="loss function for training")
add(group, "--max-words-external", type=int, default=250000, help="max external word vectors to use")
add(group, "--max-words", type=int, default=10000, help="max number of words to keep embeddings for")
add(group, "--max-lemmas", type=int, default=3000, help="max number of lemmas to keep embeddings for")
add(group, "--max-tags", type=int, default=100, help="max number of fine POS tags to keep embeddings for")
add(group, "--max-pos", type=int, default=100, help="max number of coarse POS tags to keep embeddings for")
add(group, "--max-deps", type=int, default=100, help="max number of dep labels to keep embeddings for")
add(group, "--max-edge-labels", type=int, default=15, help="max number of edge labels for embeddings")
add(group, "--max-puncts", type=int, default=5, help="max number of punctuations for embeddings")
add(group, "--max-action-types", type=int, default=10, help="max number of action types for embeddings")
add(group, "--max-action-labels", type=int, default=100, help="max number of action labels to allow")
add(group, "--max-ner-types", type=int, default=18, help="max number of entity types to allow")
add(group, "--max-shapes", type=int, default=30, help="max number of word shapes to allow")
add(group, "--max-prefixes", type=int, default=30, help="max number of 1-character word prefixes to allow")
add(group, "--max-suffixes", type=int, default=500, help="max number of 3-character word suffixes to allow")
add(group, "--word-dropout", type=float, default=0.2, help="word dropout parameter")
add(group, "--word-dropout-external", type=float, default=0, help="word dropout for word vectors")
add(group, "--lemma-dropout", type=float, default=0.2, help="lemma dropout parameter")
add(group, "--tag-dropout", type=float, default=0.2, help="fine POS tag dropout parameter")
add(group, "--pos-dropout", type=float, default=0.2, help="coarse POS tag dropout parameter")
add(group, "--dep-dropout", type=float, default=0.5, help="dependency label dropout parameter")
add(group, "--node-label-dropout", type=float, default=0.2, help="node label dropout parameter")
add(group, "--node-dropout", type=float, default=0.1, help="probability to drop features for a whole node")
add(group, "--dropout", type=float, default=0.4, help="dropout parameter between layers")
add(group, "--max-length", type=int, default=120, help="maximum length of input sentence")
add(group, "--rnn", choices=["None"] + list(RNNS), default=DEFAULT_RNN, help="type of recurrent neural network")
add(group, "--gated", type=int, nargs="?", default=2, help="gated input to BiRNN and MLP")
NN_ARG_NAMES.update(get_group_arg_names(group))
return ap
def main():
"""See module docstring at the top of this file"""
parser = ArgParser(description='Make a plot of the sky to plan and evaluate tracking')
parser.add_argument(
'--tle-filename',
help='filename of two-line element (TLE) target ephemeris'
)
parser.add_argument(
'--time-start',
help=('UTC start time of 24-hour window in which the first above-horizon pass will be '
' plotted. Many natural language date formats are supported. If not specified, the '
'current time will be used.'
),
type=str)
parser.add_argument(
'--axis-west-limit',
help='western limit for the right ascension axis in degrees from the meridian',
default=110,
type=float)
parser.add_argument(
'--axis-east-limit',
help='eastern limit for the right ascension axis in degrees from the meridian',
default=110,
type=float)
custom_model_group = parser.add_argument_group(
title='Custom Mount Model Options',
description=('Set all of these to use a custom mount model instead of a stored model. '
'If no GPS is connected, also set the Observer Location Options.'),
)
custom_model_group.add_argument(
'--mount-pole-az',
help='azimuth of the mount pole',
type=float)
custom_model_group.add_argument(
'--mount-pole-alt',
help='altitude of the mount pole',
type=float)
gps_client.add_program_arguments(
parser=parser,
group_description=(
'Set all of these to indicate observer location with a custom mount model'
),
)
args = parser.parse_args()
custom_model_args = (args.mount_pole_az, args.mount_pole_alt)
observer_args = (args.lat, args.lon, args.elevation)
if args.time_start is not None:
time_start = dateutil.parser.parse(args.time_start)
time_start = time_start.replace(tzinfo=dateutil.tz.tzutc())
else:
time_start = datetime.utcnow()
time_stop = time_start + timedelta(days=1)
if all(arg is None for arg in custom_model_args):
if any(arg is not None for arg in observer_args):
# pylint: disable=not-callable
parser.error('Observer location options require the custom mount model args to be set')
mount_model = track.model.load_stored_model(max_age=None)
elif any(arg is None for arg in custom_model_args):
# pylint: disable=not-callable
parser.error("Must give all args in the custom mount model group or none of them.")
else:
# Generate a custom mount model from program arguments
location = gps_client.make_location_from_args(args)
mount_model = track.model.load_default_model(
mount_pole_az=Longitude(args.mount_pole_az*u.deg),
mount_pole_alt=Longitude(args.mount_pole_alt*u.deg),
location=location,
)
ax = make_sky_plot()
plot_reachable_zone(
ax,
mount_model,
axis_0_west_limit=args.axis_west_limit,
axis_0_east_limit=args.axis_east_limit
)
if args.tle_filename:
print('Searching for first pass within the following time range:')
print(time_start.isoformat() + 'Z')
print(time_stop.isoformat() + 'Z')
time_rise, time_set = plot_tle(
ax,
args.tle_filename,
mount_model.location,
time_start,
time_stop
)
if all((time_rise, time_set)):
plot_mount_motion(ax, time_rise, time_set)
print('Found a pass with these start and end times:')
print(time_rise.isoformat() + 'Z')
print(time_set.isoformat() + 'Z')
else:
print('No pass was found.')
plt.legend(
bbox_to_anchor=(0.8, 1.4),
loc='upper left',
borderaxespad=0,
)
# Shrink current axis by 20% so the legend will fit inside the figure window because matplotlib
# developers apparently don't use their own code
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height * 0.8])
plt.show()
