def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Train the specified model on the specified dataset.'''
subparser = parser.add_parser(name, description=description, help='Train a model')
subparser.add_argument('param_path',
type=str,
help='path to parameter file describing the model to be trained')
subparser.add_argument('-s', '--serialization-dir',
required=True,
type=str,
help='directory in which to save the model and its logs')
subparser.add_argument('-r', '--recover',
action='store_true',
default=False,
help='recover training from the state in serialization_dir')
subparser.add_argument('-o', '--overrides',
type=str,
default="",
help='a HOCON structure used to override the experiment configuration')
subparser.add_argument('--file-friendly-logging',
action='store_true',
default=False,
help='outputs tqdm status on separate lines and slows tqdm refresh rate')
subparser.set_defaults(func=train_model_from_args)
return subparser
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Run the specified model against a JSON-lines input file.'''
subparser = parser.add_parser(
name, description=description, help='Use a trained model to make predictions.')
subparser.add_argument('archive_file', type=str, help='the archived model to make predictions with')
subparser.add_argument('input_file', type=argparse.FileType('r'), help='path to input file')
subparser.add_argument('--output-file', type=argparse.FileType('w'), help='path to output file')
subparser.add_argument('--weights-file',
type=str,
help='a path that overrides which weights file to use')
batch_size = subparser.add_mutually_exclusive_group(required=False)
batch_size.add_argument('--batch-size', type=int, default=1, help='The batch size to use for processing')
subparser.add_argument('--silent', action='store_true', help='do not print output to stdout')
cuda_device = subparser.add_mutually_exclusive_group(required=False)
cuda_device.add_argument('--cuda-device', type=int, default=-1, help='id of GPU to use (if any)')
subparser.add_argument('-o', '--overrides',
type=str,
default="",
help='a HOCON structure used to override the experiment configuration')
subparser.add_argument('--predictor',
type=str,
help='optionally specify a specific predictor to use')
subparser.set_defaults(func=_predict)
return subparser
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Create word vectors using ELMo.'''
subparser = parser.add_parser(
name, description=description, help='Use a trained model to make predictions.')
subparser.add_argument('input_file', type=argparse.FileType('r'), help='The path to the input file.')
subparser.add_argument('output_file', type=str, help='The path to the output file.')
group = subparser.add_mutually_exclusive_group(required=True)
group.add_argument('--all', action='store_true', help='Output all three ELMo vectors.')
group.add_argument('--top', action='store_true', help='Output the top ELMo vector.')
group.add_argument('--average', action='store_true', help='Output the average of the ELMo vectors.')
subparser.add_argument('--vocab-path', type=str, help='A path to a vocabulary file to generate.')
subparser.add_argument(
'--options-file',
type=str,
default=DEFAULT_OPTIONS_FILE,
help='The path to the ELMo options file.')
subparser.add_argument(
'--weight-file',
type=str,
default=DEFAULT_WEIGHT_FILE,
help='The path to the ELMo weight file.')
subparser.add_argument('--batch-size', type=int, default=DEFAULT_BATCH_SIZE, help='The batch size to use.')
subparser.add_argument('--cuda-device', type=int, default=-1, help='The cuda_device to run on.')
subparser.set_defaults(func=elmo_command)
return subparser
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Evaluate the specified model + dataset'''
subparser = parser.add_parser(
name, description=description, help='Evaluate the specified model + dataset')
subparser.add_argument('archive_file', type=str, help='path to an archived trained model')
subparser.add_argument('input_file', type=str, help='path to the file containing the evaluation data')
subparser.add_argument('--output-file', type=str, help='path to output file')
subparser.add_argument('--weights-file',
type=str,
help='a path that overrides which weights file to use')
cuda_device = subparser.add_mutually_exclusive_group(required=False)
cuda_device.add_argument('--cuda-device',
type=int,
default=-1,
help='id of GPU to use (if any)')
subparser.add_argument('-o', '--overrides',
type=str,
default="",
help='a JSON structure used to override the experiment configuration')
subparser.add_argument('--batch-weight-key',
type=str,
default="",
help='If non-empty, name of metric used to weight the loss on a per-batch basis.')
subparser.set_defaults(func=evaluate_from_args)
return subparser
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Generate a configuration stub for a specific class (or for config as a whole)'''
subparser = parser.add_parser(
name, description=description, help='Generate configuration stubs.')
subparser.add_argument('cla55', nargs='?', default='', metavar='class')
subparser.set_defaults(func=_configure)
return subparser
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Run the web service, which provides an HTTP API as well as a web demo.'''
subparser = parser.add_parser(
name, description=description, help='Run the web service and demo.')
subparser.add_argument('--port', type=int, default=8000)
subparser.set_defaults(func=_serve)
return subparser
def register_for_trigger(cls, subparser: _SubParsersAction, *args, **kwargs):
"""
Registers the plugin as an option to run for the trigger
:param subparser: the subparser on which to register
:param args: additional arguments
:param kwargs: additional keyword arguments
:return the parser created by registering, to allow subclasses to register options when running
"""
parser = subparser.add_parser(cls.__name__.lower(), help=cls.help)
parser.set_defaults(main_plugin=cls()) # pylint: disable=abstract-class-instantiated
return parser
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Test that installation works by running the unit tests.'''
subparser = parser.add_parser(
name, description=description, help='Run the unit tests.')
subparser.add_argument('--run-all', action="store_true",
help="By default, we skip tests that are slow "
"or download large files. This flag will run all tests.")
subparser.set_defaults(func=_run_test)
return subparser
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = """Continues training a saved model on a new dataset."""
subparser = parser.add_parser(name,
description=description,
help='Continue training a model on a new dataset')
subparser.add_argument('-m', '--model-archive',
required=True,
type=str,
help='path to the saved model archive from training on the original data')
subparser.add_argument('-c', '--config-file',
required=True,
type=str,
help='configuration file to use for training. Format is the same as '
'for the "train" command, but the "model" section is ignored.')
subparser.add_argument('-s', '--serialization-dir',
required=True,
type=str,
help='directory in which to save the fine-tuned model and its logs')
subparser.add_argument('-o', '--overrides',
type=str,
default="",
help='a JSON structure used to override the training configuration '
'(only affects the config_file, _not_ the model_archive)')
subparser.add_argument('--extend-vocab',
action='store_true',
default=False,
help='if specified, we will use the instances in your new dataset to '
'extend your vocabulary. Currently expansion of embedding layers '
'is not implemented, so if your model has an embedding layer '
'this will probably make fine-tune crash.')
subparser.add_argument('--file-friendly-logging',
action='store_true',
default=False,
help='outputs tqdm status on separate lines and slows tqdm refresh rate')
subparser.add_argument('--batch-weight-key',
type=str,
default="",
help='If non-empty, name of metric used to weight the loss on a per-batch basis.')
subparser.set_defaults(func=fine_tune_model_from_args)
return subparser
def add_parser(self, subparsers: argparse._SubParsersAction) -> None:
# If get_help() returns None, do not pass in a help argument at all.
# This will prevent the command from appearing in the help output at
# all.
kwargs: Dict[str, Any] = {"aliases": self.get_aliases()}
help = self.get_help()
if help is not None:
# The add_parser() code checks if 'help' is present in the keyword
# arguments. Not being present is handled differently than if it
# is present and None. It only hides the command from the help
# output if the 'help' argument is not present at all.
kwargs["help"] = help
parser = subparsers.add_parser(self.get_name(), **kwargs)
parser.set_defaults(func=self.run)
self.setup_parser(parser)
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Find a learning rate range where loss decreases quickly
for the specified model and dataset.'''
subparser = parser.add_parser(name,
description=description,
help='Find a learning rate range.')
subparser.add_argument('param_path',
type=str,
help='path to parameter file describing the model to be trained')
subparser.add_argument('-s', '--serialization-dir',
required=True,
type=str,
help='The directory in which to save results.')
subparser.add_argument('-o', '--overrides',
type=str,
default="",
help='a JSON structure used to override the experiment configuration')
subparser.add_argument('--start-lr',
type=float,
default=1e-5,
help='learning rate to start the search')
subparser.add_argument('--end-lr',
type=float,
default=10,
help='learning rate up to which search is done')
subparser.add_argument('--num-batches',
type=int,
default=100,
help='number of mini-batches to run learning rate finder')
subparser.add_argument('--stopping-factor',
type=float,
default=None,
help='stop the search when the current loss exceeds the best loss recorded by '
'multiple of stopping factor')
subparser.add_argument('--linear',
action='store_true',
help='increase learning rate linearly instead of exponential increase')
subparser.add_argument('-f', '--force',
action='store_true',
required=False,
help='overwrite the output directory if it exists')
subparser.set_defaults(func=find_learning_rate_from_args)
return subparser
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Create word vectors using ELMo.'''
subparser = parser.add_parser(
name, description=description, help='Create word vectors using a pretrained ELMo model.')
subparser.add_argument('input_file', type=argparse.FileType('r'), help='The path to the input file.')
subparser.add_argument('output_file', type=str, help='The path to the output file.')
group = subparser.add_mutually_exclusive_group(required=True)
group.add_argument('--all', action='store_true', help='Output all three ELMo vectors.')
group.add_argument('--top', action='store_true', help='Output the top ELMo vector.')
group.add_argument('--average', action='store_true', help='Output the average of the ELMo vectors.')
subparser.add_argument('--vocab-path', type=str, help='A path to a vocabulary file to generate.')
subparser.add_argument(
'--options-file',
type=str,
default=DEFAULT_OPTIONS_FILE,
help='The path to the ELMo options file.')
subparser.add_argument(
'--weight-file',
type=str,
default=DEFAULT_WEIGHT_FILE,
help='The path to the ELMo weight file.')
subparser.add_argument('--batch-size', type=int, default=DEFAULT_BATCH_SIZE, help='The batch size to use.')
subparser.add_argument('--file-friendly-logging', default=False, action='store_true',
help='outputs tqdm status on separate lines and slows tqdm refresh rate.')
subparser.add_argument('--cuda-device', type=int, default=-1, help='The cuda_device to run on.')
subparser.add_argument(
'--forget-sentences',
action='store_true',
help="If this flag is specified, and --use-sentence-keys is "
"not, remove the string serialized JSON dictionary "
"that associates sentences with their line number (its "
"HDF5 key) that is normally placed in the "
"\"sentence_to_index\" HDF5 key.")
subparser.add_argument(
'--use-sentence-keys',
action='store_true',
help="Normally a sentence's line number is used as the "
"HDF5 key for its embedding. If this flag is specified, "
"the sentence itself will be used as the key.")
subparser.set_defaults(func=elmo_command)
return subparser
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Create a vocabulary from the specified dataset.'''
subparser = parser.add_parser(
name, description=description, help='Create a vocabulary')
subparser.add_argument('param_path',
type=str,
help='path to parameter file describing the model and its inputs')
subparser.add_argument('-o', '--overrides',
type=str,
default="",
help='a HOCON structure used to override the experiment configuration')
subparser.set_defaults(func=make_vocab_from_args)
return subparser
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Create a vocabulary, compute dataset statistics and other training utilities.'''
subparser = parser.add_parser(name,
description=description,
help='Create a vocabulary, compute dataset statistics '
'and other training utilities.')
subparser.add_argument('param_path',
type=str,
help='path to parameter file describing the model and its inputs')
subparser.add_argument('-s', '--serialization-dir',
required=True,
type=str,
help='directory in which to save the output of the dry run.')
subparser.add_argument('-o', '--overrides',
type=str,
default="",
help='a JSON structure used to override the experiment configuration')
subparser.set_defaults(func=dry_run_from_args)
return subparser
def add_subparser(self, name: str, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = """Continues training a saved model on a new dataset."""
subparser = parser.add_parser(name,
description=description,
help='Continue training a model on a new dataset')
subparser.add_argument('-m', '--model-archive',
required=True,
type=str,
help='path to the saved model archive from training on the original data')
subparser.add_argument('-c', '--config-file',
required=True,
type=str,
help='configuration file to use for training. Format is the same as '
'for the "train" command, but the "model" section is ignored.')
subparser.add_argument('-s', '--serialization-dir',
required=True,
type=str,
help='directory in which to save the fine-tuned model and its logs')
subparser.add_argument('-o', '--overrides',
type=str,
default="",
help='a JSON structure used to override the training configuration '
'(only affects the config_file, _not_ the model_archive)')
subparser.add_argument('--file-friendly-logging',
action='store_true',
default=False,
help='outputs tqdm status on separate lines and slows tqdm refresh rate')
subparser.set_defaults(func=fine_tune_model_from_args)
return subparser
def add_subparser(
self,
parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
description = """Train the specified model on the specified dataset."""
subparser = parser.add_parser(self.name,
description=description,
help="Train a model.")
subparser.add_argument(
"param_path",
type=str,
help="path to parameter file describing the model to be trained")
subparser.add_argument(
"-s",
"--serialization-dir",
required=True,
type=str,
help="directory in which to save the model and its logs",
)
subparser.add_argument(
"-r",
"--recover",
action="store_true",
default=False,
help="recover training from the state in serialization_dir",
)
subparser.add_argument(
"-f",
"--force",
action="store_true",
required=False,
help="overwrite the output directory if it exists",
)
subparser.add_argument(
"-o",
"--overrides",
type=str,
default="",
help=
"a JSON structure used to override the experiment configuration",
)
subparser.add_argument(
"--node-rank",
type=int,
default=0,
help="rank of this node in the distributed setup")
subparser.add_argument(
"--dry-run",
action="store_true",
help=
"do not train a model, but create a vocabulary, show dataset statistics and "
"other training information",
)
subparser.set_defaults(func=train_model_from_args)
return subparser
def add_subparser(
self,
parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
description = """Run the specified model against a JSON-lines input file."""
subparser = parser.add_parser(
self.name,
description=description,
help="Use a trained model to make predictions.")
subparser.add_argument(
"archive_file",
type=str,
help="the archived model to make predictions with")
subparser.add_argument("input_file",
type=str,
help="path to or url of the input file")
subparser.add_argument("--output-file",
type=str,
help="path to output file")
subparser.add_argument(
"--weights-file",
type=str,
help="a path that overrides which weights file to use")
batch_size = subparser.add_mutually_exclusive_group(required=False)
batch_size.add_argument("--batch-size",
type=int,
default=1,
help="The batch size to use for processing")
subparser.add_argument("--silent",
action="store_true",
help="do not print output to stdout")
cuda_device = subparser.add_mutually_exclusive_group(required=False)
cuda_device.add_argument("--cuda-device",
type=int,
default=-1,
help="id of GPU to use (if any)")
subparser.add_argument(
"--use-dataset-reader",
action="store_true",
help=
"Whether to use the dataset reader of the original model to load Instances. "
"The validation dataset reader will be used if it exists, otherwise it will "
"fall back to the train dataset reader. This behavior can be overridden "
"with the --dataset-reader-choice flag.",
)
subparser.add_argument(
"--dataset-reader-choice",
type=str,
choices=["train", "validation"],
default="validation",
help=
"Indicates which model dataset reader to use if the --use-dataset-reader "
"flag is set.",
)
subparser.add_argument(
"-o",
"--overrides",
type=str,
default="",
help=
"a JSON structure used to override the experiment configuration",
)
subparser.add_argument(
"--predictor",
type=str,
help="optionally specify a specific predictor to use")
subparser.set_defaults(func=_predict)
return subparser
def add_subparser(self, parser: argparse._SubParsersAction):
self.parser = parser.add_parser('duc2006')
self.parser.add_argument('data_root')
self.parser.add_argument('output_dir')
self.parser.set_defaults(subfunc=self.run)
def setup(subparser: argparse._SubParsersAction, dev: bool = False):
parser = subparser.add_parser("+entry",
help="エントリーを行います",
description="出展エントリーを行います。\n" +
"このコマンドは、エントリー期間のみ、イベントに出展予定の方が使用します。")
return parser
def add_subparser(self, parser: argparse._SubParsersAction):
self.parser = parser.add_parser('score')
self.parser.add_argument('config')
self.parser.add_argument('output_jsonl')
self.parser.add_argument('--overrides')
self.parser.set_defaults(func=self.run)
def add_subparser(
self, name: str,
parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Train the specified model on the specified dataset.'''
subparser = parser.add_parser(name,
description=description,
help='Train a model')
subparser.add_argument(
'param_path',
type=str,
help='path to parameter file describing the model to be trained')
subparser.add_argument(
'-s',
'--serialization-dir',
required=False,
default="",
type=str,
help='directory in which to save the model and its logs')
subparser.add_argument(
'-r',
'--recover',
action='store_true',
default=False,
help='recover training from the state in serialization_dir')
subparser.add_argument(
'-f',
'--force',
action='store_true',
required=False,
help='overwrite the output directory if it exists')
subparser.add_argument(
'-o',
'--overrides',
type=str,
default="",
help=
'a JSON structure used to override the experiment configuration')
subparser.add_argument('-e',
'--ext-vars',
type=str,
default=None,
help='Used to provide ext variable to jsonnet')
subparser.add_argument('--fp16',
action='store_true',
required=False,
help='use fp 16 training')
subparser.add_argument(
'--file-friendly-logging',
action='store_true',
default=False,
help=
'outputs tqdm status on separate lines and slows tqdm refresh rate'
)
subparser.set_defaults(func=train_model_from_args)
return subparser
def add_subparser(
subparsers: argparse._SubParsersAction, parents: List[argparse.ArgumentParser]
):
import rasa.nlu.convert as convert
data_parser = subparsers.add_parser(
"data",
conflict_handler="resolve",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
parents=parents,
help="Utils for the Rasa training files.",
)
data_parser.set_defaults(func=lambda _: data_parser.print_help(None))
data_subparsers = data_parser.add_subparsers()
convert_parser = data_subparsers.add_parser(
"convert",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
parents=parents,
help="Converts Rasa data between different formats.",
)
convert_parser.set_defaults(func=lambda _: convert_parser.print_help(None))
convert_subparsers = convert_parser.add_subparsers()
convert_nlu_parser = convert_subparsers.add_parser(
"nlu",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
parents=parents,
help="Converts NLU data between Markdown and json formats.",
)
convert_nlu_parser.set_defaults(func=convert.main)
arguments.set_convert_arguments(convert_nlu_parser)
split_parser = data_subparsers.add_parser(
"split",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
parents=parents,
help="Splits Rasa data into training and test data.",
)
split_parser.set_defaults(func=lambda _: split_parser.print_help(None))
split_subparsers = split_parser.add_subparsers()
nlu_split_parser = split_subparsers.add_parser(
"nlu",
parents=parents,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help="Performs a split of your NLU data into training and test data "
"according to the specified percentages.",
)
nlu_split_parser.set_defaults(func=split_nlu_data)
arguments.set_split_arguments(nlu_split_parser)
validate_parser = data_subparsers.add_parser(
"validate",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
parents=parents,
help="Validates domain and data files to check for possible mistakes.",
)
validate_parser.set_defaults(func=validate_files)
arguments.set_validator_arguments(validate_parser)
def add_parser(cls: type, subparsers: argparse._SubParsersAction):
parser = subparsers.add_parser(cls.COMMAND, help="Visualize selected entries")
cls.add_arguments(parser)
parser.set_defaults(func=cls.execute)
def add_wave_args(subparser: argparse._SubParsersAction):
"""Add arguments for wave {gen,load} function to ArgumentParser.
Parameters
----------
subparser : :class:`argparse._SubParsersAction`
SubParser to add other arguments related to wave_gen function.
"""
wave = subparser.add_parser("wave")
wave_functions = wave.add_subparsers(
title="Wave Functions",
dest="wave_function",
)
wave_gen = wave_functions.add_parser("gen")
wave_gen.add_argument(
"channel",
nargs="+",
choices=["SI1", "SI2"],
help="Pin(s) on which to generate a waveform",
)
wave_gen.add_argument(
"-f",
"--frequency",
nargs="+",
type=float,
required=True,
help="Frequency in Hz",
)
wave_gen.add_argument(
"-p",
"--phase",
type=float,
default=0,
required=False,
help="Phase between waveforms in degrees",
)
description = """
TABLE:
JSON array of voltage values which make up the waveform. Array length
must be 512. If the array length less than 512, then the array will be
expanded in length of 512. Values outside the range -3.3 V to 3.3 V
will be clipped.
examples:
[1,0] or [1,...,0,...],
[0,1,0,-1,0,1,0,-1,...],
[0,.025,.05,.075,.1,.125,.15,...]
"""
load = wave_functions.add_parser("load", description=description)
load.add_argument(
"channel",
choices=["SI1", "SI2"],
help="Pin(s) on which to load a table",
)
load_table = load.add_mutually_exclusive_group(required=True)
load_table.add_argument(
"--table",
type=json.loads,
default=None,
help="Table to load in pin SI1 as json",
)
load_table.add_argument(
"--table-file",
nargs="?",
type=str,
const=0,
default=None,
help="Table to load in pin SI1 as json file. Default is stdin",
)
def add_parser(cls: type, subparsers: argparse._SubParsersAction):
parser = subparsers.add_parser(cls.COMMAND, help="Dump model outputs to a file.")
cls.add_arguments(parser)
parser.set_defaults(func=cls.execute)
def add_subparser(self, parser: argparse._SubParsersAction):
description = 'Calculate the correlation between two different metrics'
self.parser = parser.add_parser('correlate',
description=description,
help=description)
self.parser.add_argument(
'--metrics-jsonl-files',
nargs='+',
help=
'The jsonl files with the metric values. If the values are split across multiple files, they can all '
'be passed as arguments.',
required=True)
self.parser.add_argument(
'--metrics',
nargs=2,
type=str,
help=
'The flattened names of the two metrics that should be correlated',
required=True)
self.parser.add_argument(
'--summarizer-type',
choices=['all', 'reference', 'peer'],
help=
'The type of summarizer which should be included in the correlation calculation',
required=True)
self.parser.add_argument(
'--confidence-interval-method',
choices=[
'none', 'bootstrap-system', 'bootstrap-input',
'bootstrap-both', 'fisher'
],
default='bootstrap-both',
help=
'The method to use to calculate the confidence intervals on the correlation coefficients'
)
self.parser.add_argument(
'--confidence',
type=float,
default=95,
help='The confidence level of the confidence intervals')
self.parser.add_argument(
'--num-tails',
type=int,
choices=[1, 2],
default=2,
help=
'The number of tails to use in the confidence interval calculation'
)
self.parser.add_argument(
'--confidence-interval-kwargs',
default='{}',
help=
'A serialized JSON string that will be parsed and passed as kwargs to the confidence interval calculation'
)
self.parser.add_argument(
'--output-file',
type=str,
help=
'The json output file which will contain the final correlations')
self.parser.add_argument(
'--log-file',
type=str,
help='The file where the log should be written')
self.parser.add_argument(
'--silent',
action='store_true',
help='Controls whether the log should be written to stdout')
self.parser.add_argument(
'--skip-summary-level',
action='store_true',
help=
'Indicates the summary-level correlations should not be calculated'
)
self.parser.add_argument(
'--skip-system-level',
action='store_true',
help=
'Indicates the system-level correlations should not be calculated')
self.parser.add_argument(
'--skip-global',
action='store_true',
help='Indicates the global correlations should not be calculated')
self.parser.add_argument(
'--system-level-output-plot',
type=str,
help=
'If provided, saves a plot of the system-level scores to the corresponding file'
)
self.parser.add_argument(
'--global-output-plot',
type=str,
help=
'If provided, saves a plot of the global scores to the corresponding file'
)
self.parser.set_defaults(func=self.run)
def _populate_sub_parser_by_class(
cls: Type[gitlab.base.RESTObject],
sub_parser: argparse._SubParsersAction) -> None:
mgr_cls_name = cls.__name__ + "Manager"
mgr_cls = getattr(gitlab.v4.objects, mgr_cls_name)
for action_name in ["list", "get", "create", "update", "delete"]:
if not hasattr(mgr_cls, action_name):
continue
sub_parser_action = sub_parser.add_parser(action_name)
sub_parser_action.add_argument("--sudo", required=False)
if mgr_cls._from_parent_attrs:
for x in mgr_cls._from_parent_attrs:
sub_parser_action.add_argument("--%s" % x.replace("_", "-"),
required=True)
if action_name == "list":
for x in mgr_cls._list_filters:
sub_parser_action.add_argument("--%s" % x.replace("_", "-"),
required=False)
sub_parser_action.add_argument("--page", required=False)
sub_parser_action.add_argument("--per-page", required=False)
sub_parser_action.add_argument("--all",
required=False,
action="store_true")
if action_name == "delete":
if cls._id_attr is not None:
id_attr = cls._id_attr.replace("_", "-")
sub_parser_action.add_argument("--%s" % id_attr, required=True)
if action_name == "get":
if not issubclass(cls, gitlab.mixins.GetWithoutIdMixin):
if cls._id_attr is not None:
id_attr = cls._id_attr.replace("_", "-")
sub_parser_action.add_argument("--%s" % id_attr,
required=True)
for x in mgr_cls._optional_get_attrs:
sub_parser_action.add_argument("--%s" % x.replace("_", "-"),
required=False)
if action_name == "create":
for x in mgr_cls._create_attrs.required:
sub_parser_action.add_argument("--%s" % x.replace("_", "-"),
required=True)
for x in mgr_cls._create_attrs.optional:
sub_parser_action.add_argument("--%s" % x.replace("_", "-"),
required=False)
if action_name == "update":
if cls._id_attr is not None:
id_attr = cls._id_attr.replace("_", "-")
sub_parser_action.add_argument("--%s" % id_attr, required=True)
for x in mgr_cls._update_attrs.required:
if x != cls._id_attr:
sub_parser_action.add_argument("--%s" %
x.replace("_", "-"),
required=True)
for x in mgr_cls._update_attrs.optional:
if x != cls._id_attr:
sub_parser_action.add_argument("--%s" %
x.replace("_", "-"),
required=False)
if cls.__name__ in cli.custom_actions:
name = cls.__name__
for action_name in cli.custom_actions[name]:
sub_parser_action = sub_parser.add_parser(action_name)
# Get the attributes for URL/path construction
if mgr_cls._from_parent_attrs:
for x in mgr_cls._from_parent_attrs:
sub_parser_action.add_argument("--%s" %
x.replace("_", "-"),
required=True)
sub_parser_action.add_argument("--sudo", required=False)
# We need to get the object somehow
if not issubclass(cls, gitlab.mixins.GetWithoutIdMixin):
if cls._id_attr is not None:
id_attr = cls._id_attr.replace("_", "-")
sub_parser_action.add_argument("--%s" % id_attr,
required=True)
required, optional, dummy = cli.custom_actions[name][action_name]
[
sub_parser_action.add_argument("--%s" % x.replace("_", "-"),
required=True) for x in required
if x != cls._id_attr
]
[
sub_parser_action.add_argument("--%s" % x.replace("_", "-"),
required=False)
for x in optional if x != cls._id_attr
]
if mgr_cls.__name__ in cli.custom_actions:
name = mgr_cls.__name__
for action_name in cli.custom_actions[name]:
sub_parser_action = sub_parser.add_parser(action_name)
if mgr_cls._from_parent_attrs:
for x in mgr_cls._from_parent_attrs:
sub_parser_action.add_argument("--%s" %
x.replace("_", "-"),
required=True)
sub_parser_action.add_argument("--sudo", required=False)
required, optional, dummy = cli.custom_actions[name][action_name]
[
sub_parser_action.add_argument("--%s" % x.replace("_", "-"),
required=True) for x in required
if x != cls._id_attr
]
[
sub_parser_action.add_argument("--%s" % x.replace("_", "-"),
required=False)
for x in optional if x != cls._id_attr
]
def register_subparser(
subparser: argparse._SubParsersAction, # pylint: disable=protected-access
) -> argparse.ArgumentParser:
"""Registers a subparser for the current command.
Args:
subparser (argparse._SubParsersAction): Subparsers action.
"""
subcommand = subparser.add_parser(
"status",
aliases=["st"],
help="Report various statistics about a running Cromwell server.",
)
view = subcommand.add_mutually_exclusive_group()
view.add_argument(
"-d",
"--detail-view",
help="Show detailed view which displays information about each individual job.",
default=False,
action="store_true",
)
view.add_argument(
"-z",
"--steps-view",
help="Show the 'steps' view which displays summary information about how many jobs are at each step.",
default=False,
action="store_true",
)
subcommand.add_argument(
"-a",
"--aborted",
dest="show_aborted_jobs",
help="Show jobs in the 'Aborted' state.",
default=False,
action="store_true",
)
_args.add_batches_group(subcommand)
subcommand.add_argument(
"--failed-calls",
help="Only show calls with the given name that failed (useful in restarting failed steps).",
nargs="+",
type=str,
)
subcommand.add_argument(
"-f",
"--failed",
dest="show_failed_jobs",
help="Show jobs in the 'Failed' state.",
default=False,
action="store_true",
)
_args.add_oliver_job_group_args(subcommand)
subcommand.add_argument(
"-i",
"--cromwell-workflow-uuid",
type=str,
help="Filter by workflow id matching argument.",
)
_args.add_oliver_job_name_args(subcommand)
subcommand.add_argument(
"-n",
"--cromwell-workflow-name",
type=str,
help="Filter by workflow name matching argument.",
)
subcommand.add_argument(
"-r",
"--running",
dest="show_running_jobs",
help="Show jobs in the 'Running' state.",
default=False,
action="store_true",
)
subcommand.add_argument(
"-t",
"--submission-time",
help="Show only jobs which were submitted at most N hours ago.",
default=None,
type=int,
)
subcommand.add_argument(
"-s",
"--succeeded",
dest="show_succeeded_jobs",
help="Show jobs in the 'Succeeded' state.",
default=False,
action="store_true",
)
subcommand.add_argument(
"--grid-style",
help="Any valid `tablefmt` for python-tabulate.",
default="fancy_grid",
)
subcommand.set_defaults(func=call)
return subcommand
def init_parser(
subparsers: argparse._SubParsersAction) -> argparse.ArgumentParser:
parser = subparsers.add_parser(
__name__.split(".")[-1],
description="Run WebServer.",
formatter_class=argparse.RawDescriptionHelpFormatter,
help="Run WebServer.",
)
parser.add_argument(
"static",
metavar="folder",
type=str,
help="Path to static folder (created if not found).",
)
parser.add_argument(
"-u",
"--url",
metavar="url",
type=str,
default="0.0.0.0",
help="URL hosting the web server. Default: 0.0.0.0",
)
parser.add_argument(
"-p",
"--port",
metavar="port",
type=int,
default=8080,
help="Web server port. Default: 8080",
)
parser.add_argument(
"-m",
"--mail",
metavar="email",
type=str,
default="*****@*****.**",
help="Email address of server admin.",
)
parser = ap.add_version_option(parser)
advanced = parser.add_argument_group("advanced arguments")
advanced.add_argument(
"--hide-breadcrumbs",
action="store_const",
dest="show_breadcrumbs",
const=False,
default=True,
help="""Hide navigation breadcrumbs.""",
)
advanced.add_argument(
"-R",
"--custom-routes",
metavar="routesFile",
type=str,
help="Path to custom routes Python file.",
)
parser.add_argument(
"-T",
"--custom-templates",
metavar="templateFolder",
type=str,
help="Path to folder with custom templates.",
)
parser.add_argument(
"-H",
"--homepage",
metavar="homepageTemplate",
type=str,
help="""Name of homepage template. Homepage is off by default.
Use "-H home_default" to turn default homepage template on.
When using a custom homepage template, -T must be specified.""",
)
parser.set_defaults(parse=parse_arguments, run=run)
return parser
def register_subparser(parsers: argparse._SubParsersAction) -> None: # pylint: disable=protected-access
"""Add subparser for `build` command."""
build_parser = parsers.add_parser(
'build', help='Commands for downloading and preparing datasets.')
build_parser.add_argument(
'datasets', # Positional arguments
type=str,
nargs='*',
help='Name(s) of the dataset(s) to build. Default to current dir. '
'See https://www.tensorflow.org/datasets/cli for accepted values.',
)
build_parser.add_argument( # Also accept keyword arguments
'--datasets',
type=str,
nargs='+',
dest='datasets_keyword',
help='Datasets can also be provided as keyword argument.',
)
# **** Debug options ****
debug_group = build_parser.add_argument_group(
'Debug & tests',
description='--pdb Enter post-mortem debugging mode '
'if an exception is raised.')
debug_group.add_argument(
'--overwrite',
action='store_true',
help='Delete pre-existing dataset if it exists.',
)
debug_group.add_argument(
'--max_examples_per_split',
type=int,
nargs='?',
const=1,
help=
'When set, only generate the first X examples (default to 1), rather '
'than the full dataset.'
'If set to 0, only execute the `_split_generators` (which download the '
'original data), but skip `_generator_examples`',
)
# **** Path options ****
path_group = build_parser.add_argument_group('Paths')
path_group.add_argument(
'--data_dir',
type=tfds.core.as_path,
# Should match tfds.core.constant.DATA_DIR !!
default=tfds.core.as_path(
os.environ.get('TFDS_DATA_DIR',
os.path.join('~', 'tensorflow_datasets'))),
help='Where to place datasets. Default to '
'`~/tensorflow_datasets/` or `TFDS_DATA_DIR` environement variable.',
)
path_group.add_argument(
'--download_dir',
type=tfds.core.as_path,
help='Where to place downloads. Default to `<data_dir>/downloads/`.',
)
path_group.add_argument(
'--extract_dir',
type=tfds.core.as_path,
help='Where to extract files. Default to `<download_dir>/extracted/`.',
)
path_group.add_argument(
'--manual_dir',
type=tfds.core.as_path,
help='Where to manually download data (required for some datasets). '
'Default to `<download_dir>/manual/`.',
)
path_group.add_argument(
'--add_name_to_manual_dir',
action='store_true',
help='If true, append the dataset name to the `manual_dir` (e.g. '
'`<download_dir>/manual/<dataset_name>/`. Useful to avoid collisions '
'if many datasets are generated.')
# **** Generation options ****
generation_group = build_parser.add_argument_group('Generation')
generation_group.add_argument(
'--config',
'-c',
type=str,
help='Config name to build. Build all configs if not set.')
# We are forced to have 2 flags to avoid ambiguity when config name is
# a number (e.g. `voc/2017`)
generation_group.add_argument(
'--config_idx',
type=int,
help='Config id to build (`builder_cls.BUILDER_CONFIGS[config_idx]`). '
'Mutually exclusive with `--config`.')
generation_group.add_argument(
'--imports',
'-i',
type=str,
help='Comma separated list of module to import to register datasets.')
generation_group.add_argument(
'--register_checksums',
action='store_true',
help='If True, store size and checksum of downloaded files.',
)
generation_group.add_argument(
'--force_checksums_validation',
action='store_true',
help='If True, raise an error if the checksums are not found.',
)
generation_group.add_argument(
'--beam_pipeline_options',
type=str,
# nargs='+',
help='A (comma-separated) list of flags to pass to `PipelineOptions` '
'when preparing with Apache Beam. '
'(see: https://www.tensorflow.org/datasets/beam_datasets). '
'Example: `--beam_pipeline_options=job_name=my-job,project=my-project`'
)
# **** Automation options ****
automation_group = build_parser.add_argument_group(
'Automation', description='Used by automated scripts.')
automation_group.add_argument(
'--exclude_datasets',
type=str,
help='If set, generate all datasets except the one defined here. '
'Comma separated list of datasets to exclude. ')
automation_group.add_argument(
'--experimental_latest_version',
action='store_true',
help='Build the latest Version(experiments=...) available rather than '
'default version.')
build_parser.set_defaults(subparser_fn=_build_datasets)
def make_aws_parser(subparsers: argparse._SubParsersAction):
parser_aws = subparsers.add_parser("aws", help="AWS help")
aws_subparsers = parser_aws.add_subparsers(help="command", dest="command")
make_down_subparser(aws_subparsers)
make_up_subparser(aws_subparsers)
def configure_log_parser(subparsers: argparse._SubParsersAction) -> None:
"""Add a logging parser to an existing argparser.
Args:
subparsers: The parser object to be appended to.
"""
parser = subparsers.add_parser(
'logs',
description='Generates comparison graphs amongst one or more log files',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
allow_abbrev=False)
parser.add_argument(
'log_dir',
metavar='<Log Dir>',
type=str,
help="The path to a folder containing one or more log files")
parser.add_argument('--extension',
metavar='E',
type=str,
help="The file type / extension of your logs",
default=".txt")
parser.add_argument(
'--recursive',
action='store_true',
help="Recursively search sub-directories for log files")
group = parser.add_mutually_exclusive_group()
group.add_argument(
'--ignore',
metavar='I',
type=str,
nargs='+',
help=
"The names of metrics to ignore though they may be present in the log files"
)
group.add_argument(
'--include',
metavar='Y',
type=str,
nargs='+',
help=
"The names of metrics to include. If provided, any other metrics will be ignored."
)
parser.add_argument(
'--smooth',
metavar='<float>',
type=float,
help=
"The amount of gaussian smoothing to apply (zero for no smoothing)",
default=1)
parser.add_argument('--pretty_names',
help="Clean up the metric names for display",
action='store_true')
parser.add_argument(
'--group_by',
metavar='G',
type=str,
nargs=1,
help=
"A regex pattern to group different logs together and display their mean+-stdev. For "
r"example, you could use --G '(.*)_[\d]+\.txt' to group files of the form "
"<name>_<number>.txt by their <name>. We anticipate this being the common usecase, so you "
"can use --G _n as a shortcut for that functionality.")
legend_group = parser.add_argument_group('legend arguments')
legend_x_group = legend_group.add_mutually_exclusive_group(required=False)
legend_x_group.add_argument('--common_legend',
dest='share_legend',
help="Generate one legend total",
action='store_true',
default=True)
legend_x_group.add_argument('--split_legend',
dest='share_legend',
help="Generate one legend per graph",
action='store_false',
default=False)
save_group = parser.add_argument_group('output arguments')
save_x_group = save_group.add_mutually_exclusive_group(required=False)
save_x_group.add_argument(
'--save',
nargs='?',
metavar='<Save Dir>',
dest='save',
action=SaveAction,
default=False,
help="Save the output image. May be accompanied by a directory into \
which the file is saved. If no output directory is specified, the log \
directory will be used")
save_x_group.add_argument(
'--display',
dest='save',
action='store_false',
help="Render the image to the UI (rather than saving it)",
default=True)
save_x_group.set_defaults(save_dir=None)
parser.set_defaults(func=logs)
def make_up_subparser(subparsers: argparse._SubParsersAction):
subparser = subparsers.add_parser(
"up", help="deploy/update CloudFormation stack")
require_named = subparser.add_argument_group("required named arguments")
require_named.add_argument("--cluster-id",
type=str,
help="stack name for CloudFormation cluster",
required=True)
require_named.add_argument("--keypair",
type=str,
help="aws ec2 keypair for master and agent",
required=True)
subparser.add_argument(
"--master-instance-type",
type=str,
help="instance type for master",
)
subparser.add_argument(
"--agent-instance-type",
type=str,
help="instance type for agent",
)
subparser.add_argument(
"--deployment-type",
type=str,
choices=constants.deployment_types.DEPLOYMENT_TYPES,
default=constants.defaults.DEPLOYMENT_TYPE,
help=f"deployment type - "
f'must be one of [{", ".join(constants.deployment_types.DEPLOYMENT_TYPES)}]',
)
subparser.add_argument("--aws-profile",
type=str,
default=None,
help=argparse.SUPPRESS)
subparser.add_argument(
"--inbound-cidr",
type=str,
help="inbound IP Range in CIDR format",
)
subparser.add_argument(
"--det-version",
type=str,
help=argparse.SUPPRESS,
)
subparser.add_argument(
"--db-password",
type=str,
default=constants.defaults.DB_PASSWORD,
help="password for master database",
)
subparser.add_argument(
"--region",
type=str,
default=None,
help="AWS region",
)
subparser.add_argument(
"--max-idle-agent-period",
type=str,
help="max agent idle time",
)
subparser.add_argument(
"--max-agent-starting-period",
type=str,
help="max agent starting time",
)
subparser.add_argument(
"--max-dynamic-agents",
type=int,
help="maximum number of dynamic agent instances at one time",
)
subparser.add_argument(
"--dry-run",
action="store_true",
help="print deployment template",
)
def add_subparser(self, parser: argparse._SubParsersAction):
description = 'Calculate the correlation between two different metrics'
self.parser = parser.add_parser('correlate',
description=description,
help=description)
self.parser.add_argument(
'--metrics-jsonl-files',
nargs='+',
help=
'The jsonl files with the metric values. If the values are split across multiple files, they can all '
'be passed as arguments.',
required=True)
self.parser.add_argument(
'--metrics',
nargs=2,
type=str,
help=
'The flattened names of the two metrics that should be correlated',
required=True)
self.parser.add_argument(
'--summarizer-type',
choices=['all', 'reference', 'peer'],
help=
'The type of summarizer which should be included in the correlation calculation',
required=True)
self.parser.add_argument(
'--output-file',
type=str,
help=
'The json output file which will contain the final correlations')
self.parser.add_argument(
'--log-file',
type=str,
help='The file where the log should be written')
self.parser.add_argument(
'--silent',
action='store_true',
help='Controls whether the log should be written to stdout')
self.parser.add_argument(
'--summary-level-correlations-output',
type=str,
help=
'The file where all of the summary-level correlations should be written'
)
self.parser.add_argument(
'--skip-summary-level',
action='store_true',
help=
'Indicates the summary-level correlations should not be calculated'
)
self.parser.add_argument(
'--skip-system-level',
action='store_true',
help=
'Indicates the system-level correlations should not be calculated')
self.parser.add_argument(
'--skip-global',
action='store_true',
help='Indicates the global correlations should not be calculated')
self.parser.add_argument(
'--system-level-output-plot',
type=str,
help=
'If provided, saves a plot of the system-level scores to the corresponding file'
)
self.parser.add_argument(
'--global-output-plot',
type=str,
help=
'If provided, saves a plot of the global scores to the corresponding file'
)
self.parser.set_defaults(func=self.run)
def add_subparser(
self,
parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
description = """Train the specified model on the specified dataset."""
subparser = parser.add_parser(self.name,
description=description,
help="Train a model.")
subparser.add_argument(
"param_path",
type=str,
help="path to parameter file describing the model to be trained")
subparser.add_argument(
"-s",
"--serialization-dir",
required=True,
type=str,
help="directory in which to save the model and its logs",
)
subparser.add_argument(
"-r",
"--recover",
action="store_true",
default=False,
help="recover training from the state in serialization_dir",
)
subparser.add_argument(
"-f",
"--force",
action="store_true",
required=False,
help="overwrite the output directory if it exists",
)
subparser.add_argument(
"-o",
"--overrides",
type=str,
default="",
help=
("a json(net) structure used to override the experiment configuration, e.g., "
"'{\"iterator.batch_size\": 16}'. Nested parameters can be specified either"
" with nested dictionaries or with dot syntax."),
)
subparser.add_argument(
"--node-rank",
type=int,
default=0,
help="rank of this node in the distributed setup")
subparser.add_argument(
"--dry-run",
action="store_true",
help=
("do not train a model, but create a vocabulary, show dataset statistics and "
"other training information"),
)
subparser.add_argument(
"--file-friendly-logging",
action="store_true",
default=False,
help=
"outputs tqdm status on separate lines and slows tqdm refresh rate",
)
subparser.set_defaults(func=train_model_from_args)
return subparser
def add_subparser(subparsers: argparse._SubParsersAction,
parents: List[argparse.ArgumentParser]):
from rasa.core import cli
x_parser_args = {
"parents": parents,
"conflict_handler": "resolve",
"formatter_class": argparse.ArgumentDefaultsHelpFormatter,
}
if is_rasa_x_installed():
# we'll only show the help msg for the command if Rasa X is actually installed
x_parser_args["help"] = "Start Rasa X and the Interface"
shell_parser = subparsers.add_parser("x", **x_parser_args)
shell_parser.add_argument(
"--no-prompt",
action="store_true",
help=
"Automatic yes or default options to prompts and oppressed warnings",
)
shell_parser.add_argument(
"--production",
action="store_true",
help="Run Rasa X in a production environment",
)
shell_parser.add_argument("--auth-token",
type=str,
help="Rasa API auth token")
shell_parser.add_argument(
"--nlg",
type=str,
default="http://localhost:5002/api/nlg",
help="Rasa NLG endpoint",
)
shell_parser.add_argument(
"--model-endpoint-url",
type=str,
default=
"http://localhost:5002/api/projects/default/models/tags/production",
help="Rasa model endpoint URL",
)
shell_parser.add_argument(
"--project-path",
type=str,
default=".",
help="Path to the Rasa project directory",
)
shell_parser.add_argument(
"--data-path",
type=str,
default="data",
help=("Path to the directory containing Rasa NLU training data "
"and Rasa Core stories"),
)
rasa.cli.run.add_run_arguments(shell_parser)
shell_parser.set_defaults(func=rasa_x)
cli.arguments.add_logging_option_arguments(shell_parser)
def make_up_subparser(subparsers: argparse._SubParsersAction) -> None:
subparser = subparsers.add_parser(
"up", help="deploy/update CloudFormation stack")
require_named = subparser.add_argument_group("required named arguments")
require_named.add_argument("--cluster-id",
type=str,
help="stack name for CloudFormation cluster",
required=True)
require_named.add_argument("--keypair",
type=str,
help="aws ec2 keypair for master and agent",
required=True)
subparser.add_argument(
"--region",
type=str,
default=None,
help="AWS region",
)
subparser.add_argument("--profile",
type=str,
default=None,
help="AWS profile")
subparser.add_argument(
"--master-instance-type",
type=str,
help="instance type for master",
)
subparser.add_argument(
"--enable-cors",
action="store_true",
help="allow CORS requests or not: true/false",
)
subparser.add_argument("--master-tls-cert", )
subparser.add_argument("--master-tls-key", )
subparser.add_argument("--master-cert-name", )
subparser.add_argument(
"--gpu-agent-instance-type",
type=str,
help="instance type for agent in the GPU resource pool",
)
subparser.add_argument(
"--cpu-agent-instance-type",
type=str,
help="instance type for agent in the CPU resource pool",
)
subparser.add_argument(
"--deployment-type",
type=str,
choices=constants.deployment_types.DEPLOYMENT_TYPES,
default=constants.defaults.DEPLOYMENT_TYPE,
help=f"deployment type - "
f'must be one of [{", ".join(constants.deployment_types.DEPLOYMENT_TYPES)}]',
)
subparser.add_argument(
"--inbound-cidr",
type=str,
help="inbound IP Range in CIDR format",
)
subparser.add_argument(
"--agent-subnet-id",
type=str,
help=
"subnet to deploy agents into. Optional. Only used with simple deployment type",
)
subparser.add_argument(
"--det-version",
type=str,
help=argparse.SUPPRESS,
)
subparser.add_argument(
"--db-password",
type=str,
default=constants.defaults.DB_PASSWORD,
help="password for master database",
)
subparser.add_argument(
"--max-idle-agent-period",
type=str,
help="max agent idle time",
)
subparser.add_argument(
"--max-agent-starting-period",
type=str,
help="max agent starting time",
)
subparser.add_argument(
"--max-cpu-containers-per-agent",
type=int,
help=
"maximum number of cpu containers on agent in the CPU resource pool",
)
subparser.add_argument(
"--min-dynamic-agents",
type=int,
help="minimum number of dynamic agent instances at one time",
)
subparser.add_argument(
"--max-dynamic-agents",
type=int,
help="maximum number of dynamic agent instances at one time",
)
subparser.add_argument(
"--spot",
action="store_true",
help="whether to use spot instances or not",
)
subparser.add_argument(
"--spot-max-price",
type=validate_spot_max_price(),
help=
"maximum hourly price for the spot instance (do not include the dollar sign)",
)
subparser.add_argument(
"--scheduler-type",
type=validate_scheduler_type(),
default="fair_share",
help="scheduler to use (defaults to fair_share).",
)
subparser.add_argument(
"--preemption-enabled",
type=str,
default="false",
help="whether task preemption is supported in the scheduler "
"(only configurable for priority scheduler).",
)
subparser.add_argument(
"--dry-run",
action="store_true",
help="print deployment template",
)
subparser.add_argument(
"--cpu-env-image",
type=str,
help="Docker image for CPU tasks",
)
subparser.add_argument(
"--gpu-env-image",
type=str,
help="Docker image for GPU tasks",
)
def add_subparser(
self,
parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
description = """Run the specified model through a checklist suite."""
subparser = parser.add_parser(
self.name,
description=description,
help="Run a trained model through a checklist suite.",
)
subparser.add_argument(
"archive_file",
type=str,
help="The archived model to make predictions with")
subparser.add_argument("task",
type=str,
help="The name of the task suite")
subparser.add_argument("--checklist-suite",
type=str,
help="The checklist suite path")
subparser.add_argument(
"--capabilities",
nargs="+",
default=[],
help=
('An optional list of strings of capabilities. Eg. "[Vocabulary, Robustness]"'
),
)
subparser.add_argument(
"--max-examples",
type=int,
default=None,
help="Maximum number of examples to check per test.",
)
subparser.add_argument(
"--task-suite-args",
type=str,
default="",
help=
("An optional JSON structure used to provide additional parameters to the task suite"
),
)
subparser.add_argument(
"--print-summary-args",
type=str,
default="",
help=("An optional JSON structure used to provide additional "
"parameters for printing test summary"),
)
subparser.add_argument("--output-file",
type=str,
help="Path to output file")
subparser.add_argument("--cuda-device",
type=int,
default=-1,
help="ID of GPU to use (if any)")
subparser.add_argument(
"--predictor",
type=str,
help="Optionally specify a specific predictor to use")
subparser.add_argument(
"--predictor-args",
type=str,
default="",
help=
("An optional JSON structure used to provide additional parameters to the predictor"
),
)
subparser.set_defaults(func=_run_suite)
return subparser
def add_subparser(
subparsers: argparse._SubParsersAction,
cmd: str,
server: bool = False,
format: bool = False,
signon: bool = False,
stmtend: bool = False,
stmt: bool = False,
acctinforq: bool = False,
tax: bool = False,
help: Optional[str] = None,
) -> argparse.ArgumentParser:
parser = subparsers.add_parser(cmd, help=help, description=help)
parser.set_defaults(request=cmd)
parser.add_argument("server", nargs="?", help="OFX server nickname")
parser.add_argument(
"--verbose",
"-v",
action="count",
default=0,
help="Give more output (option can be repeated)",
)
# Higher-level configs (e.g. account #s)
# imply lower-level configs (e.g. username/passwd)
if stmt:
stmtend = True
if stmtend or tax:
signon = True
acctinforq = True # Support internally generated ACCTINFORQ for --all accounts
if signon:
format = True
if format:
server = True
if server:
parser.add_argument("--url", help="OFX server URL")
parser.add_argument("--ofxhome",
metavar="ID#",
help="FI id# on http://www.ofxhome.com/")
parser.add_argument(
"-w",
"--write",
action="store_true",
default=None,
help="Write working parameters to config file",
)
parser.add_argument(
"--useragent",
dest="useragent",
help=
"Value to use in HTTP 'User-Agent' header (defaults to empty string)",
)
if format:
parser.add_argument(
"-n",
"--dryrun",
action="store_true",
default=None,
help="Display OFX request and exit without sending",
)
add_format_group(parser)
if signon:
parser.add_argument(
"--savepass",
action="store_true",
default=None,
help="Store password in system keyring (requires python-keyring)",
)
parser.add_argument(
"--nokeyring",
action="store_true",
default=None,
help="Don't use system keyring to store/retrieve passwords",
)
add_signon_group(parser)
if stmtend:
add_bank_acct_group(parser)
stmt_group = add_stmt_group(parser)
if stmt:
add_stmt_args(stmt_group)
add_inv_acct_group(parser)
add_inv_stmt_group(parser)
if acctinforq:
add_acctinforq_group(parser)
if tax:
add_tax_group(parser)
return parser
def add_subparser(
self, name: str,
parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Run the specified model against a JSON-lines input file.'''
subparser = parser.add_parser(
name,
description=description,
help='Use a trained model to make predictions.')
subparser.add_argument(
'archive_file',
type=str,
help='the archived model to make predictions with')
subparser.add_argument('input_file',
type=argparse.FileType('r'),
help='path to input file')
subparser.add_argument('--output-file',
type=argparse.FileType('w'),
help='path to output file')
subparser.add_argument(
'--weights-file',
type=str,
help='a path that overrides which weights file to use')
batch_size = subparser.add_mutually_exclusive_group(required=False)
batch_size.add_argument('--batch-size',
type=int,
default=1,
help='The batch size to use for processing')
subparser.add_argument('--silent',
action='store_true',
help='do not print output to stdout')
cuda_device = subparser.add_mutually_exclusive_group(required=False)
cuda_device.add_argument('--cuda-device',
type=int,
default=-1,
help='id of GPU to use (if any)')
subparser.add_argument(
'-o',
'--overrides',
type=str,
default="",
help=
'a HOCON structure used to override the experiment configuration')
subparser.add_argument('--include-package',
type=str,
action='append',
default=[],
help='additional packages to include')
subparser.add_argument(
'--predictor',
type=str,
help='optionally specify a specific predictor to use')
subparser.set_defaults(func=_predict(self.predictors))
return subparser
def setup(subparsers: argparse._SubParsersAction) -> argparse.ArgumentParser:
sub = subparsers.add_parser('check', help='Check whether a provided schema (JSON, YAML) can be parsed.')
sub.add_argument('-s', '--source', help="Path to a spec (JSON, YAML). "
"If not specified, then the data will be read from stdin.")
sub.set_defaults(run_cmd=main)
return sub
def add_subparser(
self, name: str,
parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
# pylint: disable=protected-access
description = '''Train the specified model on the specified dataset.'''
subparser = parser.add_parser(name,
description=description,
help='Train a model.')
subparser.add_argument(
'param_path',
type=str,
help='path to parameter file describing the model to be trained')
subparser.add_argument(
'options_path',
type=str,
help=
'path to file describing external variables and possible values to search in'
)
subparser.add_argument(
'-s',
'--serialization-dir',
required=True,
type=str,
help='directory in which to save the models and their logs')
subparser.add_argument('-n',
'--num-runs',
default=1,
type=int,
help='number of runs per config')
subparser.add_argument(
'-o',
'--overrides',
type=str,
default="",
help=
'a JSON structure used to override the experiment configuration')
subparser.add_argument(
'--file-friendly-logging',
action='store_true',
default=False,
help=
'outputs tqdm status on separate lines and slows tqdm refresh rate'
)
subparser.add_argument('--random',
action='store_true',
default=False,
help='random order of runs')
subparser.add_argument(
'--cache-directory',
type=str,
default='',
help='Location to store cache of data preprocessing')
subparser.add_argument(
'--cache-prefix',
type=str,
default='',
help='Prefix to use for data caching, giving current parameter '
'settings a name in the cache, instead of computing a hash')
cuda_device = subparser.add_mutually_exclusive_group(required=False)
cuda_device.add_argument(
'--cuda-device',
type=int,
default=None,
help='id of GPU to use (if not the default one from the config)')
subparser.set_defaults(func=train_model_from_args)
return subparser
def add_subparser(cls, parser: argparse._SubParsersAction) -> None:
kill = parser.add_parser(cls.NAME)
kill.set_defaults(command=cls.from_arguments)
kill.add_argument("--with-fire",
action="store_true",
help="A no-op flag that adds emphasis.")
def add_subparser(
self, name: str, parser: argparse._SubParsersAction
) -> argparse.ArgumentParser:
description = """Evaluate the specified model + dataset"""
subparser = parser.add_parser(
name, description=description, help="Evaluate the specified model + dataset."
)
subparser.add_argument("archive_file", type=str, help="path to an archived trained model")
subparser.add_argument(
"input_file", type=str, help="path to the file containing the evaluation data"
)
subparser.add_argument("--output-file", type=str, help="path to output file")
subparser.add_argument(
"--weights-file", type=str, help="a path that overrides which weights file to use"
)
cuda_device = subparser.add_mutually_exclusive_group(required=False)
cuda_device.add_argument(
"--cuda-device", type=int, default=-1, help="id of GPU to use (if any)"
)
subparser.add_argument(
"-o",
"--overrides",
type=str,
default="",
help="a JSON structure used to override the experiment configuration",
)
subparser.add_argument(
"--batch-size", type=int, help="If non-empty, the batch size to use during evaluation."
)
subparser.add_argument(
"--batch-weight-key",
type=str,
default="",
help="If non-empty, name of metric used to weight the loss on a per-batch basis.",
)
subparser.add_argument(
"--extend-vocab",
action="store_true",
default=False,
help="if specified, we will use the instances in your new dataset to "
"extend your vocabulary. If pretrained-file was used to initialize "
"embedding layers, you may also need to pass --embedding-sources-mapping.",
)
subparser.add_argument(
"--embedding-sources-mapping",
type=str,
default="",
help="a JSON dict defining mapping from embedding module path to embedding"
"pretrained-file used during training. If not passed, and embedding needs to be "
"extended, we will try to use the original file paths used during training. If "
"they are not available we will use random vectors for embedding extension.",
)
subparser.set_defaults(func=evaluate_from_args)
return subparser
def add_subparser(subparsers: _SubParsersAction, name: str,
help: str) -> ArgumentParser:
subparser = subparsers.add_parser(name, help=help)
subparser.add_argument(
"--state",
"-s",
dest="state_file",
metavar="FILE",
default=nixops.statefile.get_default_state_file(),
help="path to state file",
)
subparser.add_argument(
"--deployment",
"-d",
dest="deployment",
metavar="UUID_OR_NAME",
default=os.environ.get("NIXOPS_DEPLOYMENT",
os.environ.get("CHARON_DEPLOYMENT", None)),
help="UUID or symbolic name of the deployment",
)
subparser.add_argument("--debug",
action="store_true",
help="enable debug output")
subparser.add_argument(
"--confirm",
action="store_true",
help="confirm dangerous operations; do not ask",
)
# Nix options that we pass along.
subparser.add_argument(
"-I",
nargs=1,
action="append",
dest="nix_path",
metavar="PATH",
help="append a directory to the Nix search path",
)
subparser.add_argument(
"--max-jobs",
"-j",
type=int,
metavar="N",
help="set maximum number of concurrent Nix builds",
)
subparser.add_argument(
"--cores",
type=int,
metavar="N",
help=
"sets the value of the NIX_BUILD_CORES environment variable in the invocation of builders",
)
subparser.add_argument("--keep-going",
action="store_true",
help="keep going after failed builds")
subparser.add_argument(
"--keep-failed",
"-K",
action="store_true",
help="keep temporary directories of failed builds",
)
subparser.add_argument(
"--show-trace",
action="store_true",
help=
"print a Nix stack trace if evaluation fails, or a python stack trace if nixops fails",
)
subparser.add_argument("--fallback",
action="store_true",
help="fall back on installation from source")
subparser.add_argument(
"--no-build-output",
action="store_true",
help="suppress output written by builders",
)
subparser.add_argument(
"--option",
nargs=2,
action="append",
dest="nix_options",
metavar=("NAME", "VALUE"),
help="set a Nix option",
)
subparser.add_argument(
"--read-only-mode",
action="store_true",
help="run Nix evaluations in read-only mode",
)
return subparser
