def _parse_flags(_) -> argparse.Namespace:
"""Command line flags."""
parser = argparse_flags.ArgumentParser(
prog='build_catalog',
description='Tool to generate the dataset catalog documentation',
)
parser.add_argument(
'--datasets',
help='Comma separated list of datasets to document. None for all '
'datasets.',
)
parser.add_argument(
'--catalog_dir',
help='Directory path where to generate the catalog. Default to TFDS dir.',
)
return parser.parse_args()
def parse_args(argv):
parser = argparse_flags.ArgumentParser()
parser.add_argument(
"input",
type=str,
help="Path to where your files are located. Files ending in .zst are treated as \
archives, all others as raw text.",
)
parser.add_argument(
"--encoder", type=str, required=True, help="Name or path of a tokenizer spec"
)
parser.add_argument(
"--sample_size", type=int, default=10, help="the size of samples to inspect"
)
args = parser.parse_args(argv[1:])
return args
def configure(self, argv=('', ), **kwargs):
"""Configures TensorBoard behavior via flags.
This method will populate the "flags" property with an argparse.Namespace
representing flag values parsed from the provided argv list, overriden by
explicit flags from remaining keyword arguments.
Args:
argv: Can be set to CLI args equivalent to sys.argv; the first arg is
taken to be the name of the path being executed.
kwargs: Additional arguments will override what was parsed from
argv. They must be passed as Python data structures, e.g.
`foo=1` rather than `foo="1"`.
Returns:
Either argv[:1] if argv was non-empty, or [''] otherwise, as a mechanism
for absl.app.run() compatibility.
Raises:
ValueError: If flag values are invalid.
"""
parser = argparse_flags.ArgumentParser(
prog='tensorboard',
description=(
'TensorBoard is a suite of web applications for '
'inspecting and understanding your TensorFlow runs '
'and graphs. https://github.com/tensorflow/tensorboard '))
for loader in self.plugin_loaders:
loader.define_flags(parser)
arg0 = argv[0] if argv else ''
flags = parser.parse_args(argv[1:]) # Strip binary name from argv.
if absl_flags and arg0:
# Only expose main module Abseil flags as TensorBoard native flags.
# This is the same logic Abseil's ArgumentParser uses for determining
# which Abseil flags to include in the short helpstring.
for flag in set(absl_flags.FLAGS.get_key_flags_for_module(arg0)):
if hasattr(flags, flag.name):
raise ValueError('Conflicting Abseil flag: %s' % flag.name)
setattr(flags, flag.name, flag.value)
for k, v in kwargs.items():
if not hasattr(flags, k):
raise ValueError('Unknown TensorBoard flag: %s' % k)
setattr(flags, k, v)
for loader in self.plugin_loaders:
loader.fix_flags(flags)
self.flags = flags
return [arg0]
def _parse_flags(argv: List[str]) -> argparse.Namespace:
"""Command lines flag parsing."""
# Normalize explicit boolean flags for absl.flags compatibility
# See b/174043007 for context.
argv = _normalize_flags(argv)
parser = argparse_flags.ArgumentParser(
description='Tensorflow Datasets CLI tool', )
parser.add_argument('--version',
action='version',
version='TensorFlow Datasets: ' + tfds.__version__)
parser.set_defaults(subparser_fn=lambda _: parser.print_help())
# Register sub-commands
subparser = parser.add_subparsers(title='command')
build.register_subparser(subparser)
new.register_subparser(subparser)
return parser.parse_args(argv[1:])
def _parse_flags(argv: List[str]) -> argparse.Namespace:
"""Parses command line flags."""
argv = flag_utils.normalize_flags(argv) # See b/174043007 for context.
parser = argparse_flags.ArgumentParser(
description='Tensorflow Datasets CLI tool', )
parser.add_argument(
'--data_dir',
type=tfds.core.Path,
help='Path to the dataset files.',
)
parser.add_argument(
'--out_dir',
type=tfds.core.Path,
help='Computed metadata will be written here.',
)
return parser.parse_args(argv[1:])
def test_parse_absl_flags(self):
parser = argparse_flags.ArgumentParser(
inherited_absl_flags=self._absl_flags)
self.assertFalse(self._absl_flags.is_parsed())
self.assertTrue(self._absl_flags['absl_string'].using_default_value)
self.assertTrue(self._absl_flags['absl_integer'].using_default_value)
self.assertTrue(self._absl_flags['absl_float'].using_default_value)
self.assertTrue(self._absl_flags['absl_enum'].using_default_value)
parser.parse_args(['--absl_string=new_string', '--absl_integer', '2'])
self.assertEqual(self._absl_flags.absl_string, 'new_string')
self.assertEqual(self._absl_flags.absl_integer, 2)
self.assertTrue(self._absl_flags.is_parsed())
self.assertFalse(self._absl_flags['absl_string'].using_default_value)
self.assertFalse(self._absl_flags['absl_integer'].using_default_value)
self.assertTrue(self._absl_flags['absl_float'].using_default_value)
self.assertTrue(self._absl_flags['absl_enum'].using_default_value)
def _parse_flags(_: List[str]) -> argparse.Namespace:
"""Command line flags."""
parser = argparse_flags.ArgumentParser(
prog='build_community_catalog',
description='Tool to generate the community dataset catalog',
)
parser.add_argument(
'--catalog_dir',
help=
'Directory path where to generate the catalog. Default to TFDS dir.',
)
parser.add_argument(
'--local_cache',
help=(
'If specified, where to store temporarily downloaded files so that '
'they can be reused when running multiple times.'))
return parser.parse_args()
def create_parser():
"""Creates and returns the argparse instance for the experiment config
expansion app.
"""
parser = argparse_flags.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description=
f"""Configurable arguments for the uv-metrics Caliban tutorial.""",
prog="python -m mnist")
parser.add_argument(
"--gcloud_path",
help=
"""Path for gcloud logs; if supplied, used for persisting logs. This must be of
the form gs://BUCKET_NAME/subfolder. Logs will be stored in the supplied
folder in a subfolder named after the current job run.""")
parser.add_argument(
"--local_path",
help=
"""Path for gcloud logs; if supplied, this location on the local filesystem is
used for persisting logs in jsonl format. The path can be relative. Logs
will be stored in the supplied folder in a subfolder named after the
current job run.""")
parser.add_argument(
"--tensorboard_path",
help=
"""project-local path for tensorboard logs; if supplied, this location on the
local filesystem is used for persisting logs that tensorboard can
read.""")
parser.add_argument("--learning_rate",
"--lr",
type=float,
default=0.01,
help="Learning rate.")
parser.add_argument("--epochs",
type=int,
default=3,
help="Epochs to train.")
return parser
def test_mixed_flags_and_positional(self):
parser = argparse_flags.ArgumentParser(
inherited_absl_flags=self._absl_flags)
parser.add_argument('--header', help='Header message to print.')
parser.add_argument('integers',
metavar='N',
type=int,
nargs='+',
help='an integer for the accumulator')
args = parser.parse_args([
'--absl_string=new_string', '--header=HEADER', '--absl_integer',
'2', '3', '4'
])
self.assertEqual(self._absl_flags.absl_string, 'new_string')
self.assertEqual(self._absl_flags.absl_integer, 2)
self.assertEqual(args.header, 'HEADER')
self.assertListEqual(args.integers, [3, 4])
def test_helpfull_message(self):
flags.DEFINE_string(
'non_main_module_flag', 'default', 'help',
module_name='other.module', flag_values=self._absl_flags)
parser = argparse_flags.ArgumentParser(
inherited_absl_flags=self._absl_flags)
with self.assertRaises(SystemExit),\
mock.patch.object(sys, 'stdout', new=six.StringIO()) as mock_stdout:
parser.parse_args(['--helpfull'])
stdout_message = mock_stdout.getvalue()
logging.info('captured stdout message:\n%s', stdout_message)
self.assertIn('--non_main_module_flag', stdout_message)
self.assertIn('other.module', stdout_message)
# Make sure the main module is not included.
self.assertNotIn(sys.argv[0], stdout_message)
# Special flags defined in absl.flags.
self.assertIn('absl.flags:', stdout_message)
self.assertIn('--flagfile', stdout_message)
self.assertIn('--undefok', stdout_message)
def _parse_flags(_) -> argparse.Namespace:
"""Command line flags."""
parser = argparse_flags.ArgumentParser(
prog="convert_dataset",
description="Tool to add hugging face datasets",
)
parser.add_argument(
"--nlp_path", type=pathlib.Path, help="Path of the nlp directory")
parser.add_argument(
"--dataset_name",
type=str,
help="Name of dataset to be converted. Ex. esnli",
)
parser.add_argument(
"--dataset_type",
type=str,
help="Type of dataset. Ex. text, question_answering",
)
return parser.parse_args()
def _parse_flags(argv: List[str]) -> argparse.Namespace:
"""Command lines flag parsing."""
parser = argparse_flags.ArgumentParser()
parser.add_argument('--num_examples',
help=('Number of examples to send to the server.'),
default=1,
type=int)
parser.add_argument(
'--server',
help=('Prediction service host:port or aiplatform:model:version'),
required=True)
parser.add_argument('--examples_file',
help=('Path to csv file containing examples.'),
required=True)
parser.add_argument('--schema_file',
help='File holding the schema for the input data')
return parser.parse_args(argv)
def parse_args(args):
# Parse command line arguments
parser = argparse_flags.ArgumentParser()
parser.add_argument("--seed", type=int, default=1337, help="seed")
subparsers = parser.add_subparsers(help="Available commands",
dest="subparser")
for name, cmd in SUBCOMMANDS.items():
cmd_parser = subparsers.add_parser(
name,
help=cmd.__doc__,
# Do not make absl FLAGS available after the subcommand `roll_dice`.
inherited_absl_flags=False,
)
cmd.parse_args(args, cmd_parser)
args = parser.parse_args(args[1:])
set_random_seed(args)
fix_logger()
return args
def _parse_flags(argv=('', )):
"""Integration point for `absl.app`.
Exits if flag values are invalid.
Args:
argv: CLI arguments, as with `sys.argv`, where the first argument is taken
to be the name of the program being executed.
Returns:
Either argv[:1] if argv was non-empty, or [''] otherwise, as a mechanism
for absl.app.run() compatibility.
"""
parser = argparse_flags.ArgumentParser(
prog='uploader',
description=('Upload your TensorBoard experiments to TensorBoard.dev'))
_define_flags(parser)
arg0 = argv[0] if argv else ''
global _FLAGS
_FLAGS = parser.parse_args(argv[1:])
return [arg0]
def parse_flags_subcommands(argv):
"""Subcommands example for absl.flags + argparse."""
parser = argparse_flags.ArgumentParser(
description='A subcommands example of argparse_flags.')
parser.add_argument('--argparse_echo',
help='The echo message from argparse_flags')
subparsers = parser.add_subparsers(help='The command to execute.')
roll_dice_parser = subparsers.add_parser('roll_dice', help='Roll a dice.')
roll_dice_parser.add_argument('--num_faces', type=int, default=6)
roll_dice_parser.set_defaults(command=roll_dice)
shuffle_parser = subparsers.add_parser('shuffle', help='Shuffle inputs.')
shuffle_parser.add_argument('inputs',
metavar='I',
nargs='+',
help='Inputs to shuffle.')
shuffle_parser.set_defaults(command=shuffle)
return parser.parse_args(argv[1:])
def _parse_flags(argv: List[str]) -> Tuple[argparse.Namespace, List[str]]:
"""Parses command line arguments.
Args:
argv: Unparsed arguments for run_executor.py. Known argument names include
--executor_class_path: Python class of executor in format of
<module>.<class>.
--json_serialized_invocation_args: Full JSON-serialized parameters for
this execution.
The remaining part of the arguments will be parsed as the beam args used
by each component executors. Some commonly used beam args are as follows:
--runner: The beam pipeline runner environment. Can be DirectRunner (for
running locally) or DataflowRunner (for running on GCP Dataflow
service).
--project: The GCP project ID. Neede when runner==DataflowRunner
--direct_num_workers: Number of threads or subprocesses executing the
work load.
For more about the beam arguments please refer to:
https://cloud.google.com/dataflow/docs/guides/specifying-exec-params
Returns:
Tuple of an argparse result and remaining beam args.
"""
parser = argparse_flags.ArgumentParser()
parser.add_argument(
'--executor_class_path',
type=str,
required=True,
help='Python class of executor in format of <module>.<class>.')
parser.add_argument(
'--json_serialized_invocation_args',
type=str,
required=True,
help='JSON-serialized metadata for this execution.')
return parser.parse_known_args(argv)
def test_subparsers(self):
parser = argparse_flags.ArgumentParser(
inherited_absl_flags=self._absl_flags)
parser.add_argument('--header', help='Header message to print.')
subparsers = parser.add_subparsers(help='The command to execute.')
sub_parser = subparsers.add_parser(
'sub_cmd', help='Sub command.', inherited_absl_flags=self._absl_flags)
sub_parser.add_argument('--sub_flag', help='Sub command flag.')
def sub_command_func():
pass
sub_parser.set_defaults(command=sub_command_func)
args = parser.parse_args([
'--header=HEADER', '--absl_string=new_value', 'sub_cmd',
'--absl_integer=2', '--sub_flag=new_sub_flag_value'])
self.assertEqual(args.header, 'HEADER')
self.assertEqual(self._absl_flags.absl_string, 'new_value')
self.assertEqual(args.command, sub_command_func)
self.assertEqual(self._absl_flags.absl_integer, 2)
self.assertEqual(args.sub_flag, 'new_sub_flag_value')
def parse_args(argv):
"""Parses command line arguments."""
parser = argparse_flags.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# High-level options.
parser.add_argument(
"--verbose", "-V", action="store_true",
help="Report bitrate and distortion when training or compressing.")
parser.add_argument(
"--num_filters", type=int, default=192,
help="Number of filters per layer.")
parser.add_argument(
"--checkpoint_dir", default="train",
help="Directory where to save/load model checkpoints.")
subparsers = parser.add_subparsers(
title="commands", dest="command",
help="What to do: 'train' loads training data and trains (or continues "
"to train) a new model. 'compress' reads an image file (lossless "
"PNG format) and writes a compressed binary file. 'decompress' "
"reads a binary file and reconstructs the image (in PNG format). "
"input and output filenames need to be provided for the latter "
"two options. Invoke '<command> -h' for more information.")
# 'train' subcommand.
train_cmd = subparsers.add_parser(
"train",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Trains (or continues to train) a new model.")
train_cmd.add_argument(
"--train_glob", default="images/*.png",
help="Glob pattern identifying training data. This pattern must expand "
"to a list of RGB images in PNG format.")
train_cmd.add_argument(
"--batchsize", type=int, default=8,
help="Batch size for training.")
train_cmd.add_argument(
"--patchsize", type=int, default=256,
help="Size of image patches for training.")
train_cmd.add_argument(
"--lambda", type=float, default=0.01, dest="lmbda",
help="Lambda for rate-distortion tradeoff.")
train_cmd.add_argument(
"--last_step", type=int, default=1000000,
help="Train up to this number of steps.")
train_cmd.add_argument(
"--preprocess_threads", type=int, default=16,
help="Number of CPU threads to use for parallel decoding of training "
"images.")
# 'compress' subcommand.
compress_cmd = subparsers.add_parser(
"compress",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Reads a PNG file, compresses it, and writes a TFCI file.")
# 'decompress' subcommand.
decompress_cmd = subparsers.add_parser(
"decompress",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Reads a TFCI file, reconstructs the image, and writes back "
"a PNG file.")
# Arguments for both 'compress' and 'decompress'.
for cmd, ext in ((compress_cmd, ".tfci"), (decompress_cmd, ".png")):
cmd.add_argument(
"input_file",
help="Input filename.")
cmd.add_argument(
"output_file", nargs="?",
help="Output filename (optional). If not provided, appends '{}' to "
"the input filename.".format(ext))
# Parse arguments.
args = parser.parse_args(argv[1:])
if args.command is None:
parser.print_usage()
sys.exit(2)
return args
class Error(Exception):
pass
class ErrorDuringRepacking(Error):
pass
# This is an absl flag, and not an argparse flag, for backwards compatibility.
flags.DEFINE_bool("verbose",
default=False,
help="Turn on verbose logging.",
allow_override=True)
parser = argparse_flags.ArgumentParser(
description="A tool for building client binaries.")
# Initialize sub parsers and their arguments.
subparsers = parser.add_subparsers(title="subcommands",
dest="subparser_name",
description="valid subcommands")
# generate config
parser_generate_config = subparsers.add_parser("generate_client_config",
help="Generate client config.")
parser_generate_config.add_argument("--client_config_output",
help="Filename to write output.",
required=True,
default=None)
def parse_args(argv):
"""Parses command line arguments."""
parser = argparse_flags.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# High-level options.
parser.add_argument("--url_prefix",
default=URL_PREFIX,
help="URL prefix for downloading model metagraphs.")
parser.add_argument("--metagraph_cache",
default=METAGRAPH_CACHE,
help="Directory where to cache model metagraphs.")
subparsers = parser.add_subparsers(
title="commands",
dest="command",
help="Invoke '<command> -h' for more information.")
# 'compress' subcommand.
compress_cmd = subparsers.add_parser(
"compress",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Reads a PNG file, compresses it using the given model, and "
"writes a TFCI file.")
compress_cmd.add_argument(
"model",
help="Unique model identifier. See 'models' command for options. If "
"'target_bpp' is provided, don't specify the index at the end of "
"the model identifier.")
compress_cmd.add_argument(
"--target_bpp",
type=float,
help=
"Target bits per pixel. If provided, a binary search is used to try "
"to match the given bpp as close as possible. In this case, don't "
"specify the index at the end of the model identifier. It will be "
"automatically determined.")
compress_cmd.add_argument(
"--bpp_strict",
action="store_true",
help="Try never to exceed 'target_bpp'. Ignored if 'target_bpp' is not "
"set.")
# 'decompress' subcommand.
decompress_cmd = subparsers.add_parser(
"decompress",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Reads a TFCI file, reconstructs the image using the model "
"it was compressed with, and writes back a PNG file.")
# Arguments for both 'compress' and 'decompress'.
for cmd, ext in ((compress_cmd, ".tfci"), (decompress_cmd, ".png")):
cmd.add_argument("input_file", help="Input filename.")
cmd.add_argument(
"output_file",
nargs="?",
help="Output filename (optional). If not provided, appends '{}' to "
"the input filename.".format(ext))
# 'models' subcommand.
subparsers.add_parser(
"models",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Lists available trained models. Requires an internet "
"connection.")
# Parse arguments.
args = parser.parse_args(argv[1:])
if args.command is None:
parser.print_usage()
sys.exit(2)
return args
def local_parse_args(args):
parser = argparse_flags.ArgumentParser()
parse_args(args, parser)
return parser.parse_args(args[1:])
def parse_args(argv):
"""Parses command line arguments."""
parser = argparse_flags.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# High-level options.
parser.add_argument(
"--verbose", "-V", action="store_true",
help="Report progress and metrics when training or compressing.")
parser.add_argument(
"--model_path", default="ms2020",
help="Path where to save/load the trained model.")
subparsers = parser.add_subparsers(
title="commands", dest="command",
help="What to do: 'train' loads training data and trains (or continues "
"to train) a new model. 'compress' reads an image file (lossless "
"PNG format) and writes a compressed binary file. 'decompress' "
"reads a binary file and reconstructs the image (in PNG format). "
"input and output filenames need to be provided for the latter "
"two options. Invoke '<command> -h' for more information.")
# 'train' subcommand.
train_cmd = subparsers.add_parser(
"train",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Trains (or continues to train) a new model. Note that this "
"model trains on a continuous stream of patches drawn from "
"the training image dataset. An epoch is always defined as "
"the same number of batches given by --steps_per_epoch. "
"The purpose of validation is mostly to evaluate the "
"rate-distortion performance of the model using actual "
"quantization rather than the differentiable proxy loss. "
"Note that when using custom training images, the validation "
"set is simply a random sampling of patches from the "
"training set.")
train_cmd.add_argument(
"--lambda", type=float, default=0.01, dest="lmbda",
help="Lambda for rate-distortion tradeoff.")
train_cmd.add_argument(
"--train_glob", type=str, default=None,
help="Glob pattern identifying custom training data. This pattern must "
"expand to a list of RGB images in PNG format. If unspecified, the "
"CLIC dataset from TensorFlow Datasets is used.")
train_cmd.add_argument(
"--num_filters", type=int, default=192,
help="Number of filters per layer.")
train_cmd.add_argument(
"--latent_depth", type=int, default=320,
help="Number of filters of the last layer of the analysis transform.")
train_cmd.add_argument(
"--hyperprior_depth", type=int, default=192,
help="Number of filters of the last layer of the hyper-analysis "
"transform.")
train_cmd.add_argument(
"--num_slices", type=int, default=10,
help="Number of channel slices for conditional entropy modeling.")
train_cmd.add_argument(
"--max_support_slices", type=int, default=5,
help="Maximum number of preceding slices to condition the current slice "
"on. See Appendix C.1 of the paper for details.")
train_cmd.add_argument(
"--num_scales", type=int, default=64,
help="Number of Gaussian scales to prepare range coding tables for.")
train_cmd.add_argument(
"--scale_min", type=float, default=.11,
help="Minimum value of standard deviation of Gaussians.")
train_cmd.add_argument(
"--scale_max", type=float, default=256.,
help="Maximum value of standard deviation of Gaussians.")
train_cmd.add_argument(
"--train_path", default="/tmp/train_ms2020",
help="Path where to log training metrics for TensorBoard and back up "
"intermediate model checkpoints.")
train_cmd.add_argument(
"--batchsize", type=int, default=8,
help="Batch size for training and validation.")
train_cmd.add_argument(
"--patchsize", type=int, default=256,
help="Size of image patches for training and validation.")
train_cmd.add_argument(
"--epochs", type=int, default=1000,
help="Train up to this number of epochs. (One epoch is here defined as "
"the number of steps given by --steps_per_epoch, not iterations "
"over the full training dataset.)")
train_cmd.add_argument(
"--steps_per_epoch", type=int, default=1000,
help="Perform validation and produce logs after this many batches.")
train_cmd.add_argument(
"--max_validation_steps", type=int, default=16,
help="Maximum number of batches to use for validation. If -1, use one "
"patch from each image in the training set.")
train_cmd.add_argument(
"--preprocess_threads", type=int, default=16,
help="Number of CPU threads to use for parallel decoding of training "
"images.")
train_cmd.add_argument(
"--check_numerics", action="store_true",
help="Enable TF support for catching NaN and Inf in tensors.")
# 'compress' subcommand.
compress_cmd = subparsers.add_parser(
"compress",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Reads a PNG file, compresses it, and writes a TFCI file.")
# 'decompress' subcommand.
decompress_cmd = subparsers.add_parser(
"decompress",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Reads a TFCI file, reconstructs the image, and writes back "
"a PNG file.")
# Arguments for both 'compress' and 'decompress'.
for cmd, ext in ((compress_cmd, ".tfci"), (decompress_cmd, ".png")):
cmd.add_argument(
"input_file",
help="Input filename.")
cmd.add_argument(
"output_file", nargs="?",
help=f"Output filename (optional). If not provided, appends '{ext}' to "
f"the input filename.")
# Parse arguments.
args = parser.parse_args(argv[1:])
if args.command is None:
parser.print_usage()
sys.exit(2)
return args
def configure(self, argv=("", ), **kwargs):
"""Configures TensorBoard behavior via flags.
This method will populate the "flags" property with an argparse.Namespace
representing flag values parsed from the provided argv list, overridden by
explicit flags from remaining keyword arguments.
Args:
argv: Can be set to CLI args equivalent to sys.argv; the first arg is
taken to be the name of the path being executed.
kwargs: Additional arguments will override what was parsed from
argv. They must be passed as Python data structures, e.g.
`foo=1` rather than `foo="1"`.
Returns:
Either argv[:1] if argv was non-empty, or [''] otherwise, as a mechanism
for absl.app.run() compatibility.
Raises:
ValueError: If flag values are invalid.
"""
base_parser = argparse_flags.ArgumentParser(
prog="tensorboard",
description=(
"TensorBoard is a suite of web applications for "
"inspecting and understanding your TensorFlow runs "
"and graphs. https://github.com/tensorflow/tensorboard "),
)
subparsers = base_parser.add_subparsers(
help="TensorBoard subcommand (defaults to %r)" %
_SERVE_SUBCOMMAND_NAME)
serve_subparser = subparsers.add_parser(
_SERVE_SUBCOMMAND_NAME,
help="start local TensorBoard server (default subcommand)",
)
serve_subparser.set_defaults(
**{_SUBCOMMAND_FLAG: _SERVE_SUBCOMMAND_NAME})
if len(argv) < 2 or argv[1].startswith("-"):
# This invocation, if valid, must not use any subcommands: we
# don't permit flags before the subcommand name.
serve_parser = base_parser
else:
# This invocation, if valid, must use a subcommand: we don't take
# any positional arguments to `serve`.
serve_parser = serve_subparser
for (name, subcommand) in self.subcommands.items():
subparser = subparsers.add_parser(
name,
help=subcommand.help(),
description=subcommand.description(),
)
subparser.set_defaults(**{_SUBCOMMAND_FLAG: name})
subcommand.define_flags(subparser)
for loader in self.plugin_loaders:
loader.define_flags(serve_parser)
arg0 = argv[0] if argv else ""
flags = base_parser.parse_args(
argv[1:]) # Strip binary name from argv.
if getattr(flags, _SUBCOMMAND_FLAG, None) is None:
# Manually assign default value rather than using `set_defaults`
# on the base parser to work around Python bug #9351 on old
# versions of `argparse`: <https://bugs.python.org/issue9351>
setattr(flags, _SUBCOMMAND_FLAG, _SERVE_SUBCOMMAND_NAME)
self.cache_key = manager.cache_key(
working_directory=os.getcwd(),
arguments=argv[1:],
configure_kwargs=kwargs,
)
if arg0:
# Only expose main module Abseil flags as TensorBoard native flags.
# This is the same logic Abseil's ArgumentParser uses for determining
# which Abseil flags to include in the short helpstring.
for flag in set(absl_flags.FLAGS.get_key_flags_for_module(arg0)):
if hasattr(flags, flag.name):
raise ValueError("Conflicting Abseil flag: %s" % flag.name)
setattr(flags, flag.name, flag.value)
for k, v in kwargs.items():
if not hasattr(flags, k):
raise ValueError("Unknown TensorBoard flag: %s" % k)
setattr(flags, k, v)
if getattr(flags, _SUBCOMMAND_FLAG) == _SERVE_SUBCOMMAND_NAME:
for loader in self.plugin_loaders:
loader.fix_flags(flags)
self.flags = flags
return [arg0]
model_resolver,
evaluator,
pusher,
],
enable_cache=True,
metadata_connection_config=metadata.sqlite_metadata_connection_config(
metadata_path),
beam_pipeline_args=beam_pipeline_args)
# To run this pipeline from the python CLI:
# $python taxi_pipeline_beam.py
if __name__ == '__main__':
absl.logging.set_verbosity(absl.logging.INFO)
parser = argparse_flags.ArgumentParser()
parser.add_argument(
'--runner',
type=str,
default='DirectRunner',
choices=['DirectRunner', 'FlinkRunner', 'SparkRunner'],
help='The Beam runner to execute Beam-powered components. '
'For FlinkRunner or SparkRunner, first run setup/setup_beam_on_flink.sh '
'or setup/setup_beam_on_spark.sh, respectively.')
parsed_args, _ = parser.parse_known_args(sys.argv)
BeamDagRunner().run(
_create_pipeline(pipeline_name=_pipeline_name,
pipeline_root=_pipeline_root,
data_root=_data_root,
module_file=_module_file,
def parse_args(argv):
"""Parses command line arguments."""
parser = argparse_flags.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# High-level options.
parser.add_argument(
"--url_prefix",
default=URL_PREFIX,
help="URL prefix for downloading model metagraphs.")
parser.add_argument(
"--metagraph_cache",
default=METAGRAPH_CACHE,
help="Directory where to cache model metagraphs.")
subparsers = parser.add_subparsers(
title="commands", dest="command",
help="Invoke '<command> -h' for more information.")
# 'compress' subcommand.
compress_cmd = subparsers.add_parser(
"compress",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Reads a PNG file, compresses it using the given model, and "
"writes a TFCI file.")
compress_cmd.add_argument(
"model",
help="Unique model identifier. See 'models' command for options. If "
"'target_bpp' is provided, don't specify the index at the end of "
"the model identifier.")
compress_cmd.add_argument(
"--rd_parameter", "-r", type=float,
help="Rate-distortion parameter (for some models). Ignored if "
"'target_bpp' is set.")
compress_cmd.add_argument(
"--rd_parameter_tolerance", type=float,
default=2 ** -4,
help="Tolerance for rate-distortion parameter. Only used if 'target_bpp' "
"is set for some models, to determine when to stop the binary "
"search.")
compress_cmd.add_argument(
"--target_bpp", "-b", type=float,
help="Target bits per pixel. If provided, a binary search is used to try "
"to match the given bpp as close as possible. In this case, don't "
"specify the index at the end of the model identifier. It will be "
"automatically determined.")
compress_cmd.add_argument(
"--bpp_strict", action="store_true",
help="Try never to exceed 'target_bpp'. Ignored if 'target_bpp' is not "
"set.")
# 'decompress' subcommand.
decompress_cmd = subparsers.add_parser(
"decompress",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Reads a TFCI file, reconstructs the image using the model "
"it was compressed with, and writes back a PNG file.")
# 'models' subcommand.
subparsers.add_parser(
"models",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Lists available trained models. Requires an internet "
"connection.")
tensors_cmd = subparsers.add_parser(
"tensors",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Lists names of internal tensors of a given model.")
tensors_cmd.add_argument(
"model",
help="Unique model identifier. See 'models' command for options.")
dump_cmd = subparsers.add_parser(
"dump",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Dumps values of given internal tensors of a model in "
"NumPy's .npz format.")
dump_cmd.add_argument(
"model",
help="Unique model identifier. See 'models' command for options.")
dump_cmd.add_argument(
"--tensor", "-t", nargs="+",
help="Name(s) of tensor(s) to dump. Must provide at least one. See "
"'tensors' command for options.")
# Arguments for 'compress', 'decompress', and 'dump'.
for cmd, ext in (
(compress_cmd, ".tfci"),
(decompress_cmd, ".png"),
(dump_cmd, ".npz"),
):
cmd.add_argument(
"input_file",
help="Input filename.")
cmd.add_argument(
"output_file", nargs="?",
help=f"Output filename (optional). If not provided, appends '{ext}' to "
f"the input filename.")
# Parse arguments.
args = parser.parse_args(argv[1:])
if args.command is None:
parser.print_usage()
sys.exit(2)
return args
def test_undefok_flag_incorrect_use(self, cmd_args):
parser = argparse_flags.ArgumentParser(
inherited_absl_flags=self._absl_flags)
with self.assertRaises(SystemExit):
parser.parse_args(cmd_args)
def test_dash_as_prefix_char_only(self):
with self.assertRaises(ValueError):
argparse_flags.ArgumentParser(prefix_chars='/')
def test_default_inherited_absl_flags_value(self): parser = argparse_flags.ArgumentParser() self.assertIs(parser._inherited_absl_flags, flags.FLAGS)
def parse_args(argv):
"""Parses command line arguments."""
parser = argparse_flags.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# High-level options.
parser.add_argument(
"--verbose",
"-V",
action="store_true",
help="Report bitrate and distortion when training or compressing.")
parser.add_argument("--num_filters",
type=int,
default=64,
help="Number of filters per layer.")
parser.add_argument("--checkpoint_dir",
default="train",
help="Directory where to save/load model checkpoints.")
parser.add_argument("--gpu",
type=str,
default='2',
help="Which gpu to use")
subparsers = parser.add_subparsers(
title="commands",
dest="command",
help="What to do: 'train' loads training data and trains (or continues "
"to train) a new model. 'compress' reads an image file (lossless "
"PNG format) and writes a compressed binary file. 'decompress' "
"reads a binary file and reconstructs the image (in PNG format). "
"input and output filenames need to be provided for the latter "
"two options. Invoke '<command> -h' for more information.")
# 'train' subcommand.
train_cmd = subparsers.add_parser(
"train",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Trains (or continues to train) a new model.")
train_cmd.add_argument(
"--train_glob",
default=
"/datatmp/Datasets/CLIC_2019_Professional/CLIC_2019_Images/train/*.png",
help="Glob pattern identifying training data. This pattern must expand "
"to a list of RGB images in PNG format.")
train_cmd.add_argument(
"--valid_glob",
default="./valid_high/*.png",
help="Glob pattern identifying testing data. This pattern must expand "
"to a list of RGB images in PNG format.")
train_cmd.add_argument(
"--test_glob",
default=
"/datatmp/Datasets/CLIC_2019_Professional/CLIC_2019_Images/valid/*.png",
help="Glob pattern identifying testing data. This pattern must expand "
"to a list of RGB images in PNG format.")
train_cmd.add_argument(
"--kodak_glob",
default="./kodak/*.png",
help="Glob pattern identifying testing data. This pattern must expand "
"to a list of RGB images in PNG format.")
train_cmd.add_argument(
"--experiment_name",
default="64filters_depthwise_encoder_only_noentropy",
help="Directory where to save/load model checkpoints.")
train_cmd.add_argument("--batchsize",
type=int,
default=8,
help="Batch size for training.")
train_cmd.add_argument("--patchsize",
type=int,
default=256,
help="Size of image patches for training.")
train_cmd.add_argument("--lambda",
type=float,
default=0.002,
dest="lmbda",
help="Lambda for rate-distortion tradeoff.")
train_cmd.add_argument("--last_step",
type=int,
default=800000,
help="Train up to this number of steps.")
train_cmd.add_argument(
"--preprocess_threads",
type=int,
default=12,
help="Number of CPU threads to use for parallel decoding of training "
"images.")
train_cmd.add_argument("--valid_patchsize", type=int, default=256)
# 'compress' subcommand.
compress_cmd = subparsers.add_parser(
"compress",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Reads a PNG file, compresses it, and writes a TFCI file.")
# 'decompress' subcommand.
decompress_cmd = subparsers.add_parser(
"decompress",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Reads a TFCI file, reconstructs the image, and writes back "
"a PNG file.")
# Arguments for both 'compress' and 'decompress'.
for cmd, ext in ((compress_cmd, ".tfci"), (decompress_cmd, ".png")):
cmd.add_argument("input_file", help="Input filename.")
cmd.add_argument(
"output_file",
nargs="?",
help="Output filename (optional). If not provided, appends '{}' to "
"the input filename.".format(ext))
# Sudeep's arguments
parser.add_argument('--ssim', dest='ssim_loss', action='store_true')
parser.set_defaults(ssim_loss=False)
parser.add_argument('--test_only', dest='test_only', action='store_true')
parser.set_defaults(test_only=False)
parser.add_argument('--save_compressed_reconstructed',
dest='save',
action='store_true')
parser.set_defaults(save=False)
parser.add_argument('--kodak', dest='kodak', action='store_true')
parser.set_defaults(kodak=False)
# Parse arguments.
args = parser.parse_args(argv[1:])
args.test_only = args.test_only or args.kodak
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
if args.command is None:
parser.print_usage()
sys.exit(2)
return args
from grr_response_client_builder import repacking
from grr_response_core import config as grr_config
from grr_response_core.config import contexts
from grr_response_core.config import server as config_server
from grr_response_core.lib import config_lib
from grr_response_server import artifact
from grr_response_server import artifact_registry
from grr_response_server import maintenance_utils
from grr_response_server import server_startup
from grr_response_server.bin import config_updater_keys_util
from grr_response_server.bin import config_updater_util
from grr_response_server.rdfvalues import objects as rdf_objects
parser = argparse_flags.ArgumentParser(
description=("Set configuration parameters for the GRR Server."
"\nThis script has numerous subcommands to perform "
"various actions. When you are first setting up, you "
"probably only care about 'initialize'."))
# Generic arguments.
parser.add_argument(
"--version",
action="version",
version=config_server.VERSION["packageversion"],
help="Print config updater version number and exit immediately.")
subparsers = parser.add_subparsers(title="subcommands",
dest="subparser_name",
description="valid subcommands")
# Subparsers.
