def define_data_download_flags():
"""Add flags specifying data download arguments."""
flags.DEFINE_string(
name="data_dir", default="/tmp/movielens-data/",
help=flags_core.help_wrap(
"Directory to download and extract data."))
flags.DEFINE_enum(
name="dataset", default=None,
enum_values=DATASETS, case_sensitive=False,
help=flags_core.help_wrap("Dataset to be trained and evaluated."))
def model_specific_flags_embed_cnn():
flags.DEFINE_string(name="cnn_filter_sizes",
default="3,4,5",
help=flags_core.help_wrap(
"Comma-separated filter sizes (default: '3,4,5')"))
flags.DEFINE_integer(
name="cnn_num_filters",
default=128,
help=flags_core.help_wrap(
"Number of filters per filter size (default: 128)"))
def define_seq2tfrec_flags():
flags.DEFINE_string(name="input_seq",
short_name="seq",
default="/tmp/input.fasta",
help=flags_core.help_wrap("Path to input reads"))
flags.DEFINE_string(name="output_tfrec",
short_name="tfrec",
default="/tmp/output.tfrec",
help=flags_core.help_wrap("Path to output tfrecord"))
flags.DEFINE_string(
name="vocab",
default="/tmp/tokens_12mer.txt",
help=flags_core.help_wrap("Path to the vocabulary file"))
flags.DEFINE_string(name="label",
default="/tmp/label2taxid.txt",
help=flags_core.help_wrap("Path to the label file"))
flags.DEFINE_string(name="seq_type",
default="fasta",
help=flags_core.help_wrap(
"Input format, fasta or fastq (default fasta)"))
flags.DEFINE_integer(name="kmer",
short_name="k",
default=12,
help=flags_core.help_wrap(
"The size of k for reads splitting (default 12)"))
flags.DEFINE_boolean(
name="is_train",
default=True,
help=flags_core.help_wrap(
"Whether processing training/eval set (default True)"))
def define_compute_bleu_flags():
"""Add flags for computing BLEU score."""
flags.DEFINE_string(
name="translation", default=None,
help=flags_core.help_wrap("File containing translated text."))
flags.mark_flag_as_required("translation")
flags.DEFINE_string(
name="reference", default=None,
help=flags_core.help_wrap("File containing reference translation."))
flags.mark_flag_as_required("reference")
flags.DEFINE_enum(
name="bleu_variant", short_name="bv", default="both",
enum_values=["both", "uncased", "cased"], case_sensitive=False,
help=flags_core.help_wrap(
"Specify one or more BLEU variants to calculate. Variants: \"cased\""
", \"uncased\", or \"both\"."))
def define_resnet_flags(resnet_size_choices=None):
"""Add flags and validators for ResNet."""
flags_core.define_base()
flags_core.define_performance(num_parallel_calls=False)
flags_core.define_image()
flags_core.define_benchmark()
flags.adopt_module_key_flags(flags_core)
flags.DEFINE_enum(
name='resnet_version',
short_name='rv',
default='1',
enum_values=['1', '2'],
help=flags_core.help_wrap(
'Version of ResNet. (1 or 2) See README.md for details.'))
flags.DEFINE_bool(
name='fine_tune',
short_name='ft',
default=False,
help=flags_core.help_wrap(
'If True do not train any parameters except for the final layer.'))
flags.DEFINE_string(
name='pretrained_model_checkpoint_path',
short_name='pmcp',
default=None,
help=flags_core.help_wrap(
'If not None initialize all the network except the final layer with '
'these values'))
flags.DEFINE_boolean(name='eval_only',
default=False,
help=flags_core.help_wrap(
'Skip training and only perform evaluation on '
'the latest checkpoint.'))
choice_kwargs = dict(
name='resnet_size',
short_name='rs',
default='50',
help=flags_core.help_wrap('The size of the ResNet model to use.'))
if resnet_size_choices is None:
flags.DEFINE_string(**choice_kwargs)
else:
flags.DEFINE_enum(enum_values=resnet_size_choices, **choice_kwargs)
def define_data_download_flags():
"""Add flags specifying data download arguments."""
flags.DEFINE_string(
name="data_dir",
short_name="dd",
default="/tmp/translate_ende",
help=flags_core.help_wrap(
"Directory for where the translate_ende_wmt32k dataset is saved."))
flags.DEFINE_string(
name="raw_dir",
short_name="rd",
default="/tmp/translate_ende_raw",
help=flags_core.help_wrap(
"Path where the raw data will be downloaded and extracted."))
flags.DEFINE_bool(
name="search",
default=False,
help=flags_core.help_wrap(
"If set, use binary search to find the vocabulary set with size"
"closest to the target size (%d)." % _TARGET_VOCAB_SIZE))
def define_mnist_eager_flags():
"""Defined flags and defaults for MNIST in eager mode."""
flags_core.define_base_eager()
flags_core.define_image()
flags.adopt_module_key_flags(flags_core)
flags.DEFINE_integer(
name='log_interval',
short_name='li',
default=10,
help=flags_core.help_wrap('batches between logging training status'))
flags.DEFINE_string(
name='output_dir',
short_name='od',
default='/tmp/tensorflow/mnist/',
help=flags_core.help_wrap('Directory to write TensorBoard summaries'))
flags.DEFINE_float(name='learning_rate',
short_name='lr',
default=0.01,
help=flags_core.help_wrap('Learning rate.'))
flags.DEFINE_float(name='momentum',
short_name='m',
default=0.5,
help=flags_core.help_wrap('SGD momentum.'))
flags.DEFINE_bool(name='no_gpu',
short_name='nogpu',
default=False,
help=flags_core.help_wrap(
'disables GPU usage even if a GPU is available'))
flags_core.set_defaults(
data_dir='/tmp/tensorflow/mnist/input_data',
model_dir='/tmp/tensorflow/mnist/checkpoints/',
batch_size=100,
train_epochs=10,
)
def model_specific_flags_embed_lstm():
flags.DEFINE_integer(
name="lstm_dim",
default=300,
help=flags_core.help_wrap(
"Dimension of one direction of biLSTM (default: 300)"))
flags.DEFINE_integer(name='da',
default=350,
help=flags_core.help_wrap('Da.'))
flags.DEFINE_integer(
name='row',
default=30,
help=flags_core.help_wrap('Number of rows of embedding matrix.'))
flags.DEFINE_string(
name="pooling_type",
default='concat',
help=flags_core.help_wrap(
"Type of pooling: avg/max/concat (default: concat)"))
def define_resnet_flags(resnet_size_choices=None):
"""Add flags and validators for ResNet."""
flags_core.define_base()
flags_core.define_performance(num_parallel_calls=False)
flags_core.define_image()
flags_core.define_benchmark()
flags.adopt_module_key_flags(flags_core)
flags.DEFINE_enum(
name='resnet_version',
short_name='rv',
default='2',
enum_values=['1', '2'],
help=flags_core.help_wrap(
'Version of ResNet. (1 or 2) See README.md for details.'))
choice_kwargs = dict(
name='resnet_size',
short_name='rs',
default='50',
help=flags_core.help_wrap('The size of the ResNet model to use.'))
if resnet_size_choices is None:
flags.DEFINE_string(**choice_kwargs)
else:
flags.DEFINE_enum(enum_values=resnet_size_choices, **choice_kwargs)
# The current implementation of ResNet v1 is numerically unstable when run
# with fp16 and will produce NaN errors soon after training begins.
msg = ('ResNet version 1 is not currently supported with fp16. '
'Please use version 2 instead.')
@flags.multi_flags_validator(['dtype', 'resnet_version'], message=msg)
def _forbid_v1_fp16(flag_values): # pylint: disable=unused-variable
return (flags_core.DTYPE_MAP[flag_values['dtype']][0] != tf.float16
or flag_values['resnet_version'] != '1')
def define_trial_run_flags():
flags.DEFINE_string(
name='resource',
default='',
help='resource specification')
flags.DEFINE_integer(
name='trial_steps',
default=100,
help='number of steps for trial')
flags.DEFINE_enum(
name="optimizer",
default="adam",
enum_values=[
"adam",
"lazyadam",
"sgd"],
case_sensitive=False,
help=flags_core.help_wrap("optimizer to use."))
flags.DEFINE_bool(
name='dense_gradient',
default='True',
help='whether to use dense gradient')
flags.DEFINE_string(
name='simulation_strategy_id',
default=None,
help='strategy id to simulate')
flags.DEFINE_string(
name='simulation_folder',
default=None,
help='folder to store simulation result')
flags.DEFINE_string(
name='autodist_strategy',
default='PS',
help='the autodist strategy')
flags.DEFINE_boolean(
name='autodist_patch_tf',
default=True,
help='AUTODIST_PATCH_TF')
flags.DEFINE_boolean(name='proxy', default=True, help='proxy')
flags.DEFINE_string(
name='default_data_dir',
default='~/movielens',
help='the default data directory')
flags.DEFINE_integer(
name='num_epochs',
default=3,
help='number of training epochs')
def define_meal_flags():
"""Define flags for meal dataset training."""
wide_deep_run_loop.define_wide_deep_flags()
flags.DEFINE_enum(
name="dataset", default=meals.FOOD,
enum_values=meals.DATASETS, case_sensitive=False,
help=flags_core.help_wrap("Dataset to be trained and evaluated."))
flags.adopt_module_key_flags(wide_deep_run_loop)
flags_core.set_defaults(data_dir="../../dataset/csv_file/",
model_dir='../model_saved_ckp/',
train_epochs=150,
epochs_between_evals=5,
inter_op_parallelism_threads=0,
intra_op_parallelism_threads=0,
batch_size=40)
@flags.validator("stop_threshold",
message="stop_threshold not supported for meals model")
def _no_stop(stop_threshold):
return stop_threshold is None
def define_translate_flags():
"""Define flags used for translation script."""
# Model flags
flags.DEFINE_string(
name="model_dir",
short_name="md",
default="/tmp/transformer_model",
help=flags_core.help_wrap(
"Directory containing Transformer model checkpoints."))
flags.DEFINE_enum(
name="param_set",
short_name="mp",
default="big",
enum_values=["base", "big", "tiny"],
help=flags_core.help_wrap(
"Parameter set to use when creating and training the model. The "
"parameters define the input shape (batch size and max length), "
"model configuration (size of embedding, # of hidden layers, etc.), "
"and various other settings. The big parameter set increases the "
"default batch size, embedding/hidden size, and filter size. For a "
"complete list of parameters, please see model/model_params.py."))
flags.DEFINE_string(
name="vocab_file",
short_name="vf",
default=None,
help=flags_core.help_wrap(
"Path to subtoken vocabulary file. If data_download.py was used to "
"download and encode the training data, look in the data_dir to find "
"the vocab file."))
flags.mark_flag_as_required("vocab_file")
flags.DEFINE_string(
name="text",
default=None,
help=flags_core.help_wrap(
"Text to translate. Output will be printed to console."))
flags.DEFINE_string(
name="file",
default=None,
help=flags_core.help_wrap(
"File containing text to translate. Translation will be printed to "
"console and, if --file_out is provided, saved to an output file.")
)
flags.DEFINE_string(
name="file_out",
default=None,
help=flags_core.help_wrap(
"If --file flag is specified, save translation to this file."))
def define_wide_deep_flags():
"""Add supervised learning flags, as well as wide-deep model type."""
flags_core.define_base()
flags_core.define_benchmark()
flags_core.define_performance(num_parallel_calls=False,
inter_op=True,
intra_op=True,
synthetic_data=False,
max_train_steps=False,
dtype=False,
all_reduce_alg=False)
flags.adopt_module_key_flags(flags_core)
flags.DEFINE_enum(name="model_type",
short_name="mt",
default="wide_deep",
enum_values=['wide', 'deep', 'wide_deep'],
help="Select model topology.")
flags.DEFINE_boolean(
name="download_if_missing",
default=True,
help=flags_core.help_wrap(
"Download data to data_dir if it is not already present."))
def define_movie_flags():
"""Define flags for movie dataset training."""
wide_deep_run_loop.define_wide_deep_flags()
flags.DEFINE_enum(
name="dataset",
default=movielens.ML_1M,
enum_values=movielens.DATASETS,
case_sensitive=False,
help=flags_core.help_wrap("Dataset to be trained and evaluated."))
flags.adopt_module_key_flags(wide_deep_run_loop)
flags_core.set_defaults(data_dir="/tmp/movielens-data/",
model_dir='/tmp/movie_model',
model_type="deep",
train_epochs=50,
epochs_between_evals=5,
inter_op_parallelism_threads=0,
intra_op_parallelism_threads=0,
batch_size=256)
@flags.validator("stop_threshold",
message="stop_threshold not supported for movielens model"
)
def _no_stop(stop_threshold):
return stop_threshold is None
def flags_of_mode():
flags.DEFINE_string(
name="running_mode",
default='train',
help=flags_core.help_wrap(
"One of: train/eval/predict_prob/predict_class/predict_paired_prob/predict_paired_class (default: train)"
))
flags.DEFINE_string(
name="pred_out",
default='/tmp/pred_out',
help=flags_core.help_wrap("/path/to/prediction/output"))
flags.DEFINE_boolean(
name="strands_average",
default=True,
help=flags_core.help_wrap(
"Whether tfrec contains interleaved double strands (default: True)"
))
flags.DEFINE_integer(
name="top_n_class",
default=3,
help=flags_core.help_wrap(
"Output category indexes and probabilities for top n classes (default: 3)"
))
flags.DEFINE_string(
name="label_file",
default='/tmp/label2taxid.txt',
help=flags_core.help_wrap(
"File mapping from label to taxid (required if translate=True)"))
flags.DEFINE_boolean(
name="translate",
default=True,
help=flags_core.help_wrap("Whether output taxid instead of label."))
def define_resnet_flags(resnet_size_choices=None,
dynamic_loss_scale=False,
fp16_implementation=False):
"""Add flags and validators for ResNet."""
flags_core.define_base()
flags_core.define_performance(num_parallel_calls=False,
tf_gpu_thread_mode=True,
datasets_num_private_threads=True,
dynamic_loss_scale=dynamic_loss_scale,
fp16_implementation=fp16_implementation,
loss_scale=True,
tf_data_experimental_slack=True)
flags_core.define_image()
flags_core.define_benchmark()
flags.adopt_module_key_flags(flags_core)
flags.DEFINE_enum(
name='resnet_version',
short_name='rv',
default='2',
enum_values=['1', '2'],
help=flags_core.help_wrap(
'Version of ResNet. (1 or 2) See README.md for details.'))
flags.DEFINE_bool(
name='fine_tune',
short_name='ft',
default=False,
help=flags_core.help_wrap(
'If True do not train any parameters except for the final layer.'))
flags.DEFINE_string( # "/home/zxc/Liu/models-master-new/official/r1/resnet/model/"
name='pretrained_model_checkpoint_path',
short_name='pmcp',
default="/home/zxc/Liu/models-master-new/official/r1/resnet/model/",
# default=None,
help=flags_core.help_wrap(
'If not None initialize all the network except the final layer with '
'these values'))
flags.DEFINE_boolean(name='eval_only',
default=False,
help=flags_core.help_wrap(
'Skip training and only perform evaluation on '
'the latest checkpoint.'))
flags.DEFINE_boolean(
name='image_bytes_as_serving_input',
default=False,
help=flags_core.help_wrap(
'If True exports savedmodel with serving signature that accepts '
'JPEG image bytes instead of a fixed size [HxWxC] tensor that '
'represents the image. The former is easier to use for serving at '
'the expense of image resize/cropping being done as part of model '
'inference. Note, this flag only applies to ImageNet and cannot '
'be used for CIFAR.'))
flags.DEFINE_boolean(
name='use_train_and_evaluate',
default=False,
help=flags_core.help_wrap(
'If True, uses `tf.estimator.train_and_evaluate` for the training '
'and evaluation loop, instead of separate calls to `classifier.train '
'and `classifier.evaluate`, which is the default behavior.'))
flags.DEFINE_string(
name='worker_hosts',
default=None,
help=flags_core.help_wrap(
'Comma-separated list of worker ip:port pairs for running '
'multi-worker models with DistributionStrategy. The user would '
'start the program on each host with identical value for this flag.'
))
flags.DEFINE_integer(name='task_index',
default=-1,
help=flags_core.help_wrap(
'If multi-worker training, the task_index of '
'this worker.'))
flags.DEFINE_bool(name='enable_lars',
default=False,
help=flags_core.help_wrap(
'Enable LARS optimizer for large batch training.'))
flags.DEFINE_float(
name='label_smoothing',
default=0.0,
help=flags_core.help_wrap(
'Label smoothing parameter used in the softmax_cross_entropy'))
flags.DEFINE_float(name='weight_decay',
default=1e-4,
help=flags_core.help_wrap(
'Weight decay coefficiant for l2 regularization.'))
choice_kwargs = dict(
name='resnet_size',
short_name='rs',
default='50',
help=flags_core.help_wrap('The size of the ResNet model to use.'))
if resnet_size_choices is None:
flags.DEFINE_string(**choice_kwargs)
else:
flags.DEFINE_enum(enum_values=resnet_size_choices, **choice_kwargs)
def define_transformer_flags():
"""Add flags and flag validators for running transformer_main."""
# Add common flags (data_dir, model_dir, train_epochs, etc.).
flags_core.define_base()
flags_core.define_performance(
num_parallel_calls=True,
inter_op=False,
intra_op=False,
synthetic_data=True,
max_train_steps=False,
dtype=False,
all_reduce_alg=True
)
flags_core.define_benchmark()
flags_core.define_device(tpu=True)
# Set flags from the flags_core module as "key flags" so they're listed when
# the '-h' flag is used. Without this line, the flags defined above are
# only shown in the full `--helpful` help text.
flags.adopt_module_key_flags(flags_core)
# Add transformer-specific flags
flags.DEFINE_enum(
name="param_set", short_name="mp", default="big",
enum_values=PARAMS_MAP.keys(),
help=flags_core.help_wrap(
"Parameter set to use when creating and training the model. The "
"parameters define the input shape (batch size and max length), "
"model configuration (size of embedding, # of hidden layers, etc.), "
"and various other settings. The big parameter set increases the "
"default batch size, embedding/hidden size, and filter size. For a "
"complete list of parameters, please see model/model_params.py."))
flags.DEFINE_bool(
name="static_batch", default=False,
help=flags_core.help_wrap(
"Whether the batches in the dataset should have static shapes. In "
"general, this setting should be False. Dynamic shapes allow the "
"inputs to be grouped so that the number of padding tokens is "
"minimized, and helps model training. In cases where the input shape "
"must be static (e.g. running on TPU), this setting will be ignored "
"and static batching will always be used."))
# Flags for training with steps (may be used for debugging)
flags.DEFINE_integer(
name="train_steps", short_name="ts", default=None,
help=flags_core.help_wrap("The number of steps used to train."))
flags.DEFINE_integer(
name="steps_between_evals", short_name="sbe", default=1000,
help=flags_core.help_wrap(
"The Number of training steps to run between evaluations. This is "
"used if --train_steps is defined."))
# BLEU score computation
flags.DEFINE_string(
name="bleu_source", short_name="bls", default=None,
help=flags_core.help_wrap(
"Path to source file containing text translate when calculating the "
"official BLEU score. Both --bleu_source and --bleu_ref must be set. "
"Use the flag --stop_threshold to stop the script based on the "
"uncased BLEU score."))
flags.DEFINE_string(
name="bleu_ref", short_name="blr", default=None,
help=flags_core.help_wrap(
"Path to source file containing text translate when calculating the "
"official BLEU score. Both --bleu_source and --bleu_ref must be set. "
"Use the flag --stop_threshold to stop the script based on the "
"uncased BLEU score."))
flags.DEFINE_string(
name="vocab_file", short_name="vf", default=None,
help=flags_core.help_wrap(
"Path to subtoken vocabulary file. If data_download.py was used to "
"download and encode the training data, look in the data_dir to find "
"the vocab file."))
flags_core.set_defaults(data_dir="/tmp/translate_ende",
model_dir="/tmp/transformer_model",
batch_size=None,
train_epochs=None)
@flags.multi_flags_validator(
["train_epochs", "train_steps"],
message="Both --train_steps and --train_epochs were set. Only one may be "
"defined.")
def _check_train_limits(flag_dict):
return flag_dict["train_epochs"] is None or flag_dict["train_steps"] is None
@flags.multi_flags_validator(
["bleu_source", "bleu_ref"],
message="Both or neither --bleu_source and --bleu_ref must be defined.")
def _check_bleu_files(flags_dict):
return (flags_dict["bleu_source"] is None) == (
flags_dict["bleu_ref"] is None)
@flags.multi_flags_validator(
["bleu_source", "bleu_ref", "vocab_file"],
message="--vocab_file must be defined if --bleu_source and --bleu_ref "
"are defined.")
def _check_bleu_vocab_file(flags_dict):
if flags_dict["bleu_source"] and flags_dict["bleu_ref"]:
return flags_dict["vocab_file"] is not None
return True
@flags.multi_flags_validator(
["export_dir", "vocab_file"],
message="--vocab_file must be defined if --export_dir is set.")
def _check_export_vocab_file(flags_dict):
if flags_dict["export_dir"]:
return flags_dict["vocab_file"] is not None
return True
flags_core.require_cloud_storage(["data_dir", "model_dir", "export_dir"])
def define_ncf_flags():
"""Add flags for running ncf_main."""
# Add common flags
flags_core.define_base(model_dir=True, clean=True, train_epochs=True,
epochs_between_evals=True, export_dir=False,
run_eagerly=True, stop_threshold=True, num_gpu=True,
hooks=True, distribution_strategy=True)
flags_core.define_performance(
synthetic_data=True,
dtype=True,
fp16_implementation=True,
loss_scale=True,
dynamic_loss_scale=True,
enable_xla=True,
force_v2_in_keras_compile=True
)
flags_core.define_device(tpu=True)
flags_core.define_benchmark()
flags.adopt_module_key_flags(flags_core)
flags_core.set_defaults(
model_dir="/tmp/ncf/",
data_dir="/tmp/movielens-data/",
train_epochs=2,
batch_size=256,
hooks="ProfilerHook",
tpu=None
)
# Add ncf-specific flags
flags.DEFINE_enum(
name="dataset", default="ml-1m",
enum_values=["ml-1m", "ml-20m", "ml-20mx16x32"], case_sensitive=False,
help=flags_core.help_wrap(
"Dataset to be trained and evaluated."))
flags.DEFINE_boolean(
name="download_if_missing", default=True, help=flags_core.help_wrap(
"Download data to data_dir if it is not already present."))
flags.DEFINE_integer(
name="eval_batch_size", default=None, help=flags_core.help_wrap(
"The batch size used for evaluation. This should generally be larger"
"than the training batch size as the lack of back propagation during"
"evaluation can allow for larger batch sizes to fit in memory. If not"
"specified, the training batch size (--batch_size) will be used."))
flags.DEFINE_integer(
name="num_factors", default=8,
help=flags_core.help_wrap("The Embedding size of MF model."))
# Set the default as a list of strings to be consistent with input
# arguments
flags.DEFINE_list(
name="layers", default=["64", "32", "16", "8"],
help=flags_core.help_wrap(
"The sizes of hidden layers for MLP. Example "
"to specify different sizes of MLP layers: --layers=32,16,8,4"))
flags.DEFINE_float(
name="mf_regularization", default=0., help=flags_core.help_wrap(
"The regularization factor for MF embeddings. The factor is used by "
"regularizer which allows to apply penalties on layer parameters or "
"layer activity during optimization."))
flags.DEFINE_list(
name="mlp_regularization", default=[
"0.", "0.", "0.", "0."], help=flags_core.help_wrap(
"The regularization factor for each MLP layer. See mf_regularization "
"help for more info about regularization factor."))
flags.DEFINE_integer(name="num_neg", default=4, help=flags_core.help_wrap(
"The Number of negative instances to pair with a positive instance."))
flags.DEFINE_float(
name="learning_rate", default=0.001,
help=flags_core.help_wrap("The learning rate."))
flags.DEFINE_float(name="beta1", default=0.9, help=flags_core.help_wrap(
"beta1 hyperparameter for the Adam optimizer."))
flags.DEFINE_float(name="beta2", default=0.999, help=flags_core.help_wrap(
"beta2 hyperparameter for the Adam optimizer."))
flags.DEFINE_float(
name="epsilon", default=1e-8,
help=flags_core.help_wrap("epsilon hyperparameter for the Adam "
"optimizer."))
flags.DEFINE_float(
name="hr_threshold", default=1.0,
help=flags_core.help_wrap(
"If passed, training will stop when the evaluation metric HR is "
"greater than or equal to hr_threshold. For dataset ml-1m, the "
"desired hr_threshold is 0.68 which is the result from the paper; "
"For dataset ml-20m, the threshold can be set as 0.95 which is "
"achieved by MLPerf implementation."))
flags.DEFINE_enum(
name="constructor_type", default="bisection",
enum_values=["bisection", "materialized"], case_sensitive=False,
help=flags_core.help_wrap(
"Strategy to use for generating false negatives. materialized has a"
"precompute that scales badly, but a faster per-epoch construction"
"time and can be faster on very large systems."))
flags.DEFINE_string(
name="train_dataset_path",
default=None,
help=flags_core.help_wrap("Path to training data."))
flags.DEFINE_string(
name="eval_dataset_path",
default=None,
help=flags_core.help_wrap("Path to evaluation data."))
flags.DEFINE_string(
name="input_meta_data_path",
default=None,
help=flags_core.help_wrap("Path to input meta data file."))
flags.DEFINE_bool(
name="ml_perf", default=False, help=flags_core.help_wrap(
"If set, changes the behavior of the model slightly to match the "
"MLPerf reference implementations here: \n"
"https://github.com/mlperf/reference/tree/master/recommendation/"
"pytorch\n"
"The two changes are:\n"
"1. When computing the HR and NDCG during evaluation, remove "
"duplicate user-item pairs before the computation. This results in "
"better HRs and NDCGs.\n"
"2. Use a different soring algorithm when sorting the input data, "
"which performs better due to the fact the sorting algorithms are "
"not stable."))
flags.DEFINE_bool(
name="output_ml_perf_compliance_logging",
default=False,
help=flags_core.help_wrap(
"If set, output the MLPerf compliance logging. This is only useful "
"if one is running the model for MLPerf. See "
"https://github.com/mlperf/policies/blob/master/training_rules.adoc"
"#submission-compliance-logs for details. This uses sudo and so may "
"ask for your password, as root access is needed to clear the system "
"caches, which is required for MLPerf compliance."))
flags.DEFINE_integer(
name="seed", default=None, help=flags_core.help_wrap(
"This value will be used to seed both NumPy and TensorFlow."))
@flags.validator("eval_batch_size", "eval_batch_size must be at least {}"
.format(rconst.NUM_EVAL_NEGATIVES + 1))
def eval_size_check(eval_batch_size):
return (eval_batch_size is None or
int(eval_batch_size) > rconst.NUM_EVAL_NEGATIVES)
flags.DEFINE_bool(
name="use_xla_for_gpu", default=False, help=flags_core.help_wrap(
"If True, use XLA for the model function. Only works when using a "
"GPU. On TPUs, XLA is always used"))
xla_message = "--use_xla_for_gpu is incompatible with --tpu"
@flags.multi_flags_validator(
["use_xla_for_gpu", "tpu"], message=xla_message)
def xla_validator(flag_dict):
return not flag_dict["use_xla_for_gpu"] or not flag_dict["tpu"]
flags.DEFINE_bool(
name="early_stopping",
default=False,
help=flags_core.help_wrap(
"If True, we stop the training when it reaches hr_threshold"))
flags.DEFINE_bool(
name="keras_use_ctl",
default=False,
help=flags_core.help_wrap(
"If True, we use a custom training loop for keras."))
def define_resnet_flags(resnet_size_choices=None):
"""Add flags and validators for ResNet."""
flags_core.define_base()
flags_core.define_performance(num_parallel_calls=False,
tf_gpu_thread_mode=True,
datasets_num_private_threads=True,
datasets_num_parallel_batches=True)
flags_core.define_image()
flags_core.define_benchmark()
flags.adopt_module_key_flags(flags_core)
flags.DEFINE_enum(
name='resnet_version',
short_name='rv',
default='2',
enum_values=['1', '2'],
help=flags_core.help_wrap(
'Version of ResNet. (1 or 2) See README.md for details.'))
flags.DEFINE_bool(
name='fine_tune',
short_name='ft',
default=False,
help=flags_core.help_wrap(
'If True do not train any parameters except for the final layer.'))
flags.DEFINE_string(
name='pretrained_model_checkpoint_path',
short_name='pmcp',
default=None,
help=flags_core.help_wrap(
'If not None initialize all the network except the final layer with '
'these values'))
flags.DEFINE_boolean(name='eval_only',
default=False,
help=flags_core.help_wrap(
'Skip training and only perform evaluation on '
'the latest checkpoint.'))
flags.DEFINE_boolean(
name='image_bytes_as_serving_input',
default=False,
help=flags_core.help_wrap(
'If True exports savedmodel with serving signature that accepts '
'JPEG image bytes instead of a fixed size [HxWxC] tensor that '
'represents the image. The former is easier to use for serving at '
'the expense of image resize/cropping being done as part of model '
'inference. Note, this flag only applies to ImageNet and cannot '
'be used for CIFAR.'))
flags.DEFINE_float(name='reconst_loss_scale',
default=10.0,
help=flags_core.help_wrap('scale the reconst_loss'))
flags.DEFINE_boolean(
name='use_ce',
default=False,
help=flags_core.help_wrap(
'use cross entropy loss for compressive sensing training'))
flags.DEFINE_string(
name='optimizer',
short_name='opt',
# default='sgd',
default='adam',
help=flags_core.help_wrap('Choose optimizer for training'))
flags.DEFINE_boolean(
name='clip_grad',
default=False,
help=flags_core.help_wrap('whether to clip weights during training'))
flags.DEFINE_boolean(name='spectral_norm',
short_name='sn',
default=True,
help=flags_core.help_wrap(
'whether to user spectral norm in the cs part'))
flags.DEFINE_float(name='ce_scale',
default=1.0,
help=flags_core.help_wrap('scale the cross_entropy'))
flags.DEFINE_boolean(
name='sep_grad_nrom',
default=False,
help=flags_core.help_wrap(
'spearate the gradients from reconstruction and ce, and norm the ce grad'
))
flags.DEFINE_boolean(
name='norm_teach_feature',
default=False,
help=flags_core.help_wrap(
'norm each channel of teaching feature with BN params'))
flags.DEFINE_boolean(name='no_dense_init',
default=False,
help=flags_core.help_wrap(
'dont init resenet/dense during fine tuning'))
flags.DEFINE_float(name='compress_ratio',
default=0.1,
help=flags_core.help_wrap(
'the compress ratio of the offloading layer'))
choice_kwargs = dict(
name='resnet_size',
short_name='rs',
default='50',
help=flags_core.help_wrap('The size of the ResNet model to use.'))
if resnet_size_choices is None:
flags.DEFINE_string(**choice_kwargs)
else:
flags.DEFINE_enum(enum_values=resnet_size_choices, **choice_kwargs)
def define_data_download_flags():
flags.DEFINE_string(name='data_dir',
default='/tmp/higgs_data',
help=flags_core.help_wrap(
'Directory to download higgs dataset and store'))
def universal_flags():
flags_core.define_base()
flags.DEFINE_integer(
name='num_classes',
default=3640,
help=flags_core.help_wrap('Number of species to classify.'))
flags.DEFINE_integer(
name='vocab_size',
default=8390657,
help=flags_core.help_wrap('Number of k-mers in the vocabulary.'))
flags.DEFINE_integer(
name='embedding_dim',
default=100,
help=flags_core.help_wrap('Dimension of embedding layer.'))
flags.DEFINE_integer(name='mlp_dim',
default=3000,
help=flags_core.help_wrap('Dimension of MLP.'))
flags.DEFINE_integer(name='max_len',
default=100,
help=flags_core.help_wrap('Max length of sequences.'))
flags.DEFINE_integer(name='kmer',
default=12,
help=flags_core.help_wrap('Length of k-mers.'))
flags.DEFINE_integer(
name='cpus',
default=8,
help=flags_core.help_wrap('Number of cpus for input pipeline.'))
flags.DEFINE_float(name='lr',
default=0.001,
help=flags_core.help_wrap('Learning rate.'))
flags.DEFINE_float(name='lr_decay',
default=0.01,
help=flags_core.help_wrap('Learning rate decay.'))
flags.DEFINE_float(
name='keep_prob',
default=1.0,
help=flags_core.help_wrap(
'Probability that an element is kept for dropout layers.'))
flags.DEFINE_string(name='input_tfrec',
default='/tmp/input.tfrec',
help=flags_core.help_wrap('Filename of training set.'))
flags.DEFINE_string(name='model_name',
default='embed_pool',
help=flags_core.help_wrap('Choose a model.'))
flags.DEFINE_string(name='encode_method',
default='kmer',
help=flags_core.help_wrap('One of kmer/one_hot.'))
def define_keras_benchmark_flags():
"""Add flags for keras built-in application models."""
flags_core.define_base(hooks=False)
flags_core.define_performance()
flags_core.define_image()
flags_core.define_benchmark()
flags.adopt_module_key_flags(flags_core)
flags_core.set_defaults(data_format="channels_last",
use_synthetic_data=True,
batch_size=32,
train_epochs=2)
flags.DEFINE_enum(name="model",
default=None,
enum_values=MODELS.keys(),
case_sensitive=False,
help=flags_core.help_wrap("Model to be benchmarked."))
flags.DEFINE_integer(
name="num_train_images",
default=1000,
help=flags_core.help_wrap(
"The number of synthetic images for training. The default value is "
"1000."))
flags.DEFINE_integer(
name="num_eval_images",
default=50,
help=flags_core.help_wrap(
"The number of synthetic images for evaluation. The default value is "
"50."))
flags.DEFINE_boolean(
name="eager",
default=False,
help=flags_core.help_wrap(
"To enable eager execution. Note that if eager execution is enabled, "
"only one GPU is utilized even if multiple GPUs are provided and "
"multi_gpu_model is used."))
flags.DEFINE_boolean(
name="dist_strat",
default=False,
help=flags_core.help_wrap(
"To enable distribution strategy for model training and evaluation. "
"Number of GPUs used for distribution strategy can be set by the "
"argument --num_gpus."))
flags.DEFINE_list(
name="callbacks",
default=["ExamplesPerSecondCallback", "LoggingMetricCallback"],
help=flags_core.help_wrap(
"A list of (case insensitive) strings to specify the names of "
"callbacks. For example: `--callbacks ExamplesPerSecondCallback,"
"LoggingMetricCallback`"))
@flags.multi_flags_validator(
["eager", "dist_strat"],
message="Both --eager and --dist_strat were set. Only one can be "
"defined, as DistributionStrategy is not supported in Eager "
"execution currently.")
# pylint: disable=unused-variable
def _check_eager_dist_strat(flag_dict):
return not (flag_dict["eager"] and flag_dict["dist_strat"])
def define_data_download_flags():
"""Add flags specifying data download arguments."""
flags.DEFINE_string(
name="data_dir",
default="/tmp/census_data/",
help=flags_core.help_wrap("Directory to download and extract data."))
