def Params(cls, metadata):
"""Params builder for APMetrics."""
p = hyperparams.InstantiableParams(cls)
p.Define(
'metadata', metadata,
'Instance of class obeying EvaluationMetadata interface consisting of '
'parameters specifying the details of the evaluation.')
p.Define(
'breakdown_metrics', [],
'List of extra breakdown metrics when computing AP. Valid values '
'include: ["num_points", "distance", "rotation"]. See '
'breakdown_metric.py:ByName for the full list. ByDifficulty is '
'always used.')
p.Define(
'metric_weights', None,
'For metrics that have multiple breakdown metrics, '
'a user may want the value() function to be a function '
'of the various breakdowns. If provided, this specifies '
'the weights assigned to each class\'s metric contribution. '
'metric_weights should be a dictionary mapping every difficulty '
'level to a numpy vector of weights whose values corresponds '
'to the order of metadata.EvalClassIndices() weighting.')
p.Define(
'box_type', '3d', 'Specifies what kind of box evaluation will '
'be performed (only supported by waymo metric). '
'One of ["2d", "3d"]: 3d means to do 3D AP calculation, and 2d '
'means to do a top down Birds-Eye-View AP calculation.')
return p
def Params(cls):
"""Params for a MLPerfProgramSchedule."""
p = hyperparams.InstantiableParams(cls)
p.Define('task_dict', None, 'dataset_name -> task params')
p.Define('task_name', None, 'High level task name')
p.Define('logdir', None, 'Log directory')
p.Define('train_program', None, 'Train program params')
p.Define('train_executions_per_eval', 1, '')
p.Define('dataset_names', [], 'List of all dataset names.')
p.Define('num_splits_per_client', None, '')
p.Define('ml_perf', hyperparams.Params(), 'MlPerf configuration.')
mlp = p.ml_perf
mlp.Define('benchmark_name', None,
'Benchmark name for compliance log.')
mlp.Define('decoder_metric_name', None,
'Name of the decoder metric to report for compliance log.')
mlp.Define('decoder_metric_success_threshold', None,
'Benchmark run must exceed this value to succeeed.')
mlp.Define('steps_per_epoch', None,
'Number of training steps per epoch.')
mlp.Define('global_batch_size', None, 'Global batch size.')
mlp.Define('max_sequence_length', None, 'Maximum sequence length.')
mlp.Define('optimizer_name', None, 'Optimizer used.')
mlp.Define('base_learning_rate', None, 'Base learning rate.')
mlp.Define('warmup_steps', None, 'Number of warm-up steps.')
return p
def Params(cls):
"""The params of this layer."""
p = hyperparams.InstantiableParams(cls)
p.Define('deterministic_dropout', False,
'Used deterministic dropout or not.')
p.Define(
'fprop_dtype', None,
'Activations datatype to use. To enable bfloat16 activations for '
'layers built using model builder, set fprop_dtype to '
'tf.bfloat16, which will be propagated to layers that support '
'bfloat16 activations. Default is None, which will use float32 '
'activations.')
# SPMD partition related params.
p.Define(
'device_mesh', None,
'A numpy.ndarray specifying the topology of a device mesh to place the '
'computations onto. If device_mesh is None, it is assumed to be a '
'single device. Here are some examples: '
'np.array([0, 1, 2, 3, 4, 5, 6, 7]) which is a 1d mesh with 8 devices, '
'np.array([[0, 1, 2, 3], [4, 5, 6, 7]]) which is 2d matrix of 8 '
'devices.')
p.Define(
'weight_split_dims_mapping', None,
'Relevant only if device_mesh above is not None. If not None, it '
'specifies how weight of this layer or those of the sublayers should '
'be sharded over device mesh. ')
p.Define(
'activation_split_dims_mapping', None,
'Relevant only if device_mesh above is not None. If not None, it '
'specifies how activation of this layer or those of the sublayers '
'should be sharded over device mesh. ')
return p
def Params(cls: Type[BaseLayerT]) -> BaseLayerParamsT:
"""Returns the layer params."""
p = hyperparams.InstantiableParams(cls)
p.Define('inference_driver_name', cls._INFERENCE_DRIVER_NAME,
'Name of the inference driver used to construct this layer.')
p.Define('name', '', 'Name of this layer object.')
p.Define('dtype', tf.float32, 'Datatype to use.')
# None value will make FProp use dtype instead of fprop_dtype.
# TODO(lepikhin): all @tf.Defun should use p.fprop_dtype if it is set.
p.Define('fprop_dtype', None, 'Activations datatype to use.')
p.Define(
'random_seed', None,
'Random seed for deterministic unittests. This '
'is inherited by child layers if they do not set a random_seed.')
p.Define('vn', py_utils.DefaultVN(),
'How variational noise should be applied.')
p.Define(
'params_init', py_utils.DefaultParamInit(),
'How model weights should be initialized. Not to be confused with '
'hyperparams.')
p.Define('add_name_to_theta', False,
'Wrap theta with tf.identity(var_name).')
# Makes additional alterations for graphs being used for inference.
p.Define('is_inference', None, 'True if in inference mode.')
# In addition to is_inference, indicate that the inference graph is
# for a single step.
p.Define(
'allow_implicit_capture', None,
'When using Defuns, code often asserts that the Defun does not '
'capture undeclared inputs. This eliminates a source of bugs '
'at the expense of making some kinds of models or utilities '
'hard/impossible to use. Setting this to True/False (versus None) '
'causes the setting to apply to this layer and its children.')
p.Define(
'skip_lp_regularization', None,
'If True, all variables in this layer will skip Lp regularization. '
'If None/False, only variables explicitly in the '
'SKIP_LP_REGULARIZATION collection will skip Lp regularization. '
'Also propagated to child layers with default settings (None).')
# SPMD partition related params.
p.Define(
'device_mesh', None,
'A numpy.ndarray specifying the topology of a device mesh to place the'
' computations onto. If device_mesh is None, it is assumed to be a'
' single device. Here are some examples:'
' np.array([0, 1, 2, 3, 4, 5, 6, 7]) which is a 1d mesh with 8 devices,'
' np.array([[0, 1, 2, 3], [4, 5, 6, 7]]) which is 2d matrix of 8'
' devices.')
p.Define(
'weight_split_dims_mapping', None,
'Relevant only if device_mesh above is not None. If not None, it '
'specifies how weight of this layer or those of the sublayers should '
'be sharded over device mesh. ')
p.Define(
'activation_split_dims_mapping', None,
'Relevant only if device_mesh above is not None. If not None, it '
'specifies how activation of this layer or those of the sublayers '
'should be sharded over device mesh. ')
return p
def Params(cls):
"""Defaults parameters for a cluster."""
p = hyperparams.InstantiableParams(cls)
p.Define(
'mode', 'async', 'A string noting the overall training method. '
'Valid values: sync, async.')
p.Define(
'job', 'trainer', 'The role of this job in the training cluster. '
'E.g., trainer_client, trainer, controller, etc.')
p.Define('task', 0, 'This process is the task-th task in the job.')
p.Define('logdir', '', 'The log directory.')
# How the cluster is composed.
#
# A typical training cluster has a few jobs (controller, worker, ps, etc).
# One can potentially place computation on any device of these jobs.
# Here, we specify how each job is configured. E.g., number of GPUs each
# task is equipped with, the number of replicas, etc.
#
# Note that trainer client may dispatch operations on just a
# smaller subset of jobs. For example, the controller only places
# computations onto the controller and ps devices; while evaler
# only places computations on the evaler devices.
#
# cluster.job refers to the role of a client process performs. It
# can be 'controller', 'trainer', 'trainer_client', 'evaler' and
# 'decoder', etc. Often, a client can be the same process as one
# of the compute devices (e.g., controller). Sometimes, they can
# be a separate processes. E.g., trainer_client is a separate
# standalone process. It places computations on the worker and
# ps devices, while itself does not host any.
p.Define('controller', cls._JobSpec(1), 'The controller job.')
p.Define('worker', cls._JobSpec(1), 'The worker job.')
p.Define('ps', cls._JobSpec(1), 'The ps job.')
p.Define('input', cls._JobSpec(0), 'The input job.')
p.Define('evaler', cls._JobSpec(0), 'The evaler job.')
p.Define('decoder', cls._JobSpec(0), 'The decoder job.')
# A few 'global' knobs.
p.Define('tf_data_service_address', '',
'The address of the tf.data service.')
p.Define(
'add_summary', None, 'Whether to add summaries. If None, '
'decides based on the job type.')
p.Define('do_eval', None, 'Whether to do eval.')
p.Define('split_id', 0, 'Split id for the model.')
p.Define('immediately_instantiate_variables', True,
'Whether to create variables immediately.')
p.Define(
'xla_device', None, 'If set to non-None, '
'this value is used instead of FLAGS.xla_device '
'for running multiple runners in the same process, '
'eg: Controller and TrainerTpu')
p.Define(
'enable_asserts', None, 'If set to non-None, '
'this value is used instead of FLAGS.enable_asserts. '
'If False, we disable all asserts ops and return tf.no_op() instead.'
)
return p
def testIsLayerParams(self):
self.assertTrue(base_layer.IsLayerParams(base_layer.BaseLayer.Params()))
self.assertTrue(base_layer.IsLayerParams(TestLayer.Params()))
self.assertFalse(base_layer.IsLayerParams(None))
self.assertFalse(base_layer.IsLayerParams(hyperparams.Params()))
self.assertFalse(
base_layer.IsLayerParams(
hyperparams.InstantiableParams(base_layer.Accumulator)))
def testInstantiate(self):
a = hyperparams.InstantiableParams(InstantiableClass)
a.Define('new_param', None, 'A meaningless param.')
a.new_param = 'hi'
obj = a.Instantiate()
self.assertIsInstance(obj, InstantiableClass)
self.assertEqual(obj.params.new_param, 'hi')
def Params(cls):
p = hyperparams.InstantiableParams(cls)
p.Define('task_dict', None, 'dataset_name -> task params')
p.Define('logdir', None, 'Log directory')
p.Define('train_program', None, '')
p.Define('train_executions_per_eval', 1, '')
p.Define('eval_programs', [], '')
p.Define('num_splits_per_client', None, '')
return p
def testInstantiateWithParams(self):
a = hyperparams.InstantiableParams(InstantiableClass)
a.Define('new_param', None, 'A meaningless param.')
a.new_param = 'hi'
# Same as the previous test, but InstantiableClass should also get
# other=15 passed as a keyword argument to the constructor.
obj = a.Instantiate(other=15)
self.assertIsInstance(obj, InstantiableClass)
self.assertEqual(obj.params.new_param, 'hi')
self.assertEqual(obj.other, 15)
def Params(cls):
""""Defaults parameters for Programs."""
p = hyperparams.InstantiableParams(cls)
p.Define('task', None, 'Underlying task')
p.Define('logdir', None, 'Log directory')
p.Define('num_splits_per_client', None, '')
p.Define('steps_per_loop', None, 'Number of steps to run.')
p.Define('dataset_name', None,
'Dataset the program is operating on, eg: "Test"')
p.Define('name', 'base_program', 'Program name.')
return p
def Params(cls):
"""Params for a SimpleProgramSchedule."""
p = hyperparams.InstantiableParams(cls)
p.Define('task_dict', None, 'dataset_name -> task params')
p.Define('task_name', None, 'High level task name')
p.Define('logdir', None, 'Log directory')
p.Define('train_program', None, 'Train program params')
p.Define('train_executions_per_eval', 1, '')
p.Define('eval_programs', [], 'List of eval program params.')
p.Define('num_splits_per_client', None, '')
p.Define('dataset_names', [], 'List of all dataset names.')
return p
def Params(cls):
p = hyperparams.InstantiableParams(cls)
p.Define(
'split_paths', dict(train='train-*', dev='dev-*', test='test-*'),
'Paths of each split of the dataset (split name => filepattern).')
p.Define(
'data_dir', '',
'Optional; base directory of any relative paths in split_paths.')
p.Define(
'bert_max_length', 48,
'Max token length of the precomputed BERT caption embeddings.')
return p
def Params(cls):
"""The params of this layer."""
p = hyperparams.InstantiableParams(cls)
p.Define('deterministic_dropout', False,
'Used deterministic dropout or not.')
p.Define(
'fprop_dtype', None,
'Activations datatype to use. To enable bfloat16 activations for '
'layers built using model builder, set fprop_dtype to '
'tf.bfloat16, which will be propagated to layers that support '
'bfloat16 activations. Default is None, which will use float32 '
'activations.')
return p
def Params(cls):
""""Defaults parameters for Programs."""
p = hyperparams.InstantiableParams(cls)
p.Define('task', None, 'Underlying task')
p.Define('logdir', None, 'Log directory')
p.Define('num_splits_per_client', None, '')
p.Define('steps_per_loop', None, 'Number of steps to run.')
p.Define('dataset_name', None,
'Dataset the program is operating on, eg: "Test"')
p.Define('name', 'base_program', 'Program name.')
p.Define('task_name', None,
'If multi-task, what the high-level task name is')
p.Define('num_threads', 1,
'Number of threads in multiprocessing pool.')
return p
def Params(cls):
p = hyperparams.InstantiableParams(cls)
p.Define('name', 'EarlyStop', '')
p.Define('metric_history', MetricHistory.Params(), 'Metric history params.')
p.Define(
'tolerance', 0.0, 'Minimum significant difference in metric; '
'useful if progress is asymptotic.')
p.Define('window', 0, 'Maximum number of steps between best and current.')
p.Define('verbose', True, 'Log early-stop checks.')
p.Define('min_steps', 0, 'Minimum number of steps before stopping.')
p.Define(
'logging_interval', 100,
'The number of global steps after which the early stop metric '
'history is updated.')
return p
def Params(cls):
"""Params for a SimpleProgramSchedule."""
p = hyperparams.InstantiableParams(cls)
p.Define('task_dict', None, 'dataset_name -> task params')
p.Define('task_name', None, 'High level task name')
p.Define('logdir', None, 'Log directory')
p.Define('train_program', None, 'Train program params')
p.Define('train_executions_per_eval', 1, '')
p.Define('eval_programs', [], 'List of eval program params.')
p.Define('num_splits_per_client', None, '')
p.Define('dataset_names', [], 'List of all dataset names.')
# TODO(blee): Clean these up.
p.Define('ml_perf', hyperparams.Params(), 'MlPerf configuration.')
mlp = p.ml_perf
mlp.Define('benchmark_name', None, 'Benchmark name for compliance log.')
return p
def Params(cls):
"""Defaults parameters for a cluster."""
p = hyperparams.InstantiableParams(cls)
p.Define(
'mode', 'async', 'A string noting the overall training method. '
'Valid values: sync, async.')
p.Define(
'job', 'trainer', 'The role of this job in the training cluster. '
'E.g., trainer_client, trainer, controller, etc.')
p.Define('task', 0, 'This process is the task-th task in the job.')
p.Define('logdir', '', 'The log directory.')
# How the cluster is composed.
#
# A typical training cluster has a few jobs (controller, worker, ps, etc).
# One can potentially place computation on any device of these jobs.
# Here, we specify how each job is configured. E.g., number of GPUs each
# task is equipped with, the number of replicas, etc.
#
# Note that trainer client may dispatch operations on just a
# smaller subset of jobs. For example, the controller only places
# computations onto the controller and ps devices; while evaler
# only places computations on the evaler devices.
#
# cluster.job refers to the role of a client process performs. It
# can be 'controller', 'trainer', 'trainer_client', 'evaler' and
# 'decoder', etc. Often, a client can be the same process as one
# of the compute devices (e.g., controller). Sometimes, they can
# be a separate processes. E.g., trainer_client is a separate
# standalone process. It places computations on the worker and
# ps devices, while itself does not host any.
p.Define('controller', cls._JobSpec(1), 'The controller job.')
p.Define('worker', cls._JobSpec(1), 'The worker job.')
p.Define('ps', cls._JobSpec(1), 'The ps job.')
p.Define('input', cls._JobSpec(0), 'The input job.')
p.Define('evaler', cls._JobSpec(0), 'The evaler job.')
p.Define('decoder', cls._JobSpec(0), 'The decoder job.')
# A few 'global' knobs.
p.Define(
'add_summary', None, 'Whether to add summaries. If None, '
'decides based on the job type.')
return p
def Params(cls):
"""Returns the layer params."""
p = hyperparams.InstantiableParams(cls)
p.Define('inference_driver_name', cls._INFERENCE_DRIVER_NAME,
'Name of the inference driver used to construct this layer.')
p.Define('name', '', 'Name of this layer object.')
p.Define('dtype', tf.float32, 'Datatype to use.')
# None value will make FProp use dtype instead of fprop_dtype.
# TODO(lepikhin): all @function.Defun should use p.fprop_dtype if it is set.
p.Define('fprop_dtype', None, 'Activations datatype to use.')
p.Define(
'random_seed', None,
'Random seed for deterministic unittests. This '
'is inherited by child layers if they do not set a random_seed.')
p.Define('vn', DefaultVN(), 'How variational noise should be applied.')
p.Define(
'params_init', py_utils.DefaultParamInit(),
'How model weights should be initialized. Not to be confused with '
'hyperparams.')
# is_eval is used to generate graph for eval purpose, typically
# the eval graph is forward pass of training graph without
# regularization, e.g. dropout.
p.Define('is_eval', None, 'True if in eval mode.')
# In addition to is_eval, also makes additional alterations for graphs
# being used for inference.
p.Define('is_inference', None, 'True if in inference mode.')
# In addition to is_eval/is_inference, indicate that the inference graph is
# for a single step.
p.Define(
'allow_implicit_capture', None,
'When using Defuns, code often asserts that the Defun does not '
'capture undeclared inputs. This eliminates a source of bugs '
'at the expense of making some kinds of models or utilities '
'hard/impossible to use. Setting this to True/False (versus None) '
'causes the setting to apply to this layer and its children.')
p.Define(
'skip_lp_regularization', None,
'If True, all variables in this layer will skip Lp regularization. '
'If None/False, only variables explicitly in the '
'SKIP_LP_REGULARIZATION collection will skip Lp regularization. '
'Also propagated to child layers with default settings (None).')
return p
def testCopyFieldsToDoesNotCopyClass(self):
source = _params.InstantiableParams(cls=_params.Params)
dest = _params.InstantiableParams(cls=_params.InstantiableParams)
_params.CopyFieldsTo(source, dest)
self.assertEqual(dest.cls, _params.InstantiableParams)
def Params(cls):
p = hyperparams.InstantiableParams(cls)
p.Define('program_schedule_dict', None,
'task_name -> ProgramScheduleParams')
return p
def testCopyFieldsToDoesNotCopyClass(self): source = hyperparams.InstantiableParams(hyperparams.Params) dest = hyperparams.InstantiableParams(hyperparams.InstantiableParams) hyperparams.CopyFieldsTo(source, dest) self.assertEqual(dest.cls, hyperparams.InstantiableParams)
def Params(cls):
p = hyperparams.InstantiableParams(cls)
return p
def testMergeCommonKeysFromDoesNotCopyClass(self):
source = hyperparams.InstantiableParams(hyperparams.Params)
dest = hyperparams.InstantiableParams(hyperparams.InstantiableParams)
dest.MergeCommonKeysFrom(source)
self.assertEqual(dest.cls, hyperparams.InstantiableParams)