FLAGS = flags.FLAGS
flags.DEFINE_string(
"report", None,
"The report whose datasets will be used to compute the predictions.")
flags.DEFINE_integer("batch_size", 32, "The batch size to use.")
flags.DEFINE_string("model_module", None,
"A python module file defining the model.")
flags.DEFINE_string(
"model_args", "",
"The arguments to be passed to the create() function of the model. Will "
"be literal_eval'ed, should take the form a='1',b='2',c=3.")
flags.DEFINE_string("output_dir", None,
"Where to store the dense output. Can be anything that "
"can be open with tf.io, including Google cloud buckets.")
flags.DEFINE_bool("tf_on_cpu", True,
"If set, will hide accelerators from TF.")
flags.DEFINE_integer(
"num_workers", 1,
"How many instances of the script are launched to parallelize compute "
"across datasets.")
flags.DEFINE_integer("worker_index", 0, "The zero-indexed id of this worker.")
flags.mark_flags_as_required(["model_module"])
def slice_per_worker(dictionary):
"""Equally partitions input over workers. Remainder is given to last worker.
Args:
dictionary: dict to split over.
flags.DEFINE_boolean(
'mask_zeros_in_client_updates', False,
'Indicates whether to average client deltas with zero masking.')
with utils_impl.record_hparam_flags() as task_flags:
# Task specification
flags.DEFINE_enum('task', None, _SUPPORTED_TASKS,
'Which task to perform federated training on.')
with utils_impl.record_hparam_flags() as cifar100_flags:
# CIFAR-100 flags
flags.DEFINE_integer(
'cifar100_crop_size', 24, 'The height and width of '
'images after preprocessing.')
flags.DEFINE_bool(
'cifar100_distort_train_images', True, 'If set to True, '
'train images will be randomly cropped. Otherwise, all '
'images will simply be resized.')
with utils_impl.record_hparam_flags() as emnist_cr_flags:
# EMNIST CR flags
flags.DEFINE_enum(
'emnist_cr_model', 'cnn', ['cnn', '2nn'], 'Which model to '
'use. This can be a convolutional model (cnn) or a two '
'hidden-layer densely connected network (2nn).')
with utils_impl.record_hparam_flags() as shakespeare_flags:
# Shakespeare flags
flags.DEFINE_integer(
'shakespeare_sequence_length', 80,
'Length of character sequences to use for the RNN model.')
import absl.logging as _logging # pylint: disable=unused-import
import tensorflow as tf
import imagenet_input
import resnet_model
from tensorflow.contrib import summary
from tensorflow.contrib.tpu.python.tpu import tpu_config
from tensorflow.contrib.tpu.python.tpu import tpu_estimator
from tensorflow.contrib.tpu.python.tpu import tpu_optimizer
from tensorflow.python.estimator import estimator
FLAGS = flags.FLAGS
flags.DEFINE_bool(
'use_tpu', True,
help=('Use TPU to execute the model for training and evaluation. If'
' --use_tpu=false, will use whatever devices are available to'
' TensorFlow by default (e.g. CPU and GPU)'))
# Cloud TPU Cluster Resolvers
flags.DEFINE_string(
'gcp_project', default=None,
help='Project name for the Cloud TPU-enabled project. If not specified, we '
'will attempt to automatically detect the GCE project from metadata.')
flags.DEFINE_string(
'tpu_zone', default=None,
help='GCE zone where the Cloud TPU is located in. If not specified, we '
'will attempt to automatically detect the GCE project from metadata.')
flags.DEFINE_string(
import six
from tensor2tensor.data_generators import generator_utils
from tensor2tensor.data_generators import image_utils
from tensor2tensor.data_generators import problem
from tensor2tensor.data_generators import text_encoder
from tensor2tensor.layers import common_layers
from tensor2tensor.layers import common_video
from tensor2tensor.layers import modalities
from tensor2tensor.utils import metrics
from tensor2tensor.utils import video_metrics
import tensorflow as tf
import tf_slim as slim
FLAGS = flags.FLAGS
flags.DEFINE_bool("disable_ffmpeg", False,
"Disable FFMPEG when generating debug videos.")
def resize_video_frames(images, size):
return [
tf.to_int64(
tf.image.resize_images(image, [size, size],
tf.image.ResizeMethod.BILINEAR))
for image in images
]
def video_augmentation(features, hue=False, saturate=False, contrast=False):
"""Augments video with optional hue, saturation and constrast.
Args:
"exec_code", None,
"If present, no IPython shell is started but the code given in "
"the flag is run instead (comparable to the -c option of "
"IPython). The code will be able to use a predefined "
"global 'grrapi' object.")
flags.DEFINE_string(
"exec_file", None,
"If present, no IPython shell is started but the code given in "
"command file is supplied as input instead. The code "
"will be able to use a predefined global 'grrapi' "
"object.")
flags.DEFINE_bool(
"version",
default=False,
allow_override=True,
help="Print the API shell version number and exit immediately.")
def main(argv=None):
del argv # Unused.
if flags.FLAGS.version:
print("GRR API shell {}".format(config_server.VERSION["packageversion"]))
return
config.CONFIG.AddContext(contexts.COMMAND_LINE_CONTEXT)
config.CONFIG.AddContext(contexts.CONSOLE_CONTEXT,
"Context applied when running the console binary.")
server_startup.Init()
flags.DEFINE_integer("decoder_cell_size", 64, "Size of decoder cell.")
flags.DEFINE_enum("optimizer_type", "Adam", ["Adam"],
"Type of optimizer to use.")
flags.DEFINE_float("learning_rate", 0.001, "Learning rate.")
flags.DEFINE_float("max_gradient_norm", 10.0, "Gradient clipping.")
flags.DEFINE_integer("max_decode_length", 200, "Max length to decode.")
flags.DEFINE_float("scheduled_sampling_prob", 0.0,
"Scheduled sampling probabiliy used during trainig.")
flags.DEFINE_bool("use_attention", False, "True to use attention.")
flags.DEFINE_float("input_keep_prob", 1.0, "Dropout input keep probability.")
flags.DEFINE_float("output_keep_prob", 1.0, "Dropout output keep probability.")
flags.DEFINE_float("state_keep_prob", 1.0, "Dropout state keep probability.")
flags.DEFINE_bool("use_mixer", False, "True to use MIXER loss.")
flags.DEFINE_integer("rl_start_step", None, "Step at which to apply MIXER.")
flags.DEFINE_integer("rl_anneal_steps", None, "Number of steps to anneal.")
# Input configurations.
flags.DEFINE_string("src_file", None, "Source input file.")
import torch.nn as nn
import torch.optim as optim
from tensorboardX import SummaryWriter
from absl import app, flags, logging
from absl.flags import FLAGS
from models.PSMnet import PSMNet
from models.smoothloss import SmoothL1Loss
from dataloader.AsterLoader import AsterLoader
from misc.dependences.File_util import File_util
flags.DEFINE_integer('epoch', 300, 'epoch number')
flags.DEFINE_integer('batch_size', 4, 'batch size')
flags.DEFINE_float('lr', 0.001, 'learning rate')
flags.DEFINE_bool('is_cuda', False, 'whether cuda is used or not')
flags.DEFINE_bool('pre_trained', False, 'whether model is pretrained or not')
flags.DEFINE_string('optimizer', 'Adam', 'select optimizer')
flags.DEFINE_string('criterion', 'SmoothL1Loss', 'select criterion')
flags.DEFINE_float('validation_rate', 0.1,
'validation number rate when spliting dataset')
flags.DEFINE_string('tensor_board_log_dir', './tensorboard_log',
'tensorboardX logging folder')
flags.DEFINE_string('config_path', './configs/configs.yml', 'config file path')
flags.DEFINE_string('csv_path', './dataset/edit/result.csv',
'csv path for dataloader')
flags.DEFINE_string('save_path', './model.tar', 'model save path')
flags.DEFINE_integer('maxdisp', 192, 'max disparity')
def main(_argv):
import efficientnet_builder
import imagenet_input
import utils
from condconv import efficientnet_condconv_builder
from edgetpu import efficientnet_edgetpu_builder
from tpu import efficientnet_tpu_builder
from tensorflow.core.protobuf import rewriter_config_pb2
from tensorflow.python.estimator import estimator
FLAGS = flags.FLAGS
FAKE_DATA_DIR = 'gs://cloud-tpu-test-datasets/fake_imagenet'
flags.DEFINE_bool(
'use_tpu',
default=True,
help=('Use TPU to execute the model for training and evaluation. If'
' --use_tpu=false, will use whatever devices are available to'
' TensorFlow by default (e.g. CPU and GPU)'))
# Cloud TPU Cluster Resolvers
flags.DEFINE_string(
'tpu',
default=None,
help='The Cloud TPU to use for training. This should be either the name '
'used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 url.'
)
flags.DEFINE_string(
'gcp_project',
default=None,
help='Project name for the Cloud TPU-enabled project. If not specified, we '
def define_flags() -> List[str]:
"""Define common flags."""
predefined_flags = set(FLAGS)
flags.DEFINE_string('experiment_name', None, 'Name of this experiment.')
# Flags relating to setting up the job.
flags.DEFINE_string('tpu', None, 'Name of the TPU to use.')
flags.DEFINE_enum(
'mode',
'train_and_eval',
['train', 'eval', 'train_and_eval'],
'Whether to execute train and/or eval.')
flags.DEFINE_bool('use_tpu', False, 'Whether to run on CPU or TPU.')
# Flags relating to the training/eval loop.
flags.DEFINE_integer(
'checkpoint_step', -1, 'Step of the checkpoint to restore from.')
flags.DEFINE_enum(
'dataset_name',
None,
datasets.get_dataset_names(),
'Name of the dataset to use.')
flags.DEFINE_integer(
'eval_frequency',
None,
'How many steps between evaluating on the (validation and) test set.')
flags.DEFINE_string('output_dir', None, 'Base output directory.')
flags.DEFINE_enum(
'model_name',
None,
models.get_model_names(),
'Name of the model to use.')
flags.DEFINE_integer(
'log_frequency',
100,
'How many steps between logging the metrics.')
flags.DEFINE_integer('train_steps', None, 'How many steps to train for.')
# Hyperparamter flags.
flags.DEFINE_integer('batch_size', None, 'Training batch size.')
flags.DEFINE_integer('eval_batch_size', None, 'Validation/test batch size.')
flags.DEFINE_float('learning_rate', None, 'Learning rate.')
flags.DEFINE_string(
'learning_rate_schedule',
'constant',
'Learning rate schedule to use.')
flags.DEFINE_string('optimizer', 'adam', 'Optimizer to use.')
flags.DEFINE_float('optimizer_hparams_beta_1', 0.9, 'Adam beta_1.')
flags.DEFINE_float('optimizer_hparams_beta_2', 0.999, 'Adam beta_2.')
flags.DEFINE_float('optimizer_hparams_epsilon', 1e-7, 'Adam epsilon.')
flags.DEFINE_float('weight_decay', 0.0, 'Weight decay.')
flags.DEFINE_integer('seed', 42, 'Random seed.')
flags.DEFINE_float(
'validation_percent',
0.0,
'Percent of training data to hold out and use as a validation set.')
# Loss function related flags.
flags.DEFINE_enum('loss_name', None,
enum_values=['crossentropy', 'dm_loss', 'one_vs_all',
'focal_loss'],
help='Loss function')
flags.DEFINE_float('dm_alpha', 1.0, 'DM Alpha parameter.')
flags.DEFINE_float('focal_gamma', 3.0, 'Gamma parameter for focal loss.')
flags.DEFINE_bool('distance_logits', False,
'Whether to use a distance-based last layer.')
flags.mark_flag_as_required('dataset_name')
flags.mark_flag_as_required('experiment_name')
flags.mark_flag_as_required('loss_name')
flags.mark_flag_as_required('model_name')
all_flags = set(FLAGS)
program_flag_names = sorted(list(all_flags - predefined_flags))
return program_flag_names
flags.DEFINE_integer(
'diskspd_stride_or_alignment', 64,
'If the access pattern is sequential, then this value'
'means the stride for the access'
'If the access pattern is random, then this value means'
'the specified number of bytes that random I/O aligns to.'
'Defaults: 64K. Unit: KB, can be set')
flags.DEFINE_enum(
'diskspd_stride_or_alignment_unit', 'K', ['K', 'M', 'G', 'b'],
'The unit of the stride_or_alignment,'
'available option: K|M|G|b'
'Defaults: K.')
flags.DEFINE_bool('diskspd_large_page', False,
'Whether use large page for IO buffers. '
'Defaults: False')
flags.DEFINE_bool('diskspd_latency_stats', False,
'Whether measure the latency statistics'
'Defaults: False')
flags.DEFINE_bool(
'diskspd_disable_affinity', False, 'Whether to diable the group affinity,'
'group affinity is to round robin tasks. '
'across processor group. '
'Defaults: False')
flags.DEFINE_bool('diskspd_write_through', True,
'Whether to enable write through IO. '
'Defaults: True')
## Other parameters
flags.DEFINE_string(
"init_checkpoint", None,
"Initial checkpoint, usually from a pre-trained BERT model. In the case of "
"exporting, one can optionally provide path to a particular checkpoint to "
"be exported here.")
flags.DEFINE_integer(
"max_seq_length",
128, # contain CLS and SEP
"The maximum total input sequence length after WordPiece tokenization. "
"Sequences longer than this will be truncated, and sequences shorter than "
"this will be padded.")
flags.DEFINE_bool("do_train", False, "Whether to run training.")
flags.DEFINE_bool("do_eval", False, "Whether to run eval on the dev set.")
flags.DEFINE_bool("do_export", False, "Whether to export a trained model.")
flags.DEFINE_bool("eval_all_checkpoints", False,
"Run through all checkpoints.")
flags.DEFINE_integer(
"eval_timeout", 600,
"The maximum amount of time (in seconds) for eval worker to wait between "
"checkpoints.")
flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.")
flags.DEFINE_integer("eval_batch_size", 8, "Total batch size for eval.")
flags.DEFINE_integer("predict_batch_size", 8, "Total batch size for predict.")
flags.DEFINE_float("learning_rate", 3e-5,
"The initial learning rate for Adam.")
flags.DEFINE_float("num_train_epochs", 3.0,
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Converts KITTI multiview extension data to the TFRecords format."""
import os
from absl import app
from absl import flags
import tensorflow as tf
FLAGS = flags.FLAGS
flags.DEFINE_string('data_dir', '.', 'Dataset folder.')
flags.DEFINE_integer('height', 384, '')
flags.DEFINE_integer('width', 1280, '')
flags.DEFINE_bool('entire_sequence', False,
'Train on the full sequence, otherwise skip frames 9-12.')
def bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def convert_dataset(data_dir):
"""Convert the data to the TFRecord format."""
for subdir in ['training', 'testing']:
import numpy as np
import six
import tensorflow as tf
import tensorflow_gan as tfgan
from tensorflow_gan.python.contrib_utils import dimension_value
# Private functions to test.
from tensorflow_gan.python.estimator.tpu_gan_estimator import get_eval_estimator_spec
from tensorflow_gan.python.estimator.tpu_gan_estimator import get_predict_estimator_spec
from tensorflow_gan.python.estimator.tpu_gan_estimator import get_train_estimator_spec
from tensorflow_gan.python.estimator.tpu_gan_estimator import LossFns
from tensorflow_gan.python.estimator.tpu_gan_estimator import Optimizers
flags.DEFINE_bool('use_tpu', False, 'Whether to run test on TPU or not.')
TpuRunConfig = tf.compat.v1.estimator.tpu.RunConfig
CrossShardOptimizer = tf.compat.v1.tpu.CrossShardOptimizer
TPUEstimatorSpec = tf.compat.v1.estimator.tpu.TPUEstimatorSpec
class TestOptimizerWrapper(tf.compat.v1.train.Optimizer):
"""An optimizer wrapper that is designed to share a real optimizer.
The idea is that multiple instances of this class can share the real optimizer
and this class will keep track of which steps executed on the real optimizer
were executed by which instance of the wrapper class. This is useful for
testing that the order of generator and discriminator steps is as desired.
This optimizer also has an assertion that two consecutive substeps do not
flags.DEFINE_float('weight_decay', 1e-4, 'L2 regularization weight.')
flags.DEFINE_integer(
'max_attempts', 5,
'Maximum number of times to try training and evaluating an individual'
' before aborting.')
flags.DEFINE_list(
'intermediate_evaluations', ['0.5'],
'Intermediate evaluations relative to --train_epochs. For example, to'
' evaluate the model at 1/4, 1/2, 3/4 of the total epochs, use [0.25, 0.5,'
' 0.75]. An evaluation is always done at the start and end of training.')
flags.DEFINE_integer(
'num_repeats', 3,
'Number of repeats evaluated for each model in the space.')
# TPU flags
flags.DEFINE_bool('use_tpu', False, 'Use TPUs for train and evaluation.')
# 'use_tpu', True, 'Use TPUs for train and evaluation.')
flags.DEFINE_integer('tpu_iterations_per_loop', 100,
'Iterations per loop of TPU execution.')
flags.DEFINE_integer('tpu_num_shards', 2,
'Number of TPU shards, a single TPU chip has 2 shards.')
# Random sampling flags
flags.DEFINE_integer('number_of_steps', 1,
'Number of batches used to evaluate the random sample.')
flags.DEFINE_integer(
'max_samples', 200,
'Number of random samples taken to evaluate the performance.')
def build_config():
flags.DEFINE_float('one_minus_momentum', 0.1, 'Optimizer momentum.')
flags.DEFINE_float('l2', 1e-4, 'L2 coefficient.')
flags.DEFINE_string('data_dir', None, 'Path to training and testing data.')
flags.DEFINE_string(
'output_dir', '/tmp/imagenet', 'The directory where the model weights and '
'training/evaluation summaries are stored.')
flags.DEFINE_integer('train_epochs', 90, 'Number of training epochs.')
flags.DEFINE_integer(
'checkpoint_interval', 25,
'Number of epochs between saving checkpoints. Use -1 to '
'never save checkpoints.')
flags.DEFINE_integer('num_bins', 15, 'Number of bins for ECE computation.')
# Mixup-related flags.
flags.DEFINE_float('mixup_alpha', 0., 'Coefficient of mixup distribution.')
flags.DEFINE_bool('same_mix_weight_per_batch', False,
'Whether to use a single mix weight across the batch.')
flags.DEFINE_bool(
'use_random_shuffling', False,
'Whether to use random shuffling to pair the points of mixup'
'within a batch.')
flags.DEFINE_bool(
'use_truncated_beta', True, 'Whether to sample the mixup weights from '
'Beta[0,1](alpha,alpha) or from the truncated distribution '
'Beta[1/2,1](alpha,alpha).')
flags.DEFINE_float(
'train_proportion', 1.0,
'What proportion of the training set to use to train versus validate on.')
# Accelerator flags.
flags.DEFINE_bool('use_gpu', False, 'Whether to run on GPU or otherwise TPU.')
flags.DEFINE_bool('use_bfloat16', True, 'Whether to use mixed precision.')
from __future__ import division
from __future__ import print_function
import os
from absl import app
from absl import flags
import tensorflow as tf
from utils import raw_data_utils
from absl import app
from absl import flags
FLAGS = flags.FLAGS
flags.DEFINE_bool("separate_doc_by_newline", False, "")
flags.DEFINE_string("output_data_dir", None, "")
flags.DEFINE_string("sub_set", "unsup_in", "")
flags.DEFINE_string("task_name", "IMDB", "")
flags.DEFINE_string("raw_data_dir", "IMDB", "")
def dump_raw_examples(examples, separate_doc_by_newline):
"""dump raw examples."""
tf.logging.info("dumpping raw examples")
text_path = os.path.join(FLAGS.output_data_dir, "text.txt")
label_path = os.path.join(FLAGS.output_data_dir, "label.txt")
# Model specific paramenters
flags.DEFINE_string(
'eval_master',
default='',
help='GRPC URL of the eval master. Set to an appropriate value when running'
' on CPU/GPU')
flags.DEFINE_string('eval_name', default=None, help='Eval job name')
flags.DEFINE_enum('strategy', None, ['tpu', 'gpus', ''],
'Training: gpus for multi-gpu, if None, use TF default.')
flags.DEFINE_integer('num_cores',
default=8,
help='Number of TPU cores for training')
flags.DEFINE_bool('use_fake_data', False, 'Use fake input.')
flags.DEFINE_bool(
'use_xla', False,
'Use XLA even if strategy is not tpu. If strategy is tpu, always use XLA, '
'and this flag has no effect.')
flags.DEFINE_string('model_dir', None, 'Location of model_dir')
flags.DEFINE_string(
'hparams', '', 'Comma separated k=v pairs of hyperparameters or a module'
' containing attributes to use as hyperparameters.')
flags.DEFINE_integer('batch_size', 64, 'training batch size')
flags.DEFINE_integer('eval_samples', 5000, 'The number of samples for '
'evaluation.')
flags.DEFINE_integer('iterations_per_loop', 100,
'Number of iterations per TPU training loop')
flags.DEFINE_string(
from absl import flags
FLAGS = flags.FLAGS
flags.DEFINE_string('framework', 'ros',
'Execution framework to use: ros | ray.')
flags.DEFINE_bool('replay', False,
('True if run in replay mode, otherwise run '
'Carla in server mode using `./CarlaUE4.sh -carla-server`'))
flags.DEFINE_string('log_file_name', None, 'Name of the log file')
flags.DEFINE_string('csv_log_file_name', None,
'csv file into which to log runtime stats')
flags.DEFINE_bool('fail_on_message_loss', True,
'True to enable operator failure when messages are lost')
flags.DEFINE_bool('ground_agent_operator', True,
'True to use the ground truth controller')
# Sensors to enable.
flags.DEFINE_bool('lidar', False, 'True to enable the lidar sensor')
# Modules to enable.
flags.DEFINE_bool('segmentation_drn', False,
'True to enable DRN segmantation operator')
flags.DEFINE_string('segmentation_drn_model_path',
'dependencies/models/drn_d_22_cityscapes.pth',
'Path to the model')
flags.DEFINE_bool('segmentation_dla', False,
'True to enable DLA segmantation operator')
flags.DEFINE_string('segmentation_dla_model_path',
'dependencies/dla/DLASeg.pth', 'Path to the model')
flags.DEFINE_bool('segmentation_gpu', True,
from absl import logging
import tensorflow as tf
import numpy as np
import imagenet_input
try:
import h5py as _ # pylint: disable=g-import-not-at-top
HAS_H5PY = True
except ImportError:
logging.warning('`h5py` is not installed. Please consider installing it '
'to save weights for long-running training.')
HAS_H5PY = False
flags.DEFINE_bool('use_tpu', True, 'Use TPU model instead of CPU.')
flags.DEFINE_string('tpu', None, 'Name of the TPU to use.')
flags.DEFINE_string('data', None, 'Path to training and testing data.')
FLAGS = flags.FLAGS
# TODO(xiejw): Revert the PER_CORE_BATCH_SIZE to 128 once the model can run.
PER_CORE_BATCH_SIZE = 64
NUM_CLASSES = 1000
IMAGE_SIZE = 224
APPROX_IMAGENET_TRAINING_IMAGES = 1280000 # Approximate number of images.
APPROX_IMAGENET_TEST_IMAGES = 48000 # Approximate number of images.
WEIGHTS_TXT = '/tmp/resnet50_weights.h5'
from tensorflow.contrib.slim.nets import inception
from tensorflow.contrib.tpu.python.tpu import tpu_config
from tensorflow.contrib.tpu.python.tpu import tpu_estimator
from tensorflow.contrib.tpu.python.tpu import tpu_optimizer
flags.DEFINE_float('learning_rate', 0.02, 'Learning rate.')
flags.DEFINE_float('depth_multiplier', 1.0, 'Depth Multiplier on Inception')
flags.DEFINE_integer(
'train_steps', 800,
'Total number of steps. Note that the actual number of '
'steps is the next multiple of --iterations greater '
'than this value.')
flags.DEFINE_integer('save_checkpoints_secs', None,
'Seconds between checkpoint saves')
flags.DEFINE_bool('use_tpu', True, 'Use TPUs rather than plain CPUs')
flags.DEFINE_string('use_data', 'fake', 'Data from "fake","real"')
flags.DEFINE_string('data_dir', '', 'Path of the data (for use_data=real)')
flags.DEFINE_string('master', 'local',
'BNS name of the TensorFlow master to use.')
flags.DEFINE_string('model_dir', None, 'Estimator model_dir')
flags.DEFINE_integer('iterations', 40,
'Number of iterations per TPU training loop.')
flags.DEFINE_string('optimizer', 'momentum',
'optimizer (one of sgd, rms, momentum)')
flags.DEFINE_integer('num_shards', 8, 'Number of shards (TPU chips).')
flags.DEFINE_integer('batch_size', 64,
'Global batch_size, not the per-shard batch_size')
flags.DEFINE_integer('num_labels', 1024, 'number of classes to distinguish')
flags.DEFINE_integer('width', 304, 'width of input image')
flags.DEFINE_integer('height', 304, 'height of input image')
from lingvo.jax import py_utils
from lingvo.jax import train
import tensorflow.compat.v2 as tf
# Required import to setup work units when running through XManager.
FLAGS = flags.FLAGS
flags.DEFINE_string('model', None, 'Lingvo Jax model name.')
flags.DEFINE_string('job_log_dir', None,
'Directory where all experiment assets will be stored.')
flags.DEFINE_enum('mode', 'train', ['train', 'eval', 'decode', 'decode_once'],
'Flag to control which job is called.')
flags.DEFINE_bool(
'eval_on_test', False, 'If True, then the training loop '
'includes a full evaluation on all the test set splits. '
'This can be set to True if we do not want an additional job '
'to run continuous eval.')
flags.DEFINE_bool(
'multi_host_checkpointing', False,
'Whether to use multi-host checkpointing or not. Only useful for '
'multi-host SPMD models.')
flags.DEFINE_bool(
'maybe_use_persistence_checkpointing', False,
'If suitable, will try to rely on persistence-based checkpointing rather '
'than Flax-based checkpointing for SPMD models.')
flags.DEFINE_string(
'checkpoint_todelete_subdir', None,
'If set, checkpoints to be deleted will be only renamed into a '
'subdirectory with the provided string. Otherwise, they will be directly '
'deleted from the file system. Useful if checkpoint deletion is time '
LOGGER = logging.getLogger("dummy-gen")
FLAGS = flags.FLAGS
flags.DEFINE_string(
'project_yml_path', None,
'the path of the project.yml, normally it is stored on the top of the project.'
)
flags.DEFINE_string(
'table_suffix', None,
'suffix appended to the model tables (not overwrite, create views with this suffix)'
)
flags.DEFINE_bool('refresh', False, 'whether to refresh this version of data, not recommended, better to change "data_id".')
class Core:
PROJ_CONF = parse_helper.parse_project_yml(FLAGS.project_yml_path)
PROJ_ROOT_PATH = os.path.join(os.getcwd, os.path.dirname(__file__))
MODEL_PATH = os.path.join(PROJ_ROOT_PATH, PROJ_CONF['models'])
SEED_PATH = os.path.join(PROJ_ROOT_PATH, PROJ_CONF['seed'])
STATIC_PATH = os.path.join(PROJ_ROOT_PATH, PROJ_CONF['static'])
def is_version_data_existed(self):
"""Check if there are already existed data in BQ
Method: iterate all models and
"""
# if exists, skip
# return
from __future__ import division
from __future__ import print_function
from absl import flags, app
from data.bert_tokenization import FullTokenizer
from data.bert_regression_data_lib import StsProcessor
FLAGS = flags.FLAGS
flags.DEFINE_string("input_data_dir", None, "")
flags.DEFINE_string("set_type", "train", "")
flags.DEFINE_string("vocab_file", None,
"The vocabulary file that the BERT model was trained on.")
flags.DEFINE_bool(
"do_lower_case", True,
"Whether to lower case the input text. Should be True for uncased "
"models and False for cased models.")
flags.DEFINE_integer(
"max_seq_length", 128,
"The maximum total input sequence length after WordPiece tokenization. "
"Sequences longer than this will be truncated, and sequences shorter "
"than this will be padded.")
def main(_):
tokenizer = FullTokenizer(FLAGS.vocab_file,
do_lower_case=FLAGS.do_lower_case)
processor = StsProcessor()
with open("token_check.txt", "w") as output_file:
for example in processor.get_examples(FLAGS.input_data_dir,
FLAGS.set_type):
'resnet_checkpoint', '',
'Location of the ResNet50 checkpoint to use for model '
'initialization.')
flags.DEFINE_string(
'training_file_pattern', None,
'Glob for training data files (e.g., COCO train - minival set)')
flags.DEFINE_string('validation_file_pattern', None,
'Glob for evaluation tfrecords (e.g., COCO val2017 set)')
flags.DEFINE_string('val_json_file', None,
'COCO validation JSON containing golden bounding boxes.')
tf.flags.DEFINE_integer('num_examples_per_epoch', 118287,
'Number of examples in one epoch')
tf.flags.DEFINE_integer('num_epochs', 15, 'Number of epochs for training')
flags.DEFINE_string('mode', 'train',
'Mode to run: train or eval (default: train)')
flags.DEFINE_bool('eval_after_training', False, 'Run one eval after the '
'training finishes.')
flags.DEFINE_bool('use_fake_data', False, 'Use fake input.')
# For Eval mode
flags.DEFINE_integer('min_eval_interval', 180,
'Minimum seconds between evaluations.')
FLAGS = flags.FLAGS
_STOP = -1
def main(argv):
del argv # Unused.
# TODO(b/132208296): remove this workaround that uses control flow v2.
control_flow_util.ENABLE_CONTROL_FLOW_V2 = True
from absl import app
from absl import flags
from utils.data import make_tfrecord
from utils.tokenization import FullTokenizer
FLAGS = flags.FLAGS
flags.DEFINE_list('file_patterns', None, "Document's files patterns.")
flags.DEFINE_integer('max_seq_length', 512, 'Maximum sequential length.')
flags.DEFINE_float('mask_prob', 0.1, 'The probability of mask LM number per sequence.')
flags.DEFINE_integer(
'max_mask_length', 20, 'Maximum number of masked LM predictions per sequence.'
)
flags.DEFINE_bool(
'random_seq', False,
'Original Bert use random next sequence to predict. For Albert, we set this as False'
)
flags.DEFINE_bool(
'whole_word_mask', False, 'Maximum number of masked LM predictions per sequence.'
)
flags.DEFINE_string(
'tfrecord_file_prefix', 'data-training/albert-training-data',
'Prefix used as saved tfrecord files.'
)
flags.DEFINE_integer('tfrecord_file_num', 10, 'How many tfrecord files to store.')
flags.DEFINE_bool('do_lower_case', True, '')
flags.DEFINE_string('vocab_file', None, '')
def main(_):
tokenizer = FullTokenizer(
"data_dir",
default=os.path.join(os.getenv("TEST_TMPDIR", "/tmp"),
"bayesian_neural_network/data"),
help="Directory where data is stored (if using real data).")
flags.DEFINE_string("model_dir",
default=os.path.join(os.getenv("TEST_TMPDIR", "/tmp"),
"bayesian_neural_network/"),
help="Directory to put the model's fit.")
flags.DEFINE_integer("viz_steps",
default=400,
help="Frequency at which save visualizations.")
flags.DEFINE_integer("num_monte_carlo",
default=50,
help="Network draws to compute predictive probabilities.")
flags.DEFINE_bool("fake_data",
default=None,
help="If true, uses fake data. Defaults to real data.")
FLAGS = flags.FLAGS
def plot_weight_posteriors(names, qm_vals, qs_vals, fname):
"""Save a PNG plot with histograms of weight means and stddevs.
Args:
names: A Python `iterable` of `str` variable names.
qm_vals: A Python `iterable`, the same length as `names`,
whose elements are Numpy `array`s, of any shape, containing
posterior means of weight varibles.
qs_vals: A Python `iterable`, the same length as `names`,
whose elements are Numpy `array`s, of any shape, containing
flags.DEFINE_string('hparams', '', 'Hyper parameters.')
flags.DEFINE_string('split_name', 'validation', 'Name of the split.')
flags.DEFINE_float('moving_average_decay', 0.9999,
'The decay to use for the moving average.')
flags.DEFINE_integer('eval_interval_secs', 120,
'The frequency, in seconds, with which evaluation is run.')
flags.DEFINE_integer(
'num_examples', -1,
'If positive - maximum number of example to use for evaluation.')
flags.DEFINE_bool('eval_once', False,
'If true then evaluate model only once.')
flags.DEFINE_string('trainable_scopes', None,
'If set then it defines list of variable scopes for '
'trainable variables.')
def main(_):
if not FLAGS.train_dir and not FLAGS.checkpoint_path:
print('Either --train_dir or --checkpoint_path flags has to be provided.')
if FLAGS.train_dir and FLAGS.checkpoint_path:
print('Only one of --train_dir or --checkpoint_path should be provided.')
params = model_lib.default_hparams()
params.parse(FLAGS.hparams)
tf.logging.info('User provided hparams: %s', FLAGS.hparams)
tf.logging.info('All hyper parameters: %s', params)
from official.modeling.hyperparams import params_dict
from official.modeling.training import distributed_executor as executor
from official.utils import hyperparams_flags
from official.vision.detection.configs import factory as config_factory
from official.vision.detection.dataloader import input_reader
from official.vision.detection.dataloader import mode_keys as ModeKeys
from official.vision.detection.executor.detection_executor import DetectionDistributedExecutor
from official.vision.detection.modeling import factory as model_factory
from official.utils.flags import core as flags_core
from official.utils.misc import distribution_utils
from official.utils.misc import keras_utils
hyperparams_flags.initialize_common_flags()
flags_core.define_log_steps()
flags.DEFINE_bool('enable_xla', default=False, help='Enable XLA for GPU')
flags.DEFINE_string('mode',
default='train',
help='Mode to run: `train`, `eval` or `train_and_eval`.')
flags.DEFINE_string('model',
default='retinanet',
help='Model to run: `retinanet` or `mask_rcnn`.')
flags.DEFINE_string('training_file_pattern', None,
'Location of the train data.')
flags.DEFINE_string('eval_file_pattern', None, 'Location of ther eval data')
flags.DEFINE_string(
flags.DEFINE_integer('corruptions_interval', 135,
'Number of epochs between evaluating on the corrupted '
'test data. Use -1 to never evaluate.')
flags.DEFINE_integer('checkpoint_interval', 27,
'Number of epochs between saving checkpoints. Use -1 to '
'never save checkpoints.')
flags.DEFINE_string('alexnet_errors_path', None,
'Path to AlexNet corruption errors file.')
flags.DEFINE_integer('num_bins', 15, 'Number of bins for ECE computation.')
flags.DEFINE_integer('num_eval_samples', 1,
'Number of model predictions to sample per example at '
'eval time.')
# Accelerator flags.
flags.DEFINE_bool('use_gpu', False, 'Whether to run on GPU or otherwise TPU.')
flags.DEFINE_bool('use_bfloat16', True, 'Whether to use mixed precision.')
flags.DEFINE_integer('num_cores', 32, 'Number of TPU cores or number of GPUs.')
flags.DEFINE_string('tpu', None,
'Name of the TPU. Only used if use_gpu is False.')
FLAGS = flags.FLAGS
# Number of images in ImageNet-1k train dataset.
APPROX_IMAGENET_TRAIN_IMAGES = 1281167
# Number of images in eval dataset.
IMAGENET_VALIDATION_IMAGES = 50000
NUM_CLASSES = 1000
_LR_SCHEDULE = [ # (multiplier, epoch to start) tuples
(1.0, 5), (0.1, 30), (0.01, 60), (0.001, 80)
]
'Maximum rendering time for plotting/playback')
flags.DEFINE_float('minRenderTime', 0.01,
'Minimum rendering time for plotting/playback')
flags.DEFINE_float('qvelMax', 1e-05,
'Joint velocity threshold to continue rendering')
flags.DEFINE_float('qvelMean', 1e-06,
'Joint velocity mean threshold to continue rendering')
flags.DEFINE_float('qaccMax', 1e-05,
'Joint acceleration threshold to continue rendering')
flags.DEFINE_float('qaccMean', 1e-06,
'Joint acceleration mean threshold to continue rendering')
flags.DEFINE_string('model_filename', 'ratMocap\\models\\ratMocap.xml',
'filename for model.')
flags.DEFINE_string('outName', None, 'filename and path for output.')
flags.DEFINE_bool('qOnly', True, 'Whether to make only .mat of joint angles.')
flags.DEFINE_bool('record', False, 'Whether to write a video file.')
flags.DEFINE_bool('play', False, 'Whether to play a video.')
flags.DEFINE_bool('save', True, 'Whether to save the rendering.')
flags.DEFINE_bool('silent', False, 'Whether to display rendering progress.')
def get_model_and_assets():
"""Returns a tuple containing the model XML string and a dict of assets."""
return common.read_model(FLAGS.model_filename), common.ASSETS
class jeffRat(base.Task):
"""A task for the Rat skeleton."""
def __init__(self, random=None):
"""Initializes an instance of `jeffRat`.
