from google.cloud import storage flags.DEFINE_string('project', None, 'Google cloud project id for uploading the dataset.') flags.DEFINE_string('gcs_output_path', None, 'GCS path for uploading the dataset.') flags.DEFINE_string('local_scratch_dir', None, 'Scratch directory path for temporary files.') flags.DEFINE_string( 'raw_data_dir', None, 'Directory path for raw Imagenet dataset. ' 'Should have train and validation subdirectories inside it.') flags.DEFINE_string('imagenet_username', None, 'Username for Imagenet.org account') flags.DEFINE_string('imagenet_access_key', None, 'Access Key for Imagenet.org account') flags.DEFINE_boolean('gcs_upload', True, 'Set to false to not upload to gcs.') FLAGS = flags.FLAGS BASE_URL = 'http://www.image-net.org/challenges/LSVRC/2012/nnoupb/' LABELS_URL = 'https://raw.githubusercontent.com/tensorflow/models/master/research/inception/inception/data/imagenet_2012_validation_synset_labels.txt' # pylint: disable=line-too-long TRAINING_FILE = 'ILSVRC2012_img_train.tar' VALIDATION_FILE = 'ILSVRC2012_img_val.tar' LABELS_FILE = 'synset_labels.txt' TRAINING_SHARDS = 1024 VALIDATION_SHARDS = 128 TRAINING_DIRECTORY = 'train' VALIDATION_DIRECTORY = 'validation'
flags.DEFINE_integer('replay_buffer_capacity', 1001, 'Replay buffer capacity per env.') flags.DEFINE_integer('num_parallel_environments', 30, 'Number of environments to run in parallel') flags.DEFINE_integer('num_environment_steps', 10000000, 'Number of environment steps to run before finishing.') flags.DEFINE_integer('num_epochs', 25, 'Number of epochs for computing policy updates.') flags.DEFINE_integer( 'collect_episodes_per_iteration', 30, 'The number of episodes to take in the environment before ' 'each update. This is the total across all parallel ' 'environments.') flags.DEFINE_integer('num_eval_episodes', 30, 'The number of episodes to run eval on.') flags.DEFINE_boolean('use_rnns', False, 'If true, use RNN for policy and value function.') FLAGS = flags.FLAGS @gin.configurable def train_eval( root_dir, tf_master='', env_name='HalfCheetah-v2', env_load_fn=suite_mujoco.load, random_seed=0, # TODO(b/127576522): rename to policy_fc_layers. actor_fc_layers=(200, 100), value_fc_layers=(200, 100), use_rnns=False, # Params for collect
from non_semantic_speech_benchmark.eval_embedding.keras import get_data from non_semantic_speech_benchmark.eval_embedding.keras import models FLAGS = flags.FLAGS flags.DEFINE_string('file_pattern', None, 'Dataset location.') flags.DEFINE_string('en', None, 'Embedding name.') flags.DEFINE_string('ed', None, 'Embedding dimension.') flags.DEFINE_string('label_name', None, 'Name of label to use.') flags.DEFINE_list('label_list', None, 'List of possible label values.') flags.DEFINE_integer('batch_size', None, 'The number of images in each batch.') flags.DEFINE_integer('tbs', None, 'not used') flags.DEFINE_integer('nc', None, 'num_clusters') flags.DEFINE_boolean('ubn', None, 'Whether to normalize') flags.DEFINE_float('lr', None, 'not used') flags.DEFINE_string('logdir', None, 'Directory where the model was written to.') flags.DEFINE_string('eval_dir', None, 'Directory where the results are saved to.') flags.DEFINE_integer('take_fixed_data', None, 'If not `None`, take a fixed number of data elements.') flags.DEFINE_integer('timeout', 7200, 'Wait-for-checkpoint timeout.') def eval_and_report(): """Eval on voxceleb.""" logging.info('embedding_name: %s', FLAGS.en)
import os import pickle import mysql import mysql.connector from absl import app from absl import flags from absl import logging from tqdm import tqdm FLAGS = flags.FLAGS flags.DEFINE_string('input', 'exp_out/location/run_8/disambiguation.tsv', '') flags.DEFINE_string('uuidmap', 'data/location/uuid.pkl', '') flags.DEFINE_boolean('create_tables', False, '') flags.DEFINE_boolean('drop_tables', False, '') logging.set_verbosity(logging.INFO) def create_tables(): cnx_g = mysql.connector.connect(option_files=os.path.join( os.environ['HOME'], '.mylogin.cnf'), database='patent_20200630') cnx_pg = mysql.connector.connect(option_files=os.path.join( os.environ['HOME'], '.mylogin.cnf'), database='pregrant_publications') g_cursor = cnx_g.cursor() g_cursor.execute(
'Name for eval.') flags.DEFINE_integer( 'keep_checkpoint_max', 5, 'Maximum number of checkpoints to keep.') flags.DEFINE_integer( 'keep_hub_module_max', 1, 'Maximum number of Hub modules to keep.') flags.DEFINE_float( 'temperature', 0.1, 'Temperature parameter for contrastive loss.') flags.DEFINE_boolean( 'hidden_norm', True, 'Temperature parameter for contrastive loss.') flags.DEFINE_enum( 'head_proj_mode', 'nonlinear', ['none', 'linear', 'nonlinear'], 'How the head projection is done.') flags.DEFINE_integer( 'head_proj_dim', 128, 'Number of head projection dimension.') flags.DEFINE_integer( 'num_nlh_layers', 1, 'Number of non-linear head layers.') flags.DEFINE_boolean(
"number of steps to take in validation") flags.DEFINE_integer("batch_size", 32, "batch size, num parallel streams to train on at once") flags.DEFINE_integer("window_size", 20480, "num frames to push into model at once") flags.DEFINE_integer("timestep", 12, "the number of frames ahead to predict") flags.DEFINE_integer("hidden_size", 512, "the hidden layer size of the encoder") flags.DEFINE_integer("out_size", 256, "the hidden layer size of the gru") flags.DEFINE_integer("no_gru_layers", 1, "the number of layers in the gru") flags.DEFINE_integer("val_every", None, "how often to perform validation") flags.DEFINE_integer("save_every", None, "save every n steps") flags.DEFINE_integer("log_every", 10, "append to log file every n steps") flags.DEFINE_integer("log_tb_every", 50, "save tb scalars every n steps") flags.DEFINE_boolean("dry_run", False, "dry run") flags.mark_flag_as_required("train_data") flags.mark_flag_as_required("val_data") flags.mark_flag_as_required("expdir") def validation(model, val_datastream, val_steps, features_in): model.eval() losses = [] accuracies = [] hidden = None with FixedRandomState(42): for step, val_batch in enumerate(val_datastream): data = val_batch["data"].to(device) if features_in == "raw":
_MOVE_SCREEN = actions.FUNCTIONS.Move_screen.id _SELECT_ARMY = actions.FUNCTIONS.select_army.id _SELECT_ALL = [0] _NOT_QUEUED = [0] step_mul = 8 FLAGS = flags.FLAGS flags.DEFINE_string("map", "MoveToBeacon", "Name of a map to use to play.") start_time = datetime.datetime.now().strftime("%Y%m%d%H%M") flags.DEFINE_string("log", "tensorboard", "logging type(stdout, tensorboard)") flags.DEFINE_string("algorithm", "deepq", "RL algorithm to use.") flags.DEFINE_integer("timesteps", 2000000, "Steps to train") flags.DEFINE_float("exploration_fraction", 0.2, "Exploration Fraction") flags.DEFINE_boolean("prioritized", True, "prioritized_replay") flags.DEFINE_boolean("dueling", True, "dueling") flags.DEFINE_float("lr", 0.0005, "Learning rate") flags.DEFINE_integer("num_agents", 4, "number of RL agents for A2C") flags.DEFINE_integer("num_scripts", 4, "number of script agents for A2C") flags.DEFINE_integer("nsteps", 20, "number of batch steps for A2C") flags.DEFINE_string("experiment", "SCREEN_DIM=16", "name of experiment") PROJ_DIR = os.path.dirname(os.path.abspath(__file__)) max_mean_reward = 0 last_filename = "" start_time = datetime.datetime.now().strftime("%m%d%H%M") SCREEN_DIM = 16
flags.DEFINE_integer('max_elements_per_user', 1000, 'Max number of training ' 'sentences to use per user.') flags.DEFINE_integer( 'num_validation_examples', 10000, 'Number of examples ' 'to use from test set for per-round validation.') # Modeling flags flags.DEFINE_integer('vocab_size', 10000, 'Size of vocab to use.') flags.DEFINE_integer('embedding_size', 96, 'Dimension of word embedding to use.') flags.DEFINE_integer('latent_size', 670, 'Dimension of latent size to use in recurrent cell') flags.DEFINE_integer('num_layers', 1, 'Number of stacked recurrent layers to use.') flags.DEFINE_boolean( 'shared_embedding', False, 'Boolean indicating whether to tie input and output embeddings.') FLAGS = flags.FLAGS def main(argv): if len(argv) > 1: raise app.UsageError('Expected no command-line arguments, ' 'got: {}'.format(argv)) tf.compat.v1.enable_v2_behavior() model_builder = functools.partial( models.create_recurrent_model, vocab_size=FLAGS.vocab_size, embedding_size=FLAGS.embedding_size,
def define_resnet_flags(resnet_size_choices=None): """Add flags and validators for ResNet.""" flags_core.define_base() flags_core.define_performance(num_parallel_calls=False, tf_gpu_thread_mode=True, datasets_num_private_threads=True, datasets_num_parallel_batches=True) flags_core.define_image() flags_core.define_benchmark() flags.adopt_module_key_flags(flags_core) flags.DEFINE_enum( name='resnet_version', short_name='rv', default='1', enum_values=['1', '2'], help=flags_core.help_wrap( 'Version of ResNet. (1 or 2) See README.md for details.')) flags.DEFINE_bool( name='fine_tune', short_name='ft', default=False, help=flags_core.help_wrap( 'If True do not train any parameters except for the final layer.')) flags.DEFINE_string( name='pretrained_model_checkpoint_path', short_name='pmcp', default=None, help=flags_core.help_wrap( 'If not None initialize all the network except the final layer with ' 'these values')) flags.DEFINE_boolean(name='eval_only', default=False, help=flags_core.help_wrap( 'Skip training and only perform evaluation on ' 'the latest checkpoint.')) flags.DEFINE_boolean( name='image_bytes_as_serving_input', default=False, help=flags_core.help_wrap( 'If True exports savedmodel with serving signature that accepts ' 'JPEG image bytes instead of a fixed size [HxWxC] tensor that ' 'represents the image. The former is easier to use for serving at ' 'the expense of image resize/cropping being done as part of model ' 'inference. Note, this flag only applies to ImageNet and cannot ' 'be used for CIFAR.')) flags.DEFINE_boolean( name='use_train_and_evaluate', default=False, help=flags_core.help_wrap( 'If True, uses `tf.estimator.train_and_evaluate` for the training ' 'and evaluation loop, instead of separate calls to `classifier.train ' 'and `classifier.evaluate`, which is the default behavior.')) flags.DEFINE_string( name='worker_hosts', default=None, help=flags_core.help_wrap( 'Comma-separated list of worker ip:port pairs for running ' 'multi-worker models with DistributionStrategy. The user would ' 'start the program on each host with identical value for this flag.' )) flags.DEFINE_integer(name='task_index', default=-1, help=flags_core.help_wrap( 'If multi-worker training, the task_index of ' 'this worker.')) flags.DEFINE_bool(name='enable_lars', default=False, help=flags_core.help_wrap( 'Enable LARS optimizer for large batch training.')) flags.DEFINE_float( name='label_smoothing', default=0.0, help=flags_core.help_wrap( 'Label smoothing parameter used in the softmax_cross_entropy')) flags.DEFINE_float(name='weight_decay', default=1e-4, help=flags_core.help_wrap( 'Weight decay coefficiant for l2 regularization.')) choice_kwargs = dict( name='resnet_size', short_name='rs', default='50', help=flags_core.help_wrap('The size of the ResNet model to use.')) if resnet_size_choices is None: flags.DEFINE_string(**choice_kwargs) else: flags.DEFINE_enum(enum_values=resnet_size_choices, **choice_kwargs)
from core.config import cfg from tensorflow.python.saved_model import tag_constants # helper files import object_detection_helper as obj_helper from threadedConsumer import Worker import time #endregion #region flags flags.DEFINE_string('framework', 'tf', '(tf, tflite, trt') flags.DEFINE_string('weights', './checkpoints/yolov4-416', 'path to weights file') flags.DEFINE_integer('size', 416, 'resize images to') flags.DEFINE_boolean('tiny', False, 'yolo or yolo-tiny') flags.DEFINE_string('model', 'yolov4', 'yolov3 or yolov4') flags.DEFINE_string('video', './data/video/test.mp4', 'path to input video or set to 0 for webcam') flags.DEFINE_float('iou', 0.45, 'iou threshold') flags.DEFINE_float('score', 0.50, 'score threshold') flags.DEFINE_boolean('dont_show', False, 'dont show video output') flags.DEFINE_boolean('info', False, 'show detailed info of tracked objects') flags.DEFINE_boolean('count', False, 'count objects being tracked on screen') #endregion # Definition of the parameters max_cosine_distance = 0.4 nn_budget = None interpreter = None
from absl import logging import tensorflow as tf from official.modeling import performance from official.staging.training import controller from official.utils.flags import core as flags_core from official.utils.logs import logger from official.utils.misc import distribution_utils from official.utils.misc import keras_utils from official.utils.misc import model_helpers from official.vision.image_classification import common from official.vision.image_classification import imagenet_preprocessing from official.vision.image_classification import resnet_runnable flags.DEFINE_boolean(name='use_tf_function', default=True, help='Wrap the train and test step inside a ' 'tf.function.') flags.DEFINE_boolean(name='single_l2_loss_op', default=False, help='Calculate L2_loss on concatenated weights, ' 'instead of using Keras per-layer L2 loss.') def build_stats(runnable, time_callback): """Normalizes and returns dictionary of stats. Args: runnable: The module containing all the training and evaluation metrics. time_callback: Time tracking callback instance. Returns:
def define_common_bert_flags(): """Define common flags for BERT tasks.""" flags_core.define_base(data_dir=False, model_dir=True, clean=False, train_epochs=False, epochs_between_evals=False, stop_threshold=False, batch_size=False, num_gpu=True, hooks=False, export_dir=False, distribution_strategy=True, run_eagerly=True) flags_core.define_distribution() flags.DEFINE_string('bert_config_file', None, 'Bert configuration file to define core bert layers.') flags.DEFINE_string( 'model_export_path', None, 'Path to the directory, where trainined model will be ' 'exported.') flags.DEFINE_string('tpu', '', 'TPU address to connect to.') flags.DEFINE_string( 'init_checkpoint', None, 'Initial checkpoint (usually from a pre-trained BERT model).') flags.DEFINE_integer('num_train_epochs', 3, 'Total number of training epochs to perform.') flags.DEFINE_integer( 'steps_per_loop', 200, 'Number of steps per graph-mode loop. Only training step ' 'happens inside the loop. Callbacks will not be called ' 'inside.') flags.DEFINE_float('learning_rate', 5e-5, 'The initial learning rate for Adam.') flags.DEFINE_boolean( 'scale_loss', False, 'Whether to divide the loss by number of replica inside the per-replica ' 'loss function.') flags.DEFINE_boolean( 'use_keras_compile_fit', False, 'If True, uses Keras compile/fit() API for training logic. Otherwise ' 'use custom training loop.') flags.DEFINE_string( 'hub_module_url', None, 'TF-Hub path/url to Bert module. ' 'If specified, init_checkpoint flag should not be used.') flags.DEFINE_enum( 'model_type', 'bert', ['bert', 'albert'], 'Specifies the type of the model. ' 'If "bert", will use canonical BERT; if "albert", will use ALBERT model.' ) flags.DEFINE_bool( 'hub_module_trainable', True, 'True to make keras layers in the hub module trainable.') # Adds flags for mixed precision and multi-worker training. flags_core.define_performance( num_parallel_calls=False, inter_op=False, intra_op=False, synthetic_data=False, max_train_steps=False, dtype=True, dynamic_loss_scale=True, loss_scale=True, all_reduce_alg=True, num_packs=False, tf_gpu_thread_mode=True, datasets_num_private_threads=True, enable_xla=True, fp16_implementation=True, )
"Dropout probability for attention probabilities.") flags.DEFINE_integer("max_positions", 256, "Max text sequence length model might be used with.") flags.DEFINE_integer("max_segments", 8, "Max segment types model might be used with.") flags.DEFINE_integer("max_conditions", 64, "Max condition sequence length model might be used with.") flags.DEFINE_integer("max_image_regions", 128, "Max encoded image regions model might be used with.") # Options for training the conditional text planner. flags.DEFINE_boolean("conditional_decoding", False, "Use conditional decoding.") flags.DEFINE_float("span_sample_p", 0.3, "Geometric distribtion parameter.") flags.DEFINE_integer("span_length", 10, "Max random span length selected.") flags.DEFINE_integer("condition_length", 40, "Max conditioning sequence length.") flags.DEFINE_integer("question_length", 30, "Max question length.") flags.DEFINE_integer("answer_length", 10, "Max answer length.") # Options for decoding. flags.DEFINE_integer("decode_length", 30, "Max decoding length.")
# See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Rewrite script for TF->JAX.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function import collections # Dependency imports from absl import app from absl import flags flags.DEFINE_boolean('numpy_to_jax', False, 'Whether or not to rewrite numpy imports to jax.numpy') flags.DEFINE_list('omit_deps', [], 'List of build deps being omitted.') FLAGS = flags.FLAGS TF_REPLACEMENTS = { 'import tensorflow ': 'from tensorflow_probability.python.internal.backend import numpy ', 'import tensorflow.compat.v1': 'from tensorflow_probability.python.internal.backend.numpy.compat ' 'import v1', 'import tensorflow.compat.v2': 'from tensorflow_probability.python.internal.backend.numpy.compat ' 'import v2', 'import tensorflow_probability as tfp': 'import tensorflow_probability as tfp; '
# from utils.dataset_util import * # from utils.label_map_util import * import utils.dataset_util as dataset_util import utils.label_map_util as label_map_util flags.DEFINE_string('data_dir', '', 'Root directory to raw PASCAL VOC dataset.') flags.DEFINE_string('set', 'train', 'Convert training set, validation set or ' 'merged set.') flags.DEFINE_string('annotations_dir', 'Annotations', '(Relative) path to annotations directory.') flags.DEFINE_string('year', 'VOC2007', 'Desired challenge year.') # flags.DEFINE_string('project', '', 'Desired project name.') flags.DEFINE_string('output_path', '', 'Path to output TFRecord') flags.DEFINE_string('label_map_path', '', 'Path to label map proto') # './data/pascal_label_map.pbtxt' flags.DEFINE_boolean('ignore_difficult_instances', False, 'Whether to ignore ' 'difficult instances') flags.DEFINE_string('action', 'tfrecord', 'Action in [tfrecord, imageset]') # flags.DEFINE_string('imageset', 'image', 'image set name') FLAGS = flags.FLAGS SETS = ['train', 'val', 'trainval', 'test'] # YEARS = ['VOC2007', 'VOC2012', 'merged'] def gen_image_set(data_dir, year): """generate image set text file from annotation xmls Args:
import time from absl import app, flags, logging from absl.flags import FLAGS import cv2 import numpy as np import tensorflow as tf from yolov3_tf2.models import (YoloV3, YoloV3Tiny) from yolov3_tf2.dataset import transform_images, load_tfrecord_dataset from yolov3_tf2.utils import draw_outputs # flags.DEFINE_string('classes', './data/voc2012.names', 'path to classes file') flags.DEFINE_string('classes', './data/coco.names', 'path to classes file') flags.DEFINE_string('weights', './checkpoints/yolov3_train_15.tf', 'path to weights file') flags.DEFINE_boolean('tiny', False, 'yolov3 or yolov3-tiny') flags.DEFINE_integer('size', 416, 'resize images to') flags.DEFINE_string('image', './data/street.jpg', 'path to input image') flags.DEFINE_string('tfrecord', None, 'tfrecord instead of image') flags.DEFINE_string('output', './output.jpg', 'path to output image') flags.DEFINE_integer('num_classes', 80, 'number of classes in the model') def main(_argv): physical_devices = tf.config.experimental.list_physical_devices('GPU') for physical_device in physical_devices: tf.config.experimental.set_memory_growth(physical_device, True) if FLAGS.tiny: yolo = YoloV3Tiny(classes=FLAGS.num_classes) else: yolo = YoloV3(classes=FLAGS.num_classes)
'any block group.')) flags.DEFINE_float( 'dropblock_keep_prob', default=None, help=('keep_prob parameter of DropBlock. Will not be used if ' 'dropblock_groups is empty.')) flags.DEFINE_integer( 'dropblock_size', default=None, help=('size parameter of DropBlock. Will not be used if dropblock_groups ' 'is empty.')) # copybara:strip_begin flags.DEFINE_boolean( 'xla_compile', default=False, help=('Compile computation with XLA, this flag has no effect when running ' 'on TPU.')) flags.DEFINE_string( 'tpu_job_name', None, 'Name of TPU worker binary. Only necessary if job name is changed from' ' default tpu_worker.') # copybara:strip_end flags.DEFINE_integer( 'profile_every_n_steps', default=0, help=('Number of steps between collecting profiles if larger than 0')) flags.DEFINE_string( 'mode',
"save to checkpoints.") flags.DEFINE_string("result_dir", "results", "Path to results") # Dataset flags.DEFINE_string( "dataset_name", "tieredImageNet", "Name of the dataset to " "train on, which will be mapped to data.MetaDataset.") flags.DEFINE_integer("num_steps_limit", int(1e5), "Number of steps to train for.") flags.DEFINE_integer( "checkpoint_steps", 200, "The frequency, in number of " "steps, of saving the checkpoints.") # flags.DEFINE_string("exp_name", "experiment", "Name of the experiment.") flags.DEFINE_string("gpus", "0", "Ids of GPUs where the program run.") flags.DEFINE_boolean("evaluation_mode", False, "Whether to run in an " "evaluation-only mode.") flags.DEFINE_string("eval_set", "val", "Which set (train/val/test) to evaluate on.") flags.DEFINE_integer("random_seed", 0, "Global random seed.") def main(argv): del argv # Unused. os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpus tf.compat.v1.set_random_seed(FLAGS.random_seed) np.random.seed(FLAGS.random_seed) if FLAGS.model_cls == 'metafun_classifier': model_cls = MetaFunClassifier task_type = "classification"
import experiments.hyperparam_search.meta_mll_hyperparm as meta_mll_hparam import experiments.hyperparam_search.meta_svgd_hyperparam as meta_svgd_hparam import experiments.hyperparam_search.meta_vi_hyperparam as meta_vi_hparam import experiments.hyperparam_search.meta_mlap_hyperparam as meta_pac_hyperparam from absl import flags from absl import app flags.DEFINE_integer('n_cpus', default=32, help='number of cpus to use') flags.DEFINE_integer('n_gpus', default=1, help='number of gpus to use') flags.DEFINE_string('algos', default='pac', help='specifies which dataset to use') flags.DEFINE_string('metric', default='test_ll', help='specifies which metric to optimize') flags.DEFINE_boolean('cluster', default=False, help='whether to submit jobs with bsub') flags.DEFINE_boolean('dry', default=False, help='whether to only print launch commands') flags.DEFINE_boolean('load_analysis', default=False, help='whether to load the analysis from existing dirs') flags.DEFINE_boolean('resume', default=False, help='whether to resume checkpointed tune session') FLAGS = flags.FLAGS algo_map_dict = {'map': meta_mll_hparam, 'vi': meta_vi_hparam, 'svgd': meta_svgd_hparam, 'pac': meta_pac_hyperparam} def main(argv): hparam_search_modules = [algo_map_dict[algo_str] for algo_str in FLAGS.algos.split(',')]
from .eval_util import compute_errors from ..datasets.common import read_images_from_tfrecords kPredDir = '/tmp/hmr_output' # Change to where you saved your tfrecords kTFDataDir = '/scratch1/williamljb/hmr_multiview/tf_datasets/human36m_multi' flags.DEFINE_string('pred_dir', kPredDir, 'where to save model output of h36m') flags.DEFINE_string('tfh36m_dir', kTFDataDir, 'data dir: top of h36m in tf_records') flags.DEFINE_integer( 'protocol', 1, 'If 2, then only frontal cam (3) and trial 1, if 1, then all camera & trials' ) flags.DEFINE_boolean( 'vis', False, 'If true, visualizes the best and worst 30 results.') model = None sess = None # For visualization. renderer = None extreme_errors, contents = [], [] import os os.environ['CUDA_VISIBLE_DEVICES'] = '1' # -- Draw Utils --- def draw_content(content, config): global renderer input_img = content['image'] vert = content['vert']
flags.DEFINE_integer('iterations_per_loop', 100, 'Number of iterations per TPU training loop.') # For mode=train_and_eval, evaluation occurs after training is finished. # Note: independently of steps_per_checkpoint, estimator will save the most # recent checkpoint every 10 minutes by default for train_and_eval flags.DEFINE_string('mode', 'train', 'Mode to run: train, eval') flags.DEFINE_integer( 'train_batch_size', None, 'Batch size for training. If ' 'this is not provided, batch size is read from training ' 'config.') flags.DEFINE_string( 'hparams_overrides', None, 'Comma-separated list of ' 'hyperparameters to override defaults.') flags.DEFINE_integer('num_train_steps', None, 'Number of train steps.') flags.DEFINE_boolean('eval_training_data', False, 'If training data should be evaluated for this job.') flags.DEFINE_integer( 'sample_1_of_n_eval_examples', 1, 'Will sample one of ' 'every n eval input examples, where n is provided.') flags.DEFINE_integer( 'sample_1_of_n_eval_on_train_examples', 5, 'Will sample ' 'one of every n train input examples for evaluation, ' 'where n is provided. This is only used if ' '`eval_training_data` is True.') flags.DEFINE_string( 'model_dir', None, 'Path to output model directory ' 'where event and checkpoint files will be written.') flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') FLAGS = tf.flags.FLAGS
import numpy as np import numpy.random as npr import tensorflow as tf from tensorflow_privacy.privacy.analysis import privacy_ledger from tensorflow_privacy.privacy.analysis.rdp_accountant import compute_rdp_from_ledger from tensorflow_privacy.privacy.analysis.rdp_accountant import get_privacy_spent from tensorflow_privacy.privacy.optimizers import dp_optimizer AdamOptimizer = tf.compat.v1.train.AdamOptimizer FLAGS = flags.FLAGS flags.DEFINE_boolean( 'dpsgd', True, 'If True, train with DP-SGD. If False, ' 'train with vanilla SGD.') flags.DEFINE_float('learning_rate', .05, 'Learning rate for training') flags.DEFINE_float('noise_multiplier', 2.0, 'Ratio of the standard deviation to the clipping norm') flags.DEFINE_float('l2_norm_clip', 1.0, 'Clipping norm') flags.DEFINE_integer('batch_size', 64, 'Batch size') flags.DEFINE_integer('epochs', 2, 'Number of epochs') flags.DEFINE_integer('training_data_size', 2000, 'Training data size') flags.DEFINE_integer('test_data_size', 2000, 'Test data size') flags.DEFINE_integer('input_dimension', 5, 'Input dimension') flags.DEFINE_string('model_dir', None, 'Model directory') class EpsilonPrintingTrainingHook(tf.estimator.SessionRunHook): """Training hook to print current value of epsilon after an epoch."""
flags.DEFINE_string('pipeline_config_path', 'training/config/ppn_pipeline.config', 'Path to pipeline config ' 'file.') flags.DEFINE_integer('num_train_steps', 5000, 'Number of train steps.') flags.DEFINE_integer('num_eval_steps', 5, 'Number of train steps.') flags.DEFINE_string( 'hparams_overrides', None, 'Hyperparameter overrides, ' 'represented as a string containing comma-separated ' 'hparam_name=value pairs.') flags.DEFINE_string( 'checkpoint_dir', None, 'Path to directory holding a checkpoint. If ' '`checkpoint_dir` is provided, this binary operates in eval-only mode, ' 'writing resulting metrics to `model_dir`.') flags.DEFINE_boolean( 'run_once', False, 'If running in eval-only mode, whether to run just ' 'one round of eval vs running continuously (default).') flags.DEFINE_boolean('eval_training_data', True, 'If training data should be evaluated for this job.') FLAGS = flags.FLAGS def main(unused_argv): #flags.mark_flag_as_required('model_dir') #flags.mark_flag_as_required('pipeline_config_path') config = tf.estimator.RunConfig(model_dir=FLAGS.model_dir, save_checkpoints_steps=100) train_and_eval_dict = model_lib.create_estimator_and_inputs( run_config=config,
Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. """ import logging from typing import Iterable, Callable from copy import deepcopy import time from absl import flags from pywinauto import Application, timings from pywinauto.base_wrapper import BaseWrapper from pywinauto.findwindows import ElementNotFoundError from pywinauto.keyboard import send_keys flags.DEFINE_boolean('dev_mode', False, 'Dev mode - expect Orbit MainWindow to be opened, ' 'and do not close Orbit at the end') class OrbitE2EError(RuntimeError): pass class E2ETestCase: """ Encapsulates a single test executed as part of a E2E Test Suite. Inherit from this class to define a new test, and pass an instance of your test in the E2ETestSuite constructor to have it executed. All named arguments passed to the constructor of your test will be forwarded to the _execute method. To create a parameterized test, simply inherit from this class and provide your own _execute methods with any number of named arguments.
flags.DEFINE_string("input_graph", None, "input graph") flags.DEFINE_string("output_graph", None, "input graph") flags.DEFINE_string("config", None, "LPOT config file") flags.DEFINE_float("conf_threshold", 0.5, "confidence threshold") flags.DEFINE_float("iou_threshold", 0.4, "IoU threshold") flags.DEFINE_integer("num_intra_threads", 0, "number of intra threads") flags.DEFINE_integer("num_inter_threads", 1, "number of inter threads") flags.DEFINE_boolean("benchmark", False, "benchmark mode") flags.DEFINE_boolean("profiling", False, "Signal of profiling") FLAGS = flags.FLAGS class NMS(): def __init__(self, conf_threshold, iou_threshold): self.conf_threshold = conf_threshold self.iou_threshold = iou_threshold def __call__(self, sample): preds, labels = sample if not isinstance(preds, np.ndarray): preds = np.array(preds)
flags.DEFINE_bool( 'debug', False, 'If true, one training step is performed and ' 'the results are dumped to a folder for debugging.') flags.DEFINE_string('input_file', 'train', 'Input file name') flags.DEFINE_float('rotation_consistency_weight', 1e-3, 'Weight of rotation ' 'cycle consistency loss.') flags.DEFINE_float('translation_consistency_weight', 1e-2, 'Weight of ' 'thanslation consistency loss.') flags.DEFINE_integer('foreground_dilation', 8, 'Dilation of the foreground ' 'mask (in pixels).') flags.DEFINE_boolean('learn_intrinsics', True, 'Whether to learn camera ' 'intrinsics.') flags.DEFINE_boolean( 'boxify', True, 'Whether to convert segmentation masks to ' 'bounding boxes.') flags.DEFINE_string('imagenet_ckpt', None, 'Path to an imagenet checkpoint to ' 'intialize from.') FLAGS = flags.FLAGS flags.mark_flag_as_required('data_dir') flags.mark_flag_as_required('checkpoint_dir') def load(filename): with gfile.Open(filename) as f:
from covicas.db_plugin import Database from covicas.settings import settings from absl import app, flags FLAGS = flags.FLAGS flags.DEFINE_string("settings", "abcd.json", "Settings File tp Load From") flags.DEFINE_boolean("follow", False, "Follow the Log") def main(_): s = settings(FLAGS.settings) d = Database() d.display(follow=FLAGS.follow) if __name__ == "__main__": app.run(main)
from absl import logging import hashlib import io import zipfile import numpy as np import PIL.Image import tensorflow as tf from bert import tokenization flags.DEFINE_string('bert_vocab_file', 'data/bert/tf1.x/cased_L-12_H-768_A-12/vocab.txt', 'Path to the Bert vocabulary file.') flags.DEFINE_boolean('do_lower_case', False, 'To be passed to the bert tokenizer.') flags.DEFINE_string('annotations_jsonl_file', 'data/vcr1annots/val.jsonl', 'Path to the annotations file in jsonl format.') flags.DEFINE_string('annotations_jsonl_file_aug', 'data/vcr1annots/val.jsonl', 'Path to the annotations file in jsonl format.') flags.DEFINE_integer('num_shards', 10, 'Number of shards of the output tfrecord files.') flags.DEFINE_integer('shard_id', 0, 'Shard id of the current process.') flags.DEFINE_string('output_tfrecord_path', 'output/val.record', 'Path to the output tfrecord files.')
FLAGS = flags.FLAGS flags.DEFINE_string('input_dir', None, 'Directory with input data.') flags.DEFINE_string('sharding_file', None, 'File which stores information about desired sharding') flags.DEFINE_string('output_file_prefix', None, 'Prefix of the output file') flags.DEFINE_integer('num_shards', 0, 'Total number of output shards') flags.DEFINE_integer('max_records_to_process', -1, 'Maximum number of records to red from input files.') flags.DEFINE_boolean( 'fail_on_missing_examples', False, 'If true then pipeline will fail if any input example is ' 'missing from sharding data.') def read_sharding_data(): with tf.io.gfile.GFile(FLAGS.sharding_file) as f: lines = f.readlines() lines = [line.strip().split(',') for line in lines] return {line[0]: (int(line[1]), int(line[2])) for line in lines} class AugmentExampleWithShardAndNewLabelFn(beam.DoFn): """DoFn which augments examples with shard id and index within shard.""" def process(self, ex, sharding_data): filename = ex.features.feature['image/filename'].bytes_list.value[0] filename = filename.decode(
def define_ncf_flags(): """Add flags for running ncf_main.""" # Add common flags flags_core.define_base(export_dir=False) flags_core.define_performance(num_parallel_calls=False, inter_op=False, intra_op=False, synthetic_data=False, max_train_steps=False, dtype=False, all_reduce_alg=False) flags_core.define_device(tpu=True) flags_core.define_benchmark() flags.adopt_module_key_flags(flags_core) flags_core.set_defaults(model_dir="/tmp/ncf/", data_dir="/tmp/movielens-data/", train_epochs=2, batch_size=256, hooks="ProfilerHook", tpu=None) # Add ncf-specific flags flags.DEFINE_enum( name="dataset", default="ml-1m", enum_values=["ml-1m", "ml-20m"], case_sensitive=False, help=flags_core.help_wrap("Dataset to be trained and evaluated.")) flags.DEFINE_boolean( name="download_if_missing", default=True, help=flags_core.help_wrap( "Download data to data_dir if it is not already present.")) flags.DEFINE_string( name="eval_batch_size", default=None, help=flags_core.help_wrap( "The batch size used for evaluation. This should generally be larger" "than the training batch size as the lack of back propagation during" "evaluation can allow for larger batch sizes to fit in memory. If not" "specified, the training batch size (--batch_size) will be used.")) flags.DEFINE_integer( name="num_factors", default=8, help=flags_core.help_wrap("The Embedding size of MF model.")) # Set the default as a list of strings to be consistent with input arguments flags.DEFINE_list( name="layers", default=["64", "32", "16", "8"], help=flags_core.help_wrap( "The sizes of hidden layers for MLP. Example " "to specify different sizes of MLP layers: --layers=32,16,8,4")) flags.DEFINE_float( name="mf_regularization", default=0., help=flags_core.help_wrap( "The regularization factor for MF embeddings. The factor is used by " "regularizer which allows to apply penalties on layer parameters or " "layer activity during optimization.")) flags.DEFINE_list( name="mlp_regularization", default=["0.", "0.", "0.", "0."], help=flags_core.help_wrap( "The regularization factor for each MLP layer. See mf_regularization " "help for more info about regularization factor.")) flags.DEFINE_integer( name="num_neg", default=4, help=flags_core.help_wrap( "The Number of negative instances to pair with a positive instance." )) flags.DEFINE_float(name="learning_rate", default=0.001, help=flags_core.help_wrap("The learning rate.")) flags.DEFINE_float( name="hr_threshold", default=None, help=flags_core.help_wrap( "If passed, training will stop when the evaluation metric HR is " "greater than or equal to hr_threshold. For dataset ml-1m, the " "desired hr_threshold is 0.68 which is the result from the paper; " "For dataset ml-20m, the threshold can be set as 0.95 which is " "achieved by MLPerf implementation."))