from google.cloud import storage
flags.DEFINE_string('project', None,
'Google cloud project id for uploading the dataset.')
flags.DEFINE_string('gcs_output_path', None,
'GCS path for uploading the dataset.')
flags.DEFINE_string('local_scratch_dir', None,
'Scratch directory path for temporary files.')
flags.DEFINE_string(
'raw_data_dir', None, 'Directory path for raw Imagenet dataset. '
'Should have train and validation subdirectories inside it.')
flags.DEFINE_string('imagenet_username', None,
'Username for Imagenet.org account')
flags.DEFINE_string('imagenet_access_key', None,
'Access Key for Imagenet.org account')
flags.DEFINE_boolean('gcs_upload', True, 'Set to false to not upload to gcs.')
FLAGS = flags.FLAGS
BASE_URL = 'http://www.image-net.org/challenges/LSVRC/2012/nnoupb/'
LABELS_URL = 'https://raw.githubusercontent.com/tensorflow/models/master/research/inception/inception/data/imagenet_2012_validation_synset_labels.txt' # pylint: disable=line-too-long
TRAINING_FILE = 'ILSVRC2012_img_train.tar'
VALIDATION_FILE = 'ILSVRC2012_img_val.tar'
LABELS_FILE = 'synset_labels.txt'
TRAINING_SHARDS = 1024
VALIDATION_SHARDS = 128
TRAINING_DIRECTORY = 'train'
VALIDATION_DIRECTORY = 'validation'
flags.DEFINE_integer('replay_buffer_capacity', 1001,
'Replay buffer capacity per env.')
flags.DEFINE_integer('num_parallel_environments', 30,
'Number of environments to run in parallel')
flags.DEFINE_integer('num_environment_steps', 10000000,
'Number of environment steps to run before finishing.')
flags.DEFINE_integer('num_epochs', 25,
'Number of epochs for computing policy updates.')
flags.DEFINE_integer(
'collect_episodes_per_iteration', 30,
'The number of episodes to take in the environment before '
'each update. This is the total across all parallel '
'environments.')
flags.DEFINE_integer('num_eval_episodes', 30,
'The number of episodes to run eval on.')
flags.DEFINE_boolean('use_rnns', False,
'If true, use RNN for policy and value function.')
FLAGS = flags.FLAGS
@gin.configurable
def train_eval(
root_dir,
tf_master='',
env_name='HalfCheetah-v2',
env_load_fn=suite_mujoco.load,
random_seed=0,
# TODO(b/127576522): rename to policy_fc_layers.
actor_fc_layers=(200, 100),
value_fc_layers=(200, 100),
use_rnns=False,
# Params for collect
from non_semantic_speech_benchmark.eval_embedding.keras import get_data
from non_semantic_speech_benchmark.eval_embedding.keras import models
FLAGS = flags.FLAGS
flags.DEFINE_string('file_pattern', None, 'Dataset location.')
flags.DEFINE_string('en', None, 'Embedding name.')
flags.DEFINE_string('ed', None, 'Embedding dimension.')
flags.DEFINE_string('label_name', None, 'Name of label to use.')
flags.DEFINE_list('label_list', None, 'List of possible label values.')
flags.DEFINE_integer('batch_size', None, 'The number of images in each batch.')
flags.DEFINE_integer('tbs', None, 'not used')
flags.DEFINE_integer('nc', None, 'num_clusters')
flags.DEFINE_boolean('ubn', None, 'Whether to normalize')
flags.DEFINE_float('lr', None, 'not used')
flags.DEFINE_string('logdir', None,
'Directory where the model was written to.')
flags.DEFINE_string('eval_dir', None,
'Directory where the results are saved to.')
flags.DEFINE_integer('take_fixed_data', None,
'If not `None`, take a fixed number of data elements.')
flags.DEFINE_integer('timeout', 7200, 'Wait-for-checkpoint timeout.')
def eval_and_report():
"""Eval on voxceleb."""
logging.info('embedding_name: %s', FLAGS.en)
import os
import pickle
import mysql
import mysql.connector
from absl import app
from absl import flags
from absl import logging
from tqdm import tqdm
FLAGS = flags.FLAGS
flags.DEFINE_string('input', 'exp_out/location/run_8/disambiguation.tsv', '')
flags.DEFINE_string('uuidmap', 'data/location/uuid.pkl', '')
flags.DEFINE_boolean('create_tables', False, '')
flags.DEFINE_boolean('drop_tables', False, '')
logging.set_verbosity(logging.INFO)
def create_tables():
cnx_g = mysql.connector.connect(option_files=os.path.join(
os.environ['HOME'], '.mylogin.cnf'),
database='patent_20200630')
cnx_pg = mysql.connector.connect(option_files=os.path.join(
os.environ['HOME'], '.mylogin.cnf'),
database='pregrant_publications')
g_cursor = cnx_g.cursor()
g_cursor.execute(
'Name for eval.')
flags.DEFINE_integer(
'keep_checkpoint_max', 5,
'Maximum number of checkpoints to keep.')
flags.DEFINE_integer(
'keep_hub_module_max', 1,
'Maximum number of Hub modules to keep.')
flags.DEFINE_float(
'temperature', 0.1,
'Temperature parameter for contrastive loss.')
flags.DEFINE_boolean(
'hidden_norm', True,
'Temperature parameter for contrastive loss.')
flags.DEFINE_enum(
'head_proj_mode', 'nonlinear', ['none', 'linear', 'nonlinear'],
'How the head projection is done.')
flags.DEFINE_integer(
'head_proj_dim', 128,
'Number of head projection dimension.')
flags.DEFINE_integer(
'num_nlh_layers', 1,
'Number of non-linear head layers.')
flags.DEFINE_boolean(
"number of steps to take in validation")
flags.DEFINE_integer("batch_size", 32,
"batch size, num parallel streams to train on at once")
flags.DEFINE_integer("window_size", 20480,
"num frames to push into model at once")
flags.DEFINE_integer("timestep", 12, "the number of frames ahead to predict")
flags.DEFINE_integer("hidden_size", 512,
"the hidden layer size of the encoder")
flags.DEFINE_integer("out_size", 256, "the hidden layer size of the gru")
flags.DEFINE_integer("no_gru_layers", 1, "the number of layers in the gru")
flags.DEFINE_integer("val_every", None, "how often to perform validation")
flags.DEFINE_integer("save_every", None, "save every n steps")
flags.DEFINE_integer("log_every", 10, "append to log file every n steps")
flags.DEFINE_integer("log_tb_every", 50, "save tb scalars every n steps")
flags.DEFINE_boolean("dry_run", False, "dry run")
flags.mark_flag_as_required("train_data")
flags.mark_flag_as_required("val_data")
flags.mark_flag_as_required("expdir")
def validation(model, val_datastream, val_steps, features_in):
model.eval()
losses = []
accuracies = []
hidden = None
with FixedRandomState(42):
for step, val_batch in enumerate(val_datastream):
data = val_batch["data"].to(device)
if features_in == "raw":
_MOVE_SCREEN = actions.FUNCTIONS.Move_screen.id
_SELECT_ARMY = actions.FUNCTIONS.select_army.id
_SELECT_ALL = [0]
_NOT_QUEUED = [0]
step_mul = 8
FLAGS = flags.FLAGS
flags.DEFINE_string("map", "MoveToBeacon", "Name of a map to use to play.")
start_time = datetime.datetime.now().strftime("%Y%m%d%H%M")
flags.DEFINE_string("log", "tensorboard", "logging type(stdout, tensorboard)")
flags.DEFINE_string("algorithm", "deepq", "RL algorithm to use.")
flags.DEFINE_integer("timesteps", 2000000, "Steps to train")
flags.DEFINE_float("exploration_fraction", 0.2, "Exploration Fraction")
flags.DEFINE_boolean("prioritized", True, "prioritized_replay")
flags.DEFINE_boolean("dueling", True, "dueling")
flags.DEFINE_float("lr", 0.0005, "Learning rate")
flags.DEFINE_integer("num_agents", 4, "number of RL agents for A2C")
flags.DEFINE_integer("num_scripts", 4, "number of script agents for A2C")
flags.DEFINE_integer("nsteps", 20, "number of batch steps for A2C")
flags.DEFINE_string("experiment", "SCREEN_DIM=16", "name of experiment")
PROJ_DIR = os.path.dirname(os.path.abspath(__file__))
max_mean_reward = 0
last_filename = ""
start_time = datetime.datetime.now().strftime("%m%d%H%M")
SCREEN_DIM = 16
flags.DEFINE_integer('max_elements_per_user', 1000, 'Max number of training '
'sentences to use per user.')
flags.DEFINE_integer(
'num_validation_examples', 10000, 'Number of examples '
'to use from test set for per-round validation.')
# Modeling flags
flags.DEFINE_integer('vocab_size', 10000, 'Size of vocab to use.')
flags.DEFINE_integer('embedding_size', 96,
'Dimension of word embedding to use.')
flags.DEFINE_integer('latent_size', 670,
'Dimension of latent size to use in recurrent cell')
flags.DEFINE_integer('num_layers', 1,
'Number of stacked recurrent layers to use.')
flags.DEFINE_boolean(
'shared_embedding', False,
'Boolean indicating whether to tie input and output embeddings.')
FLAGS = flags.FLAGS
def main(argv):
if len(argv) > 1:
raise app.UsageError('Expected no command-line arguments, '
'got: {}'.format(argv))
tf.compat.v1.enable_v2_behavior()
model_builder = functools.partial(
models.create_recurrent_model,
vocab_size=FLAGS.vocab_size,
embedding_size=FLAGS.embedding_size,
def define_resnet_flags(resnet_size_choices=None):
"""Add flags and validators for ResNet."""
flags_core.define_base()
flags_core.define_performance(num_parallel_calls=False,
tf_gpu_thread_mode=True,
datasets_num_private_threads=True,
datasets_num_parallel_batches=True)
flags_core.define_image()
flags_core.define_benchmark()
flags.adopt_module_key_flags(flags_core)
flags.DEFINE_enum(
name='resnet_version',
short_name='rv',
default='1',
enum_values=['1', '2'],
help=flags_core.help_wrap(
'Version of ResNet. (1 or 2) See README.md for details.'))
flags.DEFINE_bool(
name='fine_tune',
short_name='ft',
default=False,
help=flags_core.help_wrap(
'If True do not train any parameters except for the final layer.'))
flags.DEFINE_string(
name='pretrained_model_checkpoint_path',
short_name='pmcp',
default=None,
help=flags_core.help_wrap(
'If not None initialize all the network except the final layer with '
'these values'))
flags.DEFINE_boolean(name='eval_only',
default=False,
help=flags_core.help_wrap(
'Skip training and only perform evaluation on '
'the latest checkpoint.'))
flags.DEFINE_boolean(
name='image_bytes_as_serving_input',
default=False,
help=flags_core.help_wrap(
'If True exports savedmodel with serving signature that accepts '
'JPEG image bytes instead of a fixed size [HxWxC] tensor that '
'represents the image. The former is easier to use for serving at '
'the expense of image resize/cropping being done as part of model '
'inference. Note, this flag only applies to ImageNet and cannot '
'be used for CIFAR.'))
flags.DEFINE_boolean(
name='use_train_and_evaluate',
default=False,
help=flags_core.help_wrap(
'If True, uses `tf.estimator.train_and_evaluate` for the training '
'and evaluation loop, instead of separate calls to `classifier.train '
'and `classifier.evaluate`, which is the default behavior.'))
flags.DEFINE_string(
name='worker_hosts',
default=None,
help=flags_core.help_wrap(
'Comma-separated list of worker ip:port pairs for running '
'multi-worker models with DistributionStrategy. The user would '
'start the program on each host with identical value for this flag.'
))
flags.DEFINE_integer(name='task_index',
default=-1,
help=flags_core.help_wrap(
'If multi-worker training, the task_index of '
'this worker.'))
flags.DEFINE_bool(name='enable_lars',
default=False,
help=flags_core.help_wrap(
'Enable LARS optimizer for large batch training.'))
flags.DEFINE_float(
name='label_smoothing',
default=0.0,
help=flags_core.help_wrap(
'Label smoothing parameter used in the softmax_cross_entropy'))
flags.DEFINE_float(name='weight_decay',
default=1e-4,
help=flags_core.help_wrap(
'Weight decay coefficiant for l2 regularization.'))
choice_kwargs = dict(
name='resnet_size',
short_name='rs',
default='50',
help=flags_core.help_wrap('The size of the ResNet model to use.'))
if resnet_size_choices is None:
flags.DEFINE_string(**choice_kwargs)
else:
flags.DEFINE_enum(enum_values=resnet_size_choices, **choice_kwargs)
from core.config import cfg
from tensorflow.python.saved_model import tag_constants
# helper files
import object_detection_helper as obj_helper
from threadedConsumer import Worker
import time
#endregion
#region flags
flags.DEFINE_string('framework', 'tf', '(tf, tflite, trt')
flags.DEFINE_string('weights', './checkpoints/yolov4-416',
'path to weights file')
flags.DEFINE_integer('size', 416, 'resize images to')
flags.DEFINE_boolean('tiny', False, 'yolo or yolo-tiny')
flags.DEFINE_string('model', 'yolov4', 'yolov3 or yolov4')
flags.DEFINE_string('video', './data/video/test.mp4',
'path to input video or set to 0 for webcam')
flags.DEFINE_float('iou', 0.45, 'iou threshold')
flags.DEFINE_float('score', 0.50, 'score threshold')
flags.DEFINE_boolean('dont_show', False, 'dont show video output')
flags.DEFINE_boolean('info', False, 'show detailed info of tracked objects')
flags.DEFINE_boolean('count', False, 'count objects being tracked on screen')
#endregion
# Definition of the parameters
max_cosine_distance = 0.4
nn_budget = None
interpreter = None
from absl import logging
import tensorflow as tf
from official.modeling import performance
from official.staging.training import controller
from official.utils.flags import core as flags_core
from official.utils.logs import logger
from official.utils.misc import distribution_utils
from official.utils.misc import keras_utils
from official.utils.misc import model_helpers
from official.vision.image_classification import common
from official.vision.image_classification import imagenet_preprocessing
from official.vision.image_classification import resnet_runnable
flags.DEFINE_boolean(name='use_tf_function',
default=True,
help='Wrap the train and test step inside a '
'tf.function.')
flags.DEFINE_boolean(name='single_l2_loss_op',
default=False,
help='Calculate L2_loss on concatenated weights, '
'instead of using Keras per-layer L2 loss.')
def build_stats(runnable, time_callback):
"""Normalizes and returns dictionary of stats.
Args:
runnable: The module containing all the training and evaluation metrics.
time_callback: Time tracking callback instance.
Returns:
def define_common_bert_flags():
"""Define common flags for BERT tasks."""
flags_core.define_base(data_dir=False,
model_dir=True,
clean=False,
train_epochs=False,
epochs_between_evals=False,
stop_threshold=False,
batch_size=False,
num_gpu=True,
hooks=False,
export_dir=False,
distribution_strategy=True,
run_eagerly=True)
flags_core.define_distribution()
flags.DEFINE_string('bert_config_file', None,
'Bert configuration file to define core bert layers.')
flags.DEFINE_string(
'model_export_path', None,
'Path to the directory, where trainined model will be '
'exported.')
flags.DEFINE_string('tpu', '', 'TPU address to connect to.')
flags.DEFINE_string(
'init_checkpoint', None,
'Initial checkpoint (usually from a pre-trained BERT model).')
flags.DEFINE_integer('num_train_epochs', 3,
'Total number of training epochs to perform.')
flags.DEFINE_integer(
'steps_per_loop', 200,
'Number of steps per graph-mode loop. Only training step '
'happens inside the loop. Callbacks will not be called '
'inside.')
flags.DEFINE_float('learning_rate', 5e-5,
'The initial learning rate for Adam.')
flags.DEFINE_boolean(
'scale_loss', False,
'Whether to divide the loss by number of replica inside the per-replica '
'loss function.')
flags.DEFINE_boolean(
'use_keras_compile_fit', False,
'If True, uses Keras compile/fit() API for training logic. Otherwise '
'use custom training loop.')
flags.DEFINE_string(
'hub_module_url', None, 'TF-Hub path/url to Bert module. '
'If specified, init_checkpoint flag should not be used.')
flags.DEFINE_enum(
'model_type', 'bert', ['bert', 'albert'],
'Specifies the type of the model. '
'If "bert", will use canonical BERT; if "albert", will use ALBERT model.'
)
flags.DEFINE_bool(
'hub_module_trainable', True,
'True to make keras layers in the hub module trainable.')
# Adds flags for mixed precision and multi-worker training.
flags_core.define_performance(
num_parallel_calls=False,
inter_op=False,
intra_op=False,
synthetic_data=False,
max_train_steps=False,
dtype=True,
dynamic_loss_scale=True,
loss_scale=True,
all_reduce_alg=True,
num_packs=False,
tf_gpu_thread_mode=True,
datasets_num_private_threads=True,
enable_xla=True,
fp16_implementation=True,
)
"Dropout probability for attention probabilities.")
flags.DEFINE_integer("max_positions", 256,
"Max text sequence length model might be used with.")
flags.DEFINE_integer("max_segments", 8,
"Max segment types model might be used with.")
flags.DEFINE_integer("max_conditions", 64,
"Max condition sequence length model might be used with.")
flags.DEFINE_integer("max_image_regions", 128,
"Max encoded image regions model might be used with.")
# Options for training the conditional text planner.
flags.DEFINE_boolean("conditional_decoding", False, "Use conditional decoding.")
flags.DEFINE_float("span_sample_p", 0.3, "Geometric distribtion parameter.")
flags.DEFINE_integer("span_length", 10, "Max random span length selected.")
flags.DEFINE_integer("condition_length", 40,
"Max conditioning sequence length.")
flags.DEFINE_integer("question_length", 30, "Max question length.")
flags.DEFINE_integer("answer_length", 10, "Max answer length.")
# Options for decoding.
flags.DEFINE_integer("decode_length", 30, "Max decoding length.")
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Rewrite script for TF->JAX."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import collections
# Dependency imports
from absl import app
from absl import flags
flags.DEFINE_boolean('numpy_to_jax', False,
'Whether or not to rewrite numpy imports to jax.numpy')
flags.DEFINE_list('omit_deps', [], 'List of build deps being omitted.')
FLAGS = flags.FLAGS
TF_REPLACEMENTS = {
'import tensorflow ':
'from tensorflow_probability.python.internal.backend import numpy ',
'import tensorflow.compat.v1':
'from tensorflow_probability.python.internal.backend.numpy.compat '
'import v1',
'import tensorflow.compat.v2':
'from tensorflow_probability.python.internal.backend.numpy.compat '
'import v2',
'import tensorflow_probability as tfp':
'import tensorflow_probability as tfp; '
# from utils.dataset_util import *
# from utils.label_map_util import *
import utils.dataset_util as dataset_util
import utils.label_map_util as label_map_util
flags.DEFINE_string('data_dir', '', 'Root directory to raw PASCAL VOC dataset.')
flags.DEFINE_string('set', 'train', 'Convert training set, validation set or '
'merged set.')
flags.DEFINE_string('annotations_dir', 'Annotations',
'(Relative) path to annotations directory.')
flags.DEFINE_string('year', 'VOC2007', 'Desired challenge year.')
# flags.DEFINE_string('project', '', 'Desired project name.')
flags.DEFINE_string('output_path', '', 'Path to output TFRecord')
flags.DEFINE_string('label_map_path', '',
'Path to label map proto') # './data/pascal_label_map.pbtxt'
flags.DEFINE_boolean('ignore_difficult_instances', False, 'Whether to ignore '
'difficult instances')
flags.DEFINE_string('action', 'tfrecord', 'Action in [tfrecord, imageset]')
# flags.DEFINE_string('imageset', 'image', 'image set name')
FLAGS = flags.FLAGS
SETS = ['train', 'val', 'trainval', 'test']
# YEARS = ['VOC2007', 'VOC2012', 'merged']
def gen_image_set(data_dir, year):
"""generate image set text file from annotation xmls
Args:
import time
from absl import app, flags, logging
from absl.flags import FLAGS
import cv2
import numpy as np
import tensorflow as tf
from yolov3_tf2.models import (YoloV3, YoloV3Tiny)
from yolov3_tf2.dataset import transform_images, load_tfrecord_dataset
from yolov3_tf2.utils import draw_outputs
# flags.DEFINE_string('classes', './data/voc2012.names', 'path to classes file')
flags.DEFINE_string('classes', './data/coco.names', 'path to classes file')
flags.DEFINE_string('weights', './checkpoints/yolov3_train_15.tf',
'path to weights file')
flags.DEFINE_boolean('tiny', False, 'yolov3 or yolov3-tiny')
flags.DEFINE_integer('size', 416, 'resize images to')
flags.DEFINE_string('image', './data/street.jpg', 'path to input image')
flags.DEFINE_string('tfrecord', None, 'tfrecord instead of image')
flags.DEFINE_string('output', './output.jpg', 'path to output image')
flags.DEFINE_integer('num_classes', 80, 'number of classes in the model')
def main(_argv):
physical_devices = tf.config.experimental.list_physical_devices('GPU')
for physical_device in physical_devices:
tf.config.experimental.set_memory_growth(physical_device, True)
if FLAGS.tiny:
yolo = YoloV3Tiny(classes=FLAGS.num_classes)
else:
yolo = YoloV3(classes=FLAGS.num_classes)
'any block group.'))
flags.DEFINE_float(
'dropblock_keep_prob',
default=None,
help=('keep_prob parameter of DropBlock. Will not be used if '
'dropblock_groups is empty.'))
flags.DEFINE_integer(
'dropblock_size',
default=None,
help=('size parameter of DropBlock. Will not be used if dropblock_groups '
'is empty.'))
# copybara:strip_begin
flags.DEFINE_boolean(
'xla_compile',
default=False,
help=('Compile computation with XLA, this flag has no effect when running '
'on TPU.'))
flags.DEFINE_string(
'tpu_job_name', None,
'Name of TPU worker binary. Only necessary if job name is changed from'
' default tpu_worker.')
# copybara:strip_end
flags.DEFINE_integer(
'profile_every_n_steps',
default=0,
help=('Number of steps between collecting profiles if larger than 0'))
flags.DEFINE_string(
'mode',
"save to checkpoints.")
flags.DEFINE_string("result_dir", "results", "Path to results")
# Dataset
flags.DEFINE_string(
"dataset_name", "tieredImageNet", "Name of the dataset to "
"train on, which will be mapped to data.MetaDataset.")
flags.DEFINE_integer("num_steps_limit", int(1e5),
"Number of steps to train for.")
flags.DEFINE_integer(
"checkpoint_steps", 200, "The frequency, in number of "
"steps, of saving the checkpoints.")
#
flags.DEFINE_string("exp_name", "experiment", "Name of the experiment.")
flags.DEFINE_string("gpus", "0", "Ids of GPUs where the program run.")
flags.DEFINE_boolean("evaluation_mode", False, "Whether to run in an "
"evaluation-only mode.")
flags.DEFINE_string("eval_set", "val",
"Which set (train/val/test) to evaluate on.")
flags.DEFINE_integer("random_seed", 0, "Global random seed.")
def main(argv):
del argv # Unused.
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpus
tf.compat.v1.set_random_seed(FLAGS.random_seed)
np.random.seed(FLAGS.random_seed)
if FLAGS.model_cls == 'metafun_classifier':
model_cls = MetaFunClassifier
task_type = "classification"
import experiments.hyperparam_search.meta_mll_hyperparm as meta_mll_hparam
import experiments.hyperparam_search.meta_svgd_hyperparam as meta_svgd_hparam
import experiments.hyperparam_search.meta_vi_hyperparam as meta_vi_hparam
import experiments.hyperparam_search.meta_mlap_hyperparam as meta_pac_hyperparam
from absl import flags
from absl import app
flags.DEFINE_integer('n_cpus', default=32, help='number of cpus to use')
flags.DEFINE_integer('n_gpus', default=1, help='number of gpus to use')
flags.DEFINE_string('algos', default='pac', help='specifies which dataset to use')
flags.DEFINE_string('metric', default='test_ll', help='specifies which metric to optimize')
flags.DEFINE_boolean('cluster', default=False, help='whether to submit jobs with bsub')
flags.DEFINE_boolean('dry', default=False, help='whether to only print launch commands')
flags.DEFINE_boolean('load_analysis', default=False, help='whether to load the analysis from existing dirs')
flags.DEFINE_boolean('resume', default=False, help='whether to resume checkpointed tune session')
FLAGS = flags.FLAGS
algo_map_dict = {'map': meta_mll_hparam,
'vi': meta_vi_hparam,
'svgd': meta_svgd_hparam,
'pac': meta_pac_hyperparam}
def main(argv):
hparam_search_modules = [algo_map_dict[algo_str] for algo_str in FLAGS.algos.split(',')]
from .eval_util import compute_errors
from ..datasets.common import read_images_from_tfrecords
kPredDir = '/tmp/hmr_output'
# Change to where you saved your tfrecords
kTFDataDir = '/scratch1/williamljb/hmr_multiview/tf_datasets/human36m_multi'
flags.DEFINE_string('pred_dir', kPredDir,
'where to save model output of h36m')
flags.DEFINE_string('tfh36m_dir', kTFDataDir,
'data dir: top of h36m in tf_records')
flags.DEFINE_integer(
'protocol', 1,
'If 2, then only frontal cam (3) and trial 1, if 1, then all camera & trials'
)
flags.DEFINE_boolean(
'vis', False, 'If true, visualizes the best and worst 30 results.')
model = None
sess = None
# For visualization.
renderer = None
extreme_errors, contents = [], []
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
# -- Draw Utils ---
def draw_content(content, config):
global renderer
input_img = content['image']
vert = content['vert']
flags.DEFINE_integer('iterations_per_loop', 100,
'Number of iterations per TPU training loop.')
# For mode=train_and_eval, evaluation occurs after training is finished.
# Note: independently of steps_per_checkpoint, estimator will save the most
# recent checkpoint every 10 minutes by default for train_and_eval
flags.DEFINE_string('mode', 'train', 'Mode to run: train, eval')
flags.DEFINE_integer(
'train_batch_size', None, 'Batch size for training. If '
'this is not provided, batch size is read from training '
'config.')
flags.DEFINE_string(
'hparams_overrides', None, 'Comma-separated list of '
'hyperparameters to override defaults.')
flags.DEFINE_integer('num_train_steps', None, 'Number of train steps.')
flags.DEFINE_boolean('eval_training_data', False,
'If training data should be evaluated for this job.')
flags.DEFINE_integer(
'sample_1_of_n_eval_examples', 1, 'Will sample one of '
'every n eval input examples, where n is provided.')
flags.DEFINE_integer(
'sample_1_of_n_eval_on_train_examples', 5, 'Will sample '
'one of every n train input examples for evaluation, '
'where n is provided. This is only used if '
'`eval_training_data` is True.')
flags.DEFINE_string(
'model_dir', None, 'Path to output model directory '
'where event and checkpoint files will be written.')
flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config '
'file.')
FLAGS = tf.flags.FLAGS
import numpy as np
import numpy.random as npr
import tensorflow as tf
from tensorflow_privacy.privacy.analysis import privacy_ledger
from tensorflow_privacy.privacy.analysis.rdp_accountant import compute_rdp_from_ledger
from tensorflow_privacy.privacy.analysis.rdp_accountant import get_privacy_spent
from tensorflow_privacy.privacy.optimizers import dp_optimizer
AdamOptimizer = tf.compat.v1.train.AdamOptimizer
FLAGS = flags.FLAGS
flags.DEFINE_boolean(
'dpsgd', True, 'If True, train with DP-SGD. If False, '
'train with vanilla SGD.')
flags.DEFINE_float('learning_rate', .05, 'Learning rate for training')
flags.DEFINE_float('noise_multiplier', 2.0,
'Ratio of the standard deviation to the clipping norm')
flags.DEFINE_float('l2_norm_clip', 1.0, 'Clipping norm')
flags.DEFINE_integer('batch_size', 64, 'Batch size')
flags.DEFINE_integer('epochs', 2, 'Number of epochs')
flags.DEFINE_integer('training_data_size', 2000, 'Training data size')
flags.DEFINE_integer('test_data_size', 2000, 'Test data size')
flags.DEFINE_integer('input_dimension', 5, 'Input dimension')
flags.DEFINE_string('model_dir', None, 'Model directory')
class EpsilonPrintingTrainingHook(tf.estimator.SessionRunHook):
"""Training hook to print current value of epsilon after an epoch."""
flags.DEFINE_string('pipeline_config_path',
'training/config/ppn_pipeline.config',
'Path to pipeline config '
'file.')
flags.DEFINE_integer('num_train_steps', 5000, 'Number of train steps.')
flags.DEFINE_integer('num_eval_steps', 5, 'Number of train steps.')
flags.DEFINE_string(
'hparams_overrides', None, 'Hyperparameter overrides, '
'represented as a string containing comma-separated '
'hparam_name=value pairs.')
flags.DEFINE_string(
'checkpoint_dir', None, 'Path to directory holding a checkpoint. If '
'`checkpoint_dir` is provided, this binary operates in eval-only mode, '
'writing resulting metrics to `model_dir`.')
flags.DEFINE_boolean(
'run_once', False, 'If running in eval-only mode, whether to run just '
'one round of eval vs running continuously (default).')
flags.DEFINE_boolean('eval_training_data', True,
'If training data should be evaluated for this job.')
FLAGS = flags.FLAGS
def main(unused_argv):
#flags.mark_flag_as_required('model_dir')
#flags.mark_flag_as_required('pipeline_config_path')
config = tf.estimator.RunConfig(model_dir=FLAGS.model_dir,
save_checkpoints_steps=100)
train_and_eval_dict = model_lib.create_estimator_and_inputs(
run_config=config,
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
"""
import logging
from typing import Iterable, Callable
from copy import deepcopy
import time
from absl import flags
from pywinauto import Application, timings
from pywinauto.base_wrapper import BaseWrapper
from pywinauto.findwindows import ElementNotFoundError
from pywinauto.keyboard import send_keys
flags.DEFINE_boolean('dev_mode', False, 'Dev mode - expect Orbit MainWindow to be opened, '
'and do not close Orbit at the end')
class OrbitE2EError(RuntimeError):
pass
class E2ETestCase:
"""
Encapsulates a single test executed as part of a E2E Test Suite.
Inherit from this class to define a new test, and pass an instance of your test in the E2ETestSuite constructor
to have it executed.
All named arguments passed to the constructor of your test will be forwarded to the _execute method. To create
a parameterized test, simply inherit from this class and provide your own _execute methods with any number of
named arguments.
flags.DEFINE_string("input_graph", None, "input graph")
flags.DEFINE_string("output_graph", None, "input graph")
flags.DEFINE_string("config", None, "LPOT config file")
flags.DEFINE_float("conf_threshold", 0.5, "confidence threshold")
flags.DEFINE_float("iou_threshold", 0.4, "IoU threshold")
flags.DEFINE_integer("num_intra_threads", 0, "number of intra threads")
flags.DEFINE_integer("num_inter_threads", 1, "number of inter threads")
flags.DEFINE_boolean("benchmark", False, "benchmark mode")
flags.DEFINE_boolean("profiling", False, "Signal of profiling")
FLAGS = flags.FLAGS
class NMS():
def __init__(self, conf_threshold, iou_threshold):
self.conf_threshold = conf_threshold
self.iou_threshold = iou_threshold
def __call__(self, sample):
preds, labels = sample
if not isinstance(preds, np.ndarray):
preds = np.array(preds)
flags.DEFINE_bool(
'debug', False, 'If true, one training step is performed and '
'the results are dumped to a folder for debugging.')
flags.DEFINE_string('input_file', 'train', 'Input file name')
flags.DEFINE_float('rotation_consistency_weight', 1e-3, 'Weight of rotation '
'cycle consistency loss.')
flags.DEFINE_float('translation_consistency_weight', 1e-2, 'Weight of '
'thanslation consistency loss.')
flags.DEFINE_integer('foreground_dilation', 8, 'Dilation of the foreground '
'mask (in pixels).')
flags.DEFINE_boolean('learn_intrinsics', True, 'Whether to learn camera '
'intrinsics.')
flags.DEFINE_boolean(
'boxify', True, 'Whether to convert segmentation masks to '
'bounding boxes.')
flags.DEFINE_string('imagenet_ckpt', None, 'Path to an imagenet checkpoint to '
'intialize from.')
FLAGS = flags.FLAGS
flags.mark_flag_as_required('data_dir')
flags.mark_flag_as_required('checkpoint_dir')
def load(filename):
with gfile.Open(filename) as f:
from covicas.db_plugin import Database
from covicas.settings import settings
from absl import app, flags
FLAGS = flags.FLAGS
flags.DEFINE_string("settings", "abcd.json", "Settings File tp Load From")
flags.DEFINE_boolean("follow", False, "Follow the Log")
def main(_):
s = settings(FLAGS.settings)
d = Database()
d.display(follow=FLAGS.follow)
if __name__ == "__main__":
app.run(main)
from absl import logging
import hashlib
import io
import zipfile
import numpy as np
import PIL.Image
import tensorflow as tf
from bert import tokenization
flags.DEFINE_string('bert_vocab_file',
'data/bert/tf1.x/cased_L-12_H-768_A-12/vocab.txt',
'Path to the Bert vocabulary file.')
flags.DEFINE_boolean('do_lower_case', False,
'To be passed to the bert tokenizer.')
flags.DEFINE_string('annotations_jsonl_file', 'data/vcr1annots/val.jsonl',
'Path to the annotations file in jsonl format.')
flags.DEFINE_string('annotations_jsonl_file_aug', 'data/vcr1annots/val.jsonl',
'Path to the annotations file in jsonl format.')
flags.DEFINE_integer('num_shards', 10,
'Number of shards of the output tfrecord files.')
flags.DEFINE_integer('shard_id', 0, 'Shard id of the current process.')
flags.DEFINE_string('output_tfrecord_path', 'output/val.record',
'Path to the output tfrecord files.')
FLAGS = flags.FLAGS
flags.DEFINE_string('input_dir', None, 'Directory with input data.')
flags.DEFINE_string('sharding_file', None,
'File which stores information about desired sharding')
flags.DEFINE_string('output_file_prefix', None, 'Prefix of the output file')
flags.DEFINE_integer('num_shards', 0, 'Total number of output shards')
flags.DEFINE_integer('max_records_to_process', -1,
'Maximum number of records to red from input files.')
flags.DEFINE_boolean(
'fail_on_missing_examples', False,
'If true then pipeline will fail if any input example is '
'missing from sharding data.')
def read_sharding_data():
with tf.io.gfile.GFile(FLAGS.sharding_file) as f:
lines = f.readlines()
lines = [line.strip().split(',') for line in lines]
return {line[0]: (int(line[1]), int(line[2])) for line in lines}
class AugmentExampleWithShardAndNewLabelFn(beam.DoFn):
"""DoFn which augments examples with shard id and index within shard."""
def process(self, ex, sharding_data):
filename = ex.features.feature['image/filename'].bytes_list.value[0]
filename = filename.decode(
def define_ncf_flags():
"""Add flags for running ncf_main."""
# Add common flags
flags_core.define_base(export_dir=False)
flags_core.define_performance(num_parallel_calls=False,
inter_op=False,
intra_op=False,
synthetic_data=False,
max_train_steps=False,
dtype=False,
all_reduce_alg=False)
flags_core.define_device(tpu=True)
flags_core.define_benchmark()
flags.adopt_module_key_flags(flags_core)
flags_core.set_defaults(model_dir="/tmp/ncf/",
data_dir="/tmp/movielens-data/",
train_epochs=2,
batch_size=256,
hooks="ProfilerHook",
tpu=None)
# Add ncf-specific flags
flags.DEFINE_enum(
name="dataset",
default="ml-1m",
enum_values=["ml-1m", "ml-20m"],
case_sensitive=False,
help=flags_core.help_wrap("Dataset to be trained and evaluated."))
flags.DEFINE_boolean(
name="download_if_missing",
default=True,
help=flags_core.help_wrap(
"Download data to data_dir if it is not already present."))
flags.DEFINE_string(
name="eval_batch_size",
default=None,
help=flags_core.help_wrap(
"The batch size used for evaluation. This should generally be larger"
"than the training batch size as the lack of back propagation during"
"evaluation can allow for larger batch sizes to fit in memory. If not"
"specified, the training batch size (--batch_size) will be used."))
flags.DEFINE_integer(
name="num_factors",
default=8,
help=flags_core.help_wrap("The Embedding size of MF model."))
# Set the default as a list of strings to be consistent with input arguments
flags.DEFINE_list(
name="layers",
default=["64", "32", "16", "8"],
help=flags_core.help_wrap(
"The sizes of hidden layers for MLP. Example "
"to specify different sizes of MLP layers: --layers=32,16,8,4"))
flags.DEFINE_float(
name="mf_regularization",
default=0.,
help=flags_core.help_wrap(
"The regularization factor for MF embeddings. The factor is used by "
"regularizer which allows to apply penalties on layer parameters or "
"layer activity during optimization."))
flags.DEFINE_list(
name="mlp_regularization",
default=["0.", "0.", "0.", "0."],
help=flags_core.help_wrap(
"The regularization factor for each MLP layer. See mf_regularization "
"help for more info about regularization factor."))
flags.DEFINE_integer(
name="num_neg",
default=4,
help=flags_core.help_wrap(
"The Number of negative instances to pair with a positive instance."
))
flags.DEFINE_float(name="learning_rate",
default=0.001,
help=flags_core.help_wrap("The learning rate."))
flags.DEFINE_float(
name="hr_threshold",
default=None,
help=flags_core.help_wrap(
"If passed, training will stop when the evaluation metric HR is "
"greater than or equal to hr_threshold. For dataset ml-1m, the "
"desired hr_threshold is 0.68 which is the result from the paper; "
"For dataset ml-20m, the threshold can be set as 0.95 which is "
"achieved by MLPerf implementation."))
