self.fastmath = template_vars.ValueListVar(['', '-ffast-math'])
self.native = template_vars.ValueListVar(['', '-march=native'])
# Returns a tuple (cmdline, output_filename) -- for indexing purposes
def get_cmdline(self, input_path, input_filename, additional_flags):
# file.cpp -> file_RANDOM.ll
output_filename = (input_filename[:-4] + '_' +
template_vars.RandomStrVar()[0] + '.ll')
args = [self.compiler, self.optimization, self.fastmath, self.native]
arg_strs = [str(random.choice(arg)) for arg in args]
return (' '.join(arg_strs) + ' ' + input_path+'/'+input_filename +
' '+ additional_flags + ' -o ', output_filename)
flags.DEFINE_string('input_path', None, 'Input path of rendered .cpp files')
flags.DEFINE_string('output_path', None, 'Output path to store .ll files')
flags.DEFINE_string('compile_one', None, 'Define a single template to compile')
flags.DEFINE_integer('ir_per_file', 32, 'Number of .ll files generated per input')
flags.DEFINE_string('compiler_flags', '', 'Additional compiler flags')
FLAGS = flags.FLAGS
def _createdir(dname):
try:
os.makedirs(dname)
except FileExistsError:
pass
def main(argv):
model_dir = osp.join(curr_path, '..', 'models')
if not osp.exists(model_dir):
print('Fix path to models/')
import ipdb
ipdb.set_trace()
SMPL_MODEL_PATH = osp.join(model_dir, 'neutral_smpl_with_cocoplus_reg.pkl')
SMPL_FACE_PATH = osp.join(curr_path, '../src/tf_smpl', 'smpl_faces.npy')
DATASET_SIZES = {'vlog': {'all': 4122, 'cropped_keypoint': 1807, 'uncropped_keypoint': 1807}, \
'cross_task': {'all': 3908, 'cropped_keypoint': 1951, 'uncropped_keypoint': 1951}, \
'instructions': {'all': 2258, 'cropped_keypoint': 829, 'uncropped_keypoint': 829}, \
'youcook': {'all': 3370, 'cropped_keypoint': 1583, 'uncropped_keypoint': 1583}}
# Default pre-trained model path for the demo.
PRETRAINED_MODEL = osp.join(model_dir, 'ours/model.ckpt-694216')
flags.DEFINE_string('smpl_model_path', SMPL_MODEL_PATH,
'path to the neurtral smpl model')
flags.DEFINE_string('smpl_face_path', SMPL_FACE_PATH,
'path to smpl mesh faces (for easy rendering)')
flags.DEFINE_string('load_path', None, 'run from this checkpoint')
flags.DEFINE_integer('batch_size', 1,
'Input image size to the network after preprocessing')
flags.DEFINE_string('gpu', '0', 'GPU to use for training / testing')
# Don't change if testing:
flags.DEFINE_integer('img_size', 224,
'Input image size to the network after preprocessing')
flags.DEFINE_string('data_format', 'NHWC', 'Data format')
flags.DEFINE_integer('num_stage', 3, '# of times to iterate regressor')
flags.DEFINE_string('model_type', 'resnet_fc3_dropout',
'Specifies which network to use')
flags.DEFINE_string(
from absl import flags
from a1_beacon_agent import BeaconAgent as Agent
FLAGS = flags.FLAGS
flags.DEFINE_bool("render", True, "Whether to render with pygame.")
flags.DEFINE_integer("screen_resolution", 84,
"Resolution for screen feature layers.")
flags.DEFINE_integer("minimap_resolution", 64,
"Resolution for minimap feature layers.")
flags.DEFINE_integer("max_agent_steps", 2500, "Total agent steps.")
flags.DEFINE_integer("game_steps_per_episode", 0, "Game steps per episode.")
flags.DEFINE_integer("step_mul", 1, "Game steps per agent step.")
flags.DEFINE_string("agent", "pysc2.agents.random_agent.RandomAgent",
"Which agent to run")
flags.DEFINE_enum("agent_race", None, sc2_env.Race._member_names_,
"Agent's race.")
flags.DEFINE_enum("bot_race", None, sc2_env.Race._member_names_, "Bot's race.")
flags.DEFINE_enum("difficulty", None, sc2_env.Difficulty._member_names_,
"Bot's strength.")
flags.DEFINE_bool("profile", False, "Whether to turn on code profiling.")
flags.DEFINE_bool("trace", False, "Whether to trace the code execution.")
flags.DEFINE_integer("parallel", 1, "How many instances to run in parallel.")
flags.DEFINE_bool("save_replay", False, "Whether to save a replay at the end.")
flags.DEFINE_string("map", "MoveToBeacon", "Name of a map to use.")
flags.mark_flag_as_required("map")
from dataloader import input_reader
from dataloader import mode_keys as ModeKeys
from executor import tpu_executor
from modeling import model_builder
import sys
sys.path.insert(0, 'tpu/models')
from hyperparameters import common_hparams_flags
from hyperparameters import common_tpu_flags
from hyperparameters import params_dict
common_tpu_flags.define_common_tpu_flags()
common_hparams_flags.define_common_hparams_flags()
flags.DEFINE_string(
'mode', default='train',
help='Mode to run: `train`, `eval` or `train_and_eval`.')
flags.DEFINE_string(
'model', default='retinanet',
help='Model to run: `retinanet` or `shapemask`.')
flags.DEFINE_integer(
'num_cores', default=8, help='Number of TPU cores for training.')
flags.DEFINE_string(
'tpu_job_name', None,
'Name of TPU worker binary. Only necessary if job name is changed from'
' default tpu_worker.')
FLAGS = flags.FLAGS
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
from absl import app
from absl import flags
from open_spiel.python.algorithms import exploitability
from open_spiel.python.algorithms import fictitious_play
import pyspiel
FLAGS = flags.FLAGS
flags.DEFINE_integer("iterations", 100, "Number of iterations")
flags.DEFINE_string("game", "leduc_poker", "Name of the game")
flags.DEFINE_integer("players", 2, "Number of players")
flags.DEFINE_integer("print_freq", 10, "How often to print the exploitability")
def main(_):
game = pyspiel.load_game(FLAGS.game, {"players": FLAGS.players})
xfp_solver = fictitious_play.XFPSolver(game)
for i in range(FLAGS.iterations):
xfp_solver.iteration()
conv = exploitability.exploitability(game, xfp_solver.average_policy())
if i % FLAGS.print_freq == 0:
print("Iteration: {} Conv: {}".format(i, conv))
sys.stdout.flush()
from absl import app, flags, logging
import tfne
flags.DEFINE_integer(
'logging_level',
default=None,
help=
'Integer parameter specifying the verbosity of the absl logging library')
flags.DEFINE_string(
'config_file',
default=None,
help=
'String parameter specifying the file path to the configuration file used for '
'the TFNE evolutionary process')
flags.DEFINE_string(
'backup_dir',
default=None,
help=
'String parameter specifying the directory path to where the TFNE state backups '
'should be saved to')
flags.DEFINE_integer(
'max_generations',
default=None,
help=
'Integer parameter specifying the intended maximum number of generations the '
'population should be evolved')
flags.DEFINE_float(
'max_fitness',
default=None,
help=
flags.DEFINE_integer(
SYSBENCH_RUN_SECONDS, 480,
'The duration, in seconds, of each run phase with varying'
'thread count.')
flags.DEFINE_integer(
SYSBENCH_WARMUP_SECONDS, 0,
'The duration, in seconds, of the warmup run in which '
'results are discarded.')
flags.DEFINE_list(THREAD_COUNT_LIST, [1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
'The number of test threads on the client side.')
flags.DEFINE_integer(
SYSBENCH_REPORT_INTERVAL, 1,
'The interval, in seconds, we ask sysbench to report '
'results.')
flags.DEFINE_string(
RUN_URI, None, 'Run identifier, if provided, only run phase '
'will be completed.')
flags.DEFINE_string(
RUN_STAGE, None, 'List of phases to be executed. For example:'
'"--run_uri=provision,prepare". Available phases:'
'prepare, provision, run, cleanup, teardown.')
flags.DEFINE_string(GCE_BOOT_DISK_SIZE, '1000',
'The boot disk size in GB for GCP VMs..')
flags.DEFINE_string(GCE_BOOT_DISK_TYPE, 'pd-ssd',
'The boot disk type for GCP VMs.')
flags.DEFINE_string(MACHINE_TYPE, 'n1-standard-16',
'Machine type for GCE Virtual machines.')
flags.DEFINE_enum(
MYSQL_SVC_DB_INSTANCE_CORES, '4', ['1', '4', '8', '16'],
'The number of cores to be provisioned for the DB instance.')
flags.DEFINE_string(MYSQL_SVC_OLTP_TABLES_COUNT, '4',
import launchpad as lp
import sonnet as snt
from absl import app, flags
from launchpad.nodes.python.local_multi_processing import PythonProcess
from mava.components.tf import architectures
from mava.systems.tf import mad4pg
from mava.utils import lp_utils
from mava.utils.environments import debugging_utils
from mava.utils.loggers import logger_utils
FLAGS = flags.FLAGS
flags.DEFINE_string(
"env_name",
"simple_spread",
"Debugging environment name (str).",
)
flags.DEFINE_string(
"action_space",
"continuous",
"Environment action space type (str).",
)
flags.DEFINE_string(
"mava_id",
str(datetime.now()),
"Experiment identifier that can be used to continue experiments.",
)
flags.DEFINE_string("base_dir", "~/mava", "Base dir to store experiments.")
# ==============================================================================
"""Binary to run train and evaluation on object detection model."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from absl import flags
import tensorflow.compat.v1 as tf
from object_detection import model_hparams
from object_detection import model_lib
flags.DEFINE_string(
'model_dir', None, 'Path to output model directory '
'where event and checkpoint files will be written.')
flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config '
'file.')
flags.DEFINE_integer('num_train_steps', None, 'Number of train steps.')
flags.DEFINE_integer('num_eval_steps', None, 'Number of eval steps.')
flags.DEFINE_integer('eval_throttle_secs', 600, 'evaluate every N seconds')
flags.DEFINE_integer('eval_start_delay_secs', 120, 'start evaluating after N seconds')
flags.DEFINE_boolean('eval_training_data', False,
'If training data should be evaluated for this job. Note '
'that one call only use this in eval-only mode, and '
'`checkpoint_dir` must be supplied.')
flags.DEFINE_integer('sample_1_of_n_eval_examples', 1, 'Will sample one of '
'every n eval input examples, where n is provided.')
flags.DEFINE_integer('sample_1_of_n_eval_on_train_examples', 5, 'Will sample '
'one of every n train input examples for evaluation, '
"""Flags used by uflow training and evaluation."""
from absl import flags
FLAGS = flags.FLAGS
step = 0 #a step parameter to be used across scripts
# General flags.
flags.DEFINE_bool(
'no_tf_function', False, 'If True, run without'
' tf functions. This incurs a performance hit, but can'
' make debugging easier.')
flags.DEFINE_string('train_on', '',
'"format0:path0;format1:path1", e.g. "kitti:/usr/..."')
flags.DEFINE_string('eval_on', '',
'"format0:path0;format1:path1", e.g. "kitti:/usr/..."')
flags.DEFINE_string('plot_dir', '', 'Path to directory where plots are saved.')
flags.DEFINE_string('checkpoint_dir', '',
'Path to directory for saving and restoring checkpoints.')
flags.DEFINE_string('init_checkpoint_dir', '',
'Path to directory for initializing from a checkpoints.')
flags.DEFINE_bool(
'plot_debug_info', False,
'Flag to indicate whether to plot debug info during training.')
flags.DEFINE_bool(
'use_tensorboard', False, 'Toggles logging to tensorboard.')
flags.DEFINE_string(
'tensorboard_logdir', '', 'Where to log tensorboard summaries.')
flags.DEFINE_string(
from phd.lib.labm8 import graph as libgraph
from experimental.compilers.reachability.proto import reachability_pb2
FLAGS = flags.FLAGS
flags.DEFINE_integer('reachability_num_nodes', 6,
'The number of CFG nodes to generate.')
flags.DEFINE_integer('reachability_seed', None,
'Random number to seed numpy RNG with.')
flags.DEFINE_float(
'reachability_scaling_param', 1.0,
'Scaling parameter to use to determine the likelihood of edges between CFG '
'vertices. A larger number makes for more densely connected CFGs.')
flags.DEFINE_string(
'reachability_dot_path',
'/tmp/phd/experimental/compilers/reachability/reachability.dot',
'Path to dot file to generate.')
def NumberToLetters(num: int) -> str:
"""Convert number to name, e.g. 0 -> 'A', 1 -> 'B'."""
if num >= 26:
raise ValueError
return chr(ord('A') + num)
class ControlFlowGraph(libgraph.Graph):
"""A control flow graph."""
def __init__(self, name: str):
super(ControlFlowGraph, self).__init__(name)
self.all_nodes: typing.List['ControlFlowGraph'] = None
import sys
import os
from absl import flags
from pathlib import Path
from types import SimpleNamespace
FLAGS = flags.FLAGS
flags.DEFINE_bool("training", True, "Whether to train agents.")
flags.DEFINE_float("learning_rate", 0.1, "Learning rate for training.")
flags.DEFINE_float("discount", 0.9, "Discount rate for future rewards.")
flags.DEFINE_float("epsilon", 0.9, "Epsilon greedy parameter")
flags.DEFINE_integer("eval_agent_steps", 10, "Num of steps between agent eval")
flags.DEFINE_string("log_path", os.path.join(Path.home(), "tb_output"), "Path for log.")
flags.DEFINE_bool("profile", False, "Whether to turn on code profiling.")
flags.DEFINE_bool("trace", False, "Whether to trace the code execution.")
flags.DEFINE_integer("minimap_resolution", 64, "Resolution for minimap feature layers.")
flags.DEFINE_string("map", "DefeatRoaches", "Name of a map to use.")
flags.DEFINE_integer("max_steps", 20, "Total steps for training.") # Num episodes
flags.DEFINE_bool("render", False, "Whether to render with pygame.")
flags.DEFINE_integer("screen_resolution", 64, "Resolution for screen feature layers.")
flags.DEFINE_integer("step_mul", 8, "Game steps per agent step.")
flags.DEFINE_string("agent", "simple_viking_agent", "Which agent to run.")
flags.DEFINE_string("model", "always_attack", "Which model to use.")
flags.DEFINE_integer("experience_replay_max_size", None, "Max steps to keep in replay buffer before overwriting")
flags.DEFINE_integer("mini_batch_size", 64, "Minibatch size")
flags.DEFINE_string("run_comment", "", "A comment string to distinguish the run.")
flags.DEFINE_bool("decay_lr", True, "If learning rate should be decayed or not")
_TREE_METHOD = flags.DEFINE_enum(
'xgboost_tree_method', 'gpu_hist', ['gpu_hist', 'hist'],
'XGBoost builtin tree methods.')
_SPARSITY = flags.DEFINE_float(
'xgboost_sparsity', 0.0, 'XGBoost sparsity-aware split finding algorithm.')
_ROWS = flags.DEFINE_integer(
'xgboost_rows', 1000000, 'The number of data rows.')
_COLUMNS = flags.DEFINE_integer(
'xgboost_columns', 50, 'The number of data columns.')
_ITERATIONS = flags.DEFINE_integer(
'xgboost_iterations', 500, 'The number of training iterations.')
_TEST_SIZE = flags.DEFINE_float(
'xgboost_test_size', 0.25,
'Train-test split for evaluating machine learning algorithms')
_PARAMS = flags.DEFINE_string(
'xgboost_params', None,
'Provide additional parameters as a Python dict string, '
'e.g. --params \"{\'max_depth\':2}\"')
FLAGS = flags.FLAGS
BENCHMARK_NAME = 'xgboost'
BENCHMARK_CONFIG = """
xgboost:
description: Runs XGBoost Benchmark.
vm_groups:
default:
vm_spec:
GCP:
machine_type: n1-standard-4
gpu_type: t4
"""Binary that runs inference on a pre-trained cost model."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from absl import app
from absl import flags
import tensorflow.compat.v1 as tf
from tunas import mobile_cost_model
from tunas import mobile_search_space_v3
from tunas import search_space_utils
flags.DEFINE_string(
'indices', '',
'Colon-separated list of integers specifying the network architecture '
'to evaluate.')
flags.DEFINE_string('ssd', mobile_search_space_v3.PROXYLESSNAS_SEARCH,
'Search space definition to use.')
FLAGS = flags.FLAGS
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
indices = search_space_utils.parse_list(FLAGS.indices, int)
ssd = FLAGS.ssd
cost = mobile_cost_model.estimate_cost(indices, ssd)
print('estimated cost: {:f}'.format(cost))
FLAGS = flags.FLAGS
flags.DEFINE_bool("visualize", False, "Whether to render with pygame.")
flags.DEFINE_integer("resolution", 32,
"Resolution for screen and minimap feature layers.")
flags.DEFINE_integer("step_mul", 8, "Game steps per agent step.")
flags.DEFINE_integer("n_envs", 1, "Number of environments to run in parallel")
flags.DEFINE_integer("episodes", 3, "Number of complete episodes")
flags.DEFINE_integer(
"n_steps_per_batch", None,
"Number of steps per batch, if None use 8 for a2c and 128 for ppo"
) # (MINE) TIMESTEPS HERE!!!
flags.DEFINE_integer("all_summary_freq", 50,
"Record all summaries every n batch")
flags.DEFINE_integer("scalar_summary_freq", 5,
"Record scalar summaries every n batch")
flags.DEFINE_string("checkpoint_path", "_files/models",
"Path for agent checkpoints")
flags.DEFINE_string("summary_path", "_files/summaries",
"Path for tensorboard summaries")
flags.DEFINE_string("model_name", "my_beacon_beta_model",
"Name for checkpoints and tensorboard summaries")
flags.DEFINE_integer(
"K_batches", 50,
"Number of training batches to run in thousands, use -1 to run forever"
) #(MINE) not for now
flags.DEFINE_string("map_name", "MoveToBeacon_beta", "Name of a map to use.")
flags.DEFINE_float("discount", 0.95, "Reward-discount for the agent")
flags.DEFINE_boolean(
"training", False,
"if should train the model, if false then save only episode score summaries"
)
flags.DEFINE_enum(
import random
import sys
from absl import flags
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt # pylint: disable=g-import-not-at-top
import numpy as np
import PIL
import tensorflow.compat.v2 as tf
FLAGS = flags.FLAGS
flags.DEFINE_boolean('run_tf_functions_eagerly', False,
'Run TF functions eagerly for debugging.')
flags.DEFINE_string('mnist_path', None, 'Path to the `mnist.npz`')
flags.DEFINE_string('celeba_path', None,
'Path to the `celeba_40000_32.pickle`')
IMAGE_BINS = 256
def mnist_dataset(num_channels=1):
"""Loads the MNIST dataset.
This loads the dataset from `mnist_path` flag. The dataset is padded to be
32x32. Optionally, this also duplicates the grayscale channel `num_channels`
times.
Args:
num_channels: How many channels the output should have.
vt = np.arange(0, len(v), 10)
labels = np.round(v[vt] * 100) / 100
ax.set_xticks(vt)
ax.set_xticklabels(labels)
plt.xlabel("velo")
plt.title(h)
ixP += 1
plt.show()
if __name__ == "__main__":
from absl import flags
import sys
flags.DEFINE_string("learn", "sim", "")
flags.DEFINE_float("tau", 0.995, "")
flags.DEFINE_float("gamma", 0.9, "")
flags.DEFINE_float("lr_pi", 1e-4, "")
flags.DEFINE_float("lr_q", 1e-3, "")
flags.DEFINE_integer("numTrains", 20, "")
flags.DEFINE_float("numEpisodes", float('inf'), "")
flags.DEFINE_string("directory", "auto", "")
flags.DEFINE_string("add2Dir", "", "")
flags.DEFINE_bool("runByOrder", True, "")
flags.DEFINE_integer("numUpdates", 1, "")
flags.DEFINE_integer("batchSize", 128, "")
flags.FLAGS(sys.argv)
from perfkitbenchmarker import sample
GIT_REPO = 'https://github.com/shaygalon/memcache-perf.git'
MCPERF_DIR = '%s/mcperf_benchmark' % linux_packages.INSTALL_DIR
MCPERF_BIN = '%s/mcperf' % MCPERF_DIR
APT_PACKAGES = 'scons libevent-dev gengetopt libzmq3-dev'
FLAGS = flags.FLAGS
flags.DEFINE_enum(
'mcperf_protocol', 'binary', ['binary', 'ascii'],
'Protocol to use. Supported protocols are binary and ascii.')
flags.DEFINE_list('mcperf_qps', [],
'Target aggregate QPS. If not set, target for peak qps.')
flags.DEFINE_integer('mcperf_time', 300, 'Maximum time to run (seconds).')
flags.DEFINE_string('mcperf_keysize', '16',
'Length of memcached keys (distribution).')
flags.DEFINE_string('mcperf_valuesize', '128',
'Length of memcached values (distribution).')
flags.DEFINE_integer('mcperf_records', 10000,
'Number of memcached records to use.')
flags.DEFINE_float('mcperf_ratio', 0.0,
'Ratio of set:get. By default, read only.')
flags.DEFINE_list(
'mcperf_options', ['iadist=exponential:0.0'],
'Additional mcperf long-form options (--) in comma separated form. e.g.'
'--mcperf_options=blocking,search=99:1000.'
'See https://github.com/shaygalon/memcache-perf for all available options.'
)
# If more than one value provided for threads, connections, depths, we will
# enumerate all test configurations. e.g.
from absl import app
from absl import flags
from absl import logging
import tensorflow as tf
from official.nlp import optimization
from official.nlp.bert import bert_models
from official.nlp.bert import common_flags
from official.nlp.bert import configs as bert_configs
from official.nlp.bert import input_pipeline
#from official.utils.misc import keras_utils
flags.DEFINE_string('predict_data_path', None,
'Path to testing data for BERT classifier.')
flags.DEFINE_string('predict_output_dir', None,
'dir to predict output.')
flags.DEFINE_string(
'input_meta_data_path', None,
'Path to file that contains meta data about input '
'to be used for training and evaluation.')
flags.DEFINE_integer('predict_batch_size', 32, 'Batch size for prediction.')
common_flags.define_common_bert_flags()
FLAGS = flags.FLAGS
def get_dataset_fn(input_file_pattern, max_seq_length, global_batch_size,
is_training):
"""Gets a closure to create a dataset."""
"""Evaluates the performance of all the checkpoints on validation set.
Sample command:
python -u eval_all_fast.py --inc_ckpt ~/workspace/ckpt/inception_v4.ckpt --job_dir saving_self_imcap --device 0 --threads 12
"""
import glob
import json
import multiprocessing
import os
from absl import app
from absl import flags
flags.DEFINE_integer('threads', 1, 'num of threads')
flags.DEFINE_string('img_dir', '~/mscoco_image_features', 'image features dir')
flags.DEFINE_string(
'img_file', 'img_inceptionv4_val.hdf5',
'image features file: [img_inceptionv4_val.hdf5, img_resnet101v2_val.hdf5]'
)
flags.DEFINE_string(
'gts_file', 'val_test_dict.json',
'ground-truth caption file: [val_test_dict.json, val_test_dict_v4.json]')
flags.DEFINE_string('device', '0', 'device')
from caption_infer_fast import Infer
import h5py
from speaksee import evaluation
"""Binary to generate training/evaluation dataset for NCF model."""
import json
# pylint: disable=g-bad-import-order
# Import libraries
from absl import app
from absl import flags
import tensorflow as tf
# pylint: enable=g-bad-import-order
from official.recommendation import movielens
from official.recommendation import data_preprocessing
flags.DEFINE_string(
"data_dir", None,
"The input data dir at which training and evaluation tf record files "
"will be saved.")
flags.DEFINE_string("meta_data_file_path", None,
"The path in which input meta data will be written.")
flags.DEFINE_enum("dataset", "ml-20m", ["ml-1m", "ml-20m"],
"Dataset to be trained/evaluated.")
flags.DEFINE_enum(
"constructor_type", "bisection", ["bisection", "materialized"],
"Strategy to use for generating false negatives. materialized has a "
"precompute that scales badly, but a faster per-epoch construction "
"time and can be faster on very large systems.")
flags.DEFINE_integer("num_train_epochs", 14,
"Total number of training epochs to generate.")
flags.DEFINE_integer(
"num_negative_samples", 4,
"Number of negative instances to pair with positive instance.")
flags.DEFINE_integer('batch_size', 16, '')
flags.DEFINE_integer('max_seq_length', 128, '')
flags.DEFINE_integer('intent_ranking_length', 5, '')
flags.DEFINE_integer('gpus', 1, '')
flags.DEFINE_integer('patience', 3, '')
flags.DEFINE_integer('pad_token_label_id', -100, '')
flags.DEFINE_integer('warmup_steps', 0, 'Linear warmup over warmup_steps.')
flags.DEFINE_integer('gradient_accumulation_steps', 1, 'Number of updates steps to accumulate before performing a backward/update pass.')
flags.DEFINE_float('learning_rate', 1e-5, '')
flags.DEFINE_float('crf_learning_rate', 1e-3, '')
flags.DEFINE_float('weight_decay', 1e-2, '')
flags.DEFINE_float('adam_epsilon', 1e-8, '')
flags.DEFINE_float('test_size', 0.3, '')
flags.DEFINE_string('monitor', 'val_loss', '')
flags.DEFINE_string('metric_mode', 'min', '')
flags.DEFINE_string('model_name_or_path', 'bert-base-chinese', '')
flags.DEFINE_string('data_dir', os.path.join(os.getcwd(), 'data'), '')
flags.DEFINE_string('output_dir', os.path.join(os.getcwd(), 'output'), '')
flags.DEFINE_string('cache_dir', os.path.join(os.getcwd(), 'cache'), '')
FLAGS = flags.FLAGS
sh.rm('-r', '-f', 'logs')
sh.mkdir('logs')
class NerClassifier(pl.LightningModule):
def __init__(self):
from __future__ import print_function
from absl import app
from absl import flags
from absl import logging
from common import Actor
import gym
import lfd_envs
import tensorflow as tf
from utils import do_rollout
from tensorflow.contrib import summary as contrib_summary
from tensorflow.contrib import training as contrib_training
from tensorflow.contrib.eager.python import tfe as contrib_eager_python_tfe
FLAGS = flags.FLAGS
flags.DEFINE_string('env', 'Hopper-v1', 'Environment for training/evaluation.')
flags.DEFINE_string('load_dir', '', 'Directory to save models.')
flags.DEFINE_boolean('use_gpu', False,
'Directory to write TensorBoard summaries.')
flags.DEFINE_boolean('wrap_for_absorbing', False,
'Use the wrapper for absorbing states.')
flags.DEFINE_integer('num_trials', 10, 'Number of evaluation trials to run.')
flags.DEFINE_string('master', 'local', 'Location of the session.')
flags.DEFINE_integer('ps_tasks', 0, 'Number of Parameter Server tasks.')
flags.DEFINE_integer('task_id', 0, 'Id of the current TF task.')
def wait_for_next_checkpoint(log_dir,
last_checkpoint=None,
seconds_to_sleep=1,
timeout=20):
from absl.flags import FLAGS
import cv2
import matplotlib.pyplot as plt
import tensorflow as tf
from yolov3_tf2.models import (YoloV3, YoloV3Tiny)
from yolov3_tf2.dataset import transform_images
from yolov3_tf2.utils import draw_outputs, convert_boxes
from deep_sort import preprocessing
from deep_sort import nn_matching
from deep_sort.detection import Detection
from deep_sort.tracker import Tracker
from tools import generate_detections as gdet
from PIL import Image
flags.DEFINE_string('classes', './data/labels/coco.names',
'path to classes file')
flags.DEFINE_string('weights', './weights/yolov3.tf', 'path to weights file')
flags.DEFINE_boolean('tiny', False, 'yolov3 or yolov3-tiny')
flags.DEFINE_integer('size', 416, 'resize images to')
flags.DEFINE_string('video', './data/video/test.mp4',
'path to video file or number for webcam)')
flags.DEFINE_string('output', None, 'path to output video')
flags.DEFINE_string('output_format', 'XVID',
'codec used in VideoWriter when saving video to file')
flags.DEFINE_integer('num_classes', 80, 'number of classes in the model')
dict_tracks = {"Koi": {}, "Tilapia": {}}
def get_patterns(center, track_id, class_name):
#This function stores all tracked fish and their moving patterns
import absl.logging as _logging # pylint: disable=unused-import
import tensorflow as tf
import sentencepiece as spm
import tokenization
from data_utils import SEP_ID, VOCAB_SIZE, CLS_ID
import model_utils
import function_builder
from classifier_utils import PaddingInputExample
from classifier_utils import convert_single_example
from prepro_utils import preprocess_text, encode_ids
# Model
flags.DEFINE_string("model_config_path", default=None,
help="Model config path.")
flags.DEFINE_float("dropout", default=0.1,
help="Dropout rate.")
flags.DEFINE_float("dropatt", default=0.1,
help="Attention dropout rate.")
flags.DEFINE_integer("clamp_len", default=-1,
help="Clamp length")
flags.DEFINE_string("summary_type", default="last",
help="Method used to summarize a sequence into a compact vector.")
flags.DEFINE_bool("use_summ_proj", default=True,
help="Whether to use projection for summarizing sequences.")
flags.DEFINE_bool("use_bfloat16", False,
help="Whether to use bfloat16.")
# Parameter initialization
flags.DEFINE_enum("init", default="normal",
such as being able to inject custom images during training. So instead, this
file is spawned as a Python process, which supports the added functionality.
"""
from absl import flags as absl_flags
import numpy as np
import tensorflow as tf
import benchmark_cnn
import flags
import preprocessing
import test_util
absl_flags.DEFINE_string(
'fake_input', 'none', """What fake input to inject into benchmark_cnn. This
is ignored if --model=test_model.
Options are:
none: Do not inject any fake input.
zeros_and_ones: Half the images will be all 0s with
a label of 0. Half the images will be all 1s with a
label of 1.""")
flags.define_flags()
FLAGS = flags.FLAGS
def get_test_image_preprocessor(batch_size, params):
"""Returns the preprocessing.TestImagePreprocessor that should be injected.
Returns None if no preprocessor should be injected.
Args:
batch_size: The batch size across all GPUs.
def define_ncf_flags():
"""Add flags for running ncf_main."""
# Add common flags
flags_core.define_base(export_dir=False)
flags_core.define_performance(num_parallel_calls=False,
inter_op=False,
intra_op=False,
synthetic_data=False,
max_train_steps=False,
dtype=False,
all_reduce_alg=False)
flags_core.define_device(tpu=True)
flags_core.define_benchmark()
flags.adopt_module_key_flags(flags_core)
flags_core.set_defaults(model_dir="/tmp/ncf/",
data_dir="/tmp/movielens-data/",
train_epochs=2,
batch_size=256,
hooks="ProfilerHook",
tpu=None)
# Add ncf-specific flags
flags.DEFINE_enum(
name="dataset",
default="ml-1m",
enum_values=["ml-1m", "ml-20m"],
case_sensitive=False,
help=flags_core.help_wrap("Dataset to be trained and evaluated."))
flags.DEFINE_boolean(
name="download_if_missing",
default=True,
help=flags_core.help_wrap(
"Download data to data_dir if it is not already present."))
flags.DEFINE_string(
name="eval_batch_size",
default=None,
help=flags_core.help_wrap(
"The batch size used for evaluation. This should generally be larger"
"than the training batch size as the lack of back propagation during"
"evaluation can allow for larger batch sizes to fit in memory. If not"
"specified, the training batch size (--batch_size) will be used."))
flags.DEFINE_integer(
name="num_factors",
default=8,
help=flags_core.help_wrap("The Embedding size of MF model."))
# Set the default as a list of strings to be consistent with input arguments
flags.DEFINE_list(
name="layers",
default=["64", "32", "16", "8"],
help=flags_core.help_wrap(
"The sizes of hidden layers for MLP. Example "
"to specify different sizes of MLP layers: --layers=32,16,8,4"))
flags.DEFINE_float(
name="mf_regularization",
default=0.,
help=flags_core.help_wrap(
"The regularization factor for MF embeddings. The factor is used by "
"regularizer which allows to apply penalties on layer parameters or "
"layer activity during optimization."))
flags.DEFINE_list(
name="mlp_regularization",
default=["0.", "0.", "0.", "0."],
help=flags_core.help_wrap(
"The regularization factor for each MLP layer. See mf_regularization "
"help for more info about regularization factor."))
flags.DEFINE_integer(
name="num_neg",
default=4,
help=flags_core.help_wrap(
"The Number of negative instances to pair with a positive instance."
))
flags.DEFINE_float(name="learning_rate",
default=0.001,
help=flags_core.help_wrap("The learning rate."))
flags.DEFINE_float(
name="hr_threshold",
default=None,
help=flags_core.help_wrap(
"If passed, training will stop when the evaluation metric HR is "
"greater than or equal to hr_threshold. For dataset ml-1m, the "
"desired hr_threshold is 0.68 which is the result from the paper; "
"For dataset ml-20m, the threshold can be set as 0.95 which is "
"achieved by MLPerf implementation."))
import os
from absl import app
from absl import flags
from absl import logging
import numpy as np
import dataloader
import det_model_fn
import hparams_config
import utils
import tensorflow.compat.v1 as tf
flags.DEFINE_string(
'tpu',
default=None,
help='The Cloud TPU to use for training. This should be either the name '
'used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 '
'url.')
flags.DEFINE_string(
'gcp_project',
default=None,
help='Project name for the Cloud TPU-enabled project. If not specified, we '
'will attempt to automatically detect the GCE project from metadata.')
flags.DEFINE_string(
'tpu_zone',
default=None,
help='GCE zone where the Cloud TPU is located in. If not specified, we '
'will attempt to automatically detect the GCE project from metadata.')
flags.DEFINE_string('eval_name', default=None, help='Eval job name')
flags.DEFINE_enum('strategy', None, ['tpu', 'gpus', ''],
'Training: gpus for multi-gpu, if None, use TF default.')
from absl import flags
import apache_beam as beam
from apache_beam.transforms import combiners
from moonlight.pipeline import pipeline_flags
from moonlight.training.clustering import staffline_patches_dofn
import tensorflow as tf
from tensorflow.contrib.learn.python.learn import learn_runner
from tensorflow.python.lib.io import file_io
from tensorflow.python.lib.io import tf_record
FLAGS = flags.FLAGS
flags.DEFINE_multi_string('music_pattern', [],
'Pattern for the input music score PNGs.')
flags.DEFINE_string('output_path', None, 'Path to the output TFRecords.')
flags.DEFINE_integer('patch_height', 18,
'The normalized height of a staffline.')
flags.DEFINE_integer('patch_width', 15,
'The width of a horizontal patch of a staffline.')
flags.DEFINE_integer('num_stafflines', 19,
'The number of stafflines to extract.')
flags.DEFINE_integer('num_pages', 0, 'Subsample the pages to run on.')
flags.DEFINE_integer('num_outputs', 0, 'Number of output patches.')
flags.DEFINE_integer('max_patches_per_page', 10,
'Sample patches per page if above this amount.')
flags.DEFINE_integer('timeout_ms', 600000, 'Timeout for processing a page.')
flags.DEFINE_integer('kmeans_num_clusters', 1000,
'Number of k-means clusters.')
flags.DEFINE_integer('kmeans_batch_size', 10000,
'Batch size for mini-batch k-means.')
dev_meta = os.path.join(target_dir, "dev_meta.json")
test_meta = os.path.join(target_dir, "test_meta.json")
word2idx_file = os.path.join(target_dir, "word2idx.json")
char2idx_file = os.path.join(target_dir, "char2idx.json")
answer_file = os.path.join(answer_dir, "answer.json")
if not os.path.exists(target_dir):
os.makedirs(target_dir)
if not os.path.exists(event_dir):
os.makedirs(event_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
if not os.path.exists(answer_dir):
os.makedirs(answer_dir)
flags.DEFINE_string("mode", "train", "train/debug/test")
flags.DEFINE_string("target_dir", target_dir, "")
flags.DEFINE_string("event_dir", event_dir, "")
flags.DEFINE_string("save_dir", save_dir, "")
flags.DEFINE_string("train_file", train_file, "")
flags.DEFINE_string("dev_file", dev_file, "")
flags.DEFINE_string("test_file", test_file, "")
flags.DEFINE_string("glove_word_file", glove_word_file, "")
flags.DEFINE_string("train_record_file", train_record_file, "")
flags.DEFINE_string("dev_record_file", dev_record_file, "")
flags.DEFINE_string("test_record_file", test_record_file, "")
flags.DEFINE_string("word_emb_file", word_emb_file, "")
flags.DEFINE_string("char_emb_file", char_emb_file, "")
flags.DEFINE_string("train_eval_file", train_eval, "")