import tensorflow as tf
import tensorflow_federated as tff
from tensorflow_federated.python.research.differential_privacy import dp_utils
from tensorflow_federated.python.research.optimization.emnist import dataset
from tensorflow_federated.python.research.optimization.emnist import models
from tensorflow_federated.python.research.optimization.shared import optimizer_utils
from tensorflow_federated.python.research.utils import training_loop
from tensorflow_federated.python.research.utils import training_utils
from tensorflow_federated.python.research.utils import utils_impl
with utils_impl.record_hparam_flags():
# Experiment hyperparameters
flags.DEFINE_enum(
'model', 'cnn', ['cnn', '2nn'], 'Which model to use. This '
'can be a convolutional model (cnn) or a two hidden-layer '
'densely connected network (2nn).')
flags.DEFINE_integer('client_batch_size', 20,
'Batch size used on the client.')
flags.DEFINE_integer('clients_per_round', 10,
'How many clients to sample per round.')
flags.DEFINE_integer(
'client_epochs_per_round', 1,
'Number of client (inner optimizer) epochs per federated round.')
flags.DEFINE_boolean(
'uniform_weighting', False,
'Whether to weigh clients uniformly. If false, clients '
'are weighted by the number of samples.')
# Optimizer configuration (this defines one or more flags per optimizer).
utils_impl.define_optimizer_flags('server')
"""
WORD_COUNT_CONFIGURATION = dict(
[
(dpb_service.DATAPROC, ('org.apache.spark.examples.JavaWordCount',
dpb_service.BaseDpbService.SPARK_JOB_TYPE)),
(dpb_service.DATAFLOW, ('org.example.WordCount',
dpb_service.BaseDpbService.DATAFLOW_JOB_TYPE)),
(dpb_service.EMR, ('org.apache.spark.examples.JavaWordCount',
dpb_service.BaseDpbService.SPARK_JOB_TYPE))
]
)
flags.DEFINE_string('dpb_wordcount_input', None, 'Input for word count')
flags.DEFINE_enum('dpb_wordcount_fs', dpb_service.BaseDpbService.GCS_FS,
[dpb_service.BaseDpbService.GCS_FS,
dpb_service.BaseDpbService.S3_FS],
'File System to use for the job output')
flags.DEFINE_string('dpb_wordcount_out_base', None,
'Base directory for word count output')
FLAGS = flags.FLAGS
def GetConfig(user_config):
return configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
def CheckPrerequisites(benchmark_config):
"""Verifies that the required resources are present.
Raises:
flags.DEFINE_float("dropout", default=0.0,
help="Dropout rate.")
flags.DEFINE_float("dropatt", default=0.0,
help="Attention dropout rate.")
flags.DEFINE_bool("untie_r", default=False,
help="Untie r_w_bias and r_r_bias")
flags.DEFINE_string("summary_type", default="last",
help="Method used to summarize a sequence into a compact vector.")
flags.DEFINE_string("ff_activation", default="relu",
help="Activation type used in position-wise feed-forward.")
flags.DEFINE_bool("use_bfloat16", False,
help="Whether to use bfloat16.")
# Parameter initialization
flags.DEFINE_enum("init", default="normal",
enum_values=["normal", "uniform"],
help="Initialization method.")
flags.DEFINE_float("init_std", default=0.02,
help="Initialization std when init is normal.")
flags.DEFINE_float("init_range", default=0.1,
help="Initialization std when init is uniform.")
FLAGS = flags.FLAGS
def get_model_fn():
"""doc."""
def model_fn(features, labels, mode, params):
"""doc."""
#### Training or Evaluation
import os.path
from absl import app
from absl import flags
from absl import logging
import gin
import tensorflow as tf
from dedal.train import training_loop
flags.DEFINE_string('base_dir', None, 'Directory to save trained model in.')
flags.DEFINE_string(
'reference_dir', None,
'Directory where to read the reference model from (if exists).')
flags.DEFINE_enum('task', 'train', ['train', 'eval', 'downstream'],
'Whether this is a train, eval or downstream task.')
flags.DEFINE_multi_string('gin_config', [],
'List of paths to the config files.')
flags.DEFINE_multi_string('gin_bindings', [],
'Newline separated list of Gin parameter bindings.')
flags.DEFINE_string('config_path', 'dedal/configs',
'Where to find the gin configurations.')
FLAGS = flags.FLAGS
def main(unused_argv):
filenames = [os.path.join(FLAGS.config_path, p) for p in FLAGS.gin_config]
gin.parse_config_files_and_bindings(filenames, FLAGS.gin_bindings)
strategy = training_loop.get_strategy()
logging.info('Distribution strategy: %s', strategy)
from perfkitbenchmarker import errors
from perfkitbenchmarker import hpc_util
from perfkitbenchmarker import sample
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.linux_packages import openfoam
from perfkitbenchmarker.linux_packages import openmpi
_DEFAULT_CASE = 'motorbike'
_CASE_PATHS = {
'motorbike': 'tutorials/incompressible/simpleFoam/motorBike',
}
assert _DEFAULT_CASE in _CASE_PATHS
FLAGS = flags.FLAGS
_CASE = flags.DEFINE_enum('openfoam_case', _DEFAULT_CASE,
sorted(list(_CASE_PATHS.keys())),
'Name of the OpenFOAM case to run.')
_DIMENSIONS = flags.DEFINE_list('openfoam_dimensions', ['20_8_8'],
'Dimensions of the case.')
_THREADS_PER_VM = flags.DEFINE_integer(
'openfoam_num_threads_per_vm', None,
'The number of threads per VM to run OpenFOAM with. If None, defaults to '
'half the total number of vCPUs available.')
_MPI_MAPPING = flags.DEFINE_string(
'openfoam_mpi_mapping', 'core:SPAN',
'Mpirun process mapping to use as arguments to "mpirun --map-by".')
_DECOMP = flags.DEFINE_enum(
'openfoam_decomp_method', 'scotch', ['scotch', 'hierarchical', 'simple'],
'Decomposition method to use in decomposePar. See: '
'https://cfd.direct/openfoam/user-guide/v7-running-applications-parallel/')
_MAX_GLOBAL_CELLS = flags.DEFINE_integer(
"weights",
"/home/md273/model_zoo/416_darknet_1/checkpoints/yolov3_train_15.tf",
"path to weights file",
)
flags.DEFINE_string("output", "/home/md273/model_zoo/416_darknet_1/eval",
"path to output the results")
flags.DEFINE_string(
"anchor_path",
"/home/md273/model_zoo/416_darknet_1/anchors.npy",
"path to the anchor file",
)
flags.DEFINE_enum(
"backbone",
"original",
["original", "dense"],
"original: the original YOLOv3 with darknet 53, "
"dense: Customised yolo v3 with a downsampling factor of 16, ",
)
flags.DEFINE_enum(
"head",
"yolo3",
["yolo3", "yolo3_dense_2", "yolo3_dense_1"],
"original: the original YOLOv3 with darknet 53,"
"dense: Customised yolo v3 with a downsampling factor of 16, and two scales ",
)
# os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import numpy as np
import robustness_metrics as rm
import tensorflow as tf
import tensorflow_datasets as tfds
import uncertainty_baselines as ub
import utils # local file import
import uncertainty_metrics as um
flags.DEFINE_string('checkpoint_dir', None,
'The directory where the model weights are stored.')
flags.mark_flag_as_required('checkpoint_dir')
flags.DEFINE_integer('seed', 42, 'Random seed.')
flags.DEFINE_integer('per_core_batch_size', 64, 'Batch size per TPU core/GPU.')
flags.DEFINE_enum('dataset', 'cifar10',
enum_values=['cifar10', 'cifar100'],
help='Dataset.')
flags.DEFINE_string('cifar100_c_path', None,
'Path to the TFRecords files for CIFAR-100-C. Only valid '
'(and required) if dataset is cifar100 and corruptions.')
flags.DEFINE_integer('num_bins', 15, 'Number of bins for ECE.')
flags.DEFINE_string('output_dir', '/tmp/cifar', 'Output directory.')
# Accelerator flags.
flags.DEFINE_bool('use_gpu', False, 'Whether to run on GPU or otherwise TPU.')
flags.DEFINE_integer('num_cores', 8, 'Number of TPU cores or number of GPUs.')
FLAGS = flags.FLAGS
def parse_checkpoint_dir(checkpoint_dir):
"""Parse directory of checkpoints."""
try:
# imports when running on the VM
from providers import azure_flags
from providers import gcs_flags
from providers import s3_flags
except ImportError:
# imports when running tests
from perfkitbenchmarker.scripts.object_storage_api_test_scripts import azure_flags
from perfkitbenchmarker.scripts.object_storage_api_test_scripts import gcs_flags
from perfkitbenchmarker.scripts.object_storage_api_test_scripts import s3_flags
FLAGS = flags.FLAGS
flags.DEFINE_enum(
'storage_provider', 'GCS', ['GCS', 'S3', 'AZURE'],
'The target storage provider to test.')
flags.DEFINE_string('bucket', None,
'The name of the bucket to test with. Caller is '
'responsible to create an empty bucket for a particular '
'invocation of this test and then clean-up the bucket '
'after this test returns.')
flags.DEFINE_enum(
'scenario', 'OneByteRW', [
'OneByteRW', 'ListConsistency', 'SingleStreamThroughput',
'CleanupBucket', 'MultiStreamWrite', 'MultiStreamRead',
'MultiStreamDelete'
],
'The various scenarios to run. OneByteRW: read and write of single byte. '
return dataset
if split == "train" and num_hosts > 1:
record_info["num_batch"] = num_batch // num_hosts
return input_fn, record_info
def get_corpus_info(corpus_info_path):
with open(corpus_info_path, "r") as fp:
corpus_info = json.load(fp)
return corpus_info
if __name__ == "__main__":
FLAGS = flags.FLAGS
flags.DEFINE_string("data_dir", None, help="Location of the data corpus")
flags.DEFINE_enum("dataset", "cdata", ["ptb", "wt2", "wt103", "lm1b", "enwik8", "text8", "cdata"], help="Dataset name.")
flags.DEFINE_integer("per_host_train_bsz", 60, help="train batch size each host")
flags.DEFINE_integer("per_host_valid_bsz", 60, help="valid batch size each host")
flags.DEFINE_integer("per_host_test_bsz", 0, help="If > 0, enter test mode and process test set only. Otherwise, process train and dev sets only.")
flags.DEFINE_integer("tgt_len", 70, help="number of tokens to predict")
flags.DEFINE_integer("max_batch", -1, help="run in debug mode")
flags.DEFINE_integer("num_core_per_host", 1, help="8 for TPU v2.")
flags.DEFINE_bool("debug", default=False, help="Process only the first batch without shuffle for lm1b.")
flags.DEFINE_integer("num_procs", 1, help="number of processes")
flags.DEFINE_integer("num_shuffle", 4, help="number of shuffles for lm1b")
tf.compat.v1.app.run(main)
from absl import flags
from absl.testing import parameterized
import hypothesis as hp
from hypothesis import strategies as hps
import tensorflow.compat.v2 as tf
import tensorflow_probability as tfp
from tensorflow_probability.python.bijectors import hypothesis_testlib as bijector_hps
from tensorflow_probability.python.internal import hypothesis_testlib as tfp_hps
from tensorflow_probability.python.internal import tensor_util
from tensorflow_probability.python.internal import tensorshape_util
from tensorflow.python.framework import test_util # pylint: disable=g-direct-tensorflow-import
tfb = tfp.bijectors
tfd = tfp.distributions
flags.DEFINE_enum('tf_mode', 'graph', ['eager', 'graph'],
'TF execution mode to use')
FLAGS = flags.FLAGS
TF2_FRIENDLY_BIJECTORS = (
'AffineScalar',
'BatchNormalization',
# 'CategoricalToDiscrete', TODO(b/137956955): Add support
# for hypothesis testing
'CholeskyOuterProduct',
'Cumsum',
'DiscreteCosineTransform',
'Exp',
'Expm1',
'FillTriangular',
'Gumbel',
# Specific for evaluation
flags.DEFINE_string("output_file", "results/results_best_target-jnu-347.yaml",
"Output filename to write the yaml file with the results")
flags.DEFINE_float(
"gpumem", 1024,
"GPU memory to let TensorFlow use, in MiB (0 for all, divided among jobs)")
flags.DEFINE_string(
"match", "jnu-347-wsmix_np-[0-9]*",
"String matching to determine which logs/models to process")
flags.DEFINE_integer("jobs", 1, "Number of TensorFlow jobs to run at once")
flags.DEFINE_integer(
"gpus", 1,
"Split jobs between GPUs -- overrides jobs (1 == run multiple jobs on first GPU)"
)
flags.DEFINE_enum("selection", "best_target",
["last", "best_source", "best_target"],
"Which model to select")
flags.DEFINE_boolean(
"test", False,
"Whether to evaluate on the true test set or if --notest, then the validation set"
)
flags.mark_flag_as_required("output_file")
def get_gpus():
"""
Get the list of GPU ID's that SLURM is giving us
"""
return [int(x) for x in os.getenv("SLURM_JOB_GPUS", "").split(",")]
from absl import flags
from . import backtrace, flow, format, testing, utils
from .basis import _dummy
from .computation import computation_manager
FLAGS = flags.FLAGS
# run: Invoked by users.
# test: Verifies cyberbrain's internal state is the same as golden.
# golden: Generates golden file.
# debug: Prints cyberbrain's internal state, but don't do assertion.
flags.DEFINE_enum(
"mode",
"run",
["run", "test", "golden", "debug"],
"The mode which Cyberbrain runs in.",
)
flags.DEFINE_string("test_dir", None, "Directory to save test output to.")
def global_tracer(frame, event_type, arg):
"""Global trace function."""
if utils.should_exclude(frame.f_code.co_filename):
return
# print("\nglobal: ", frame, event_type, frame.f_code.co_filename, frame.f_lineno)
assert event_type == "call"
succeeded = computation_manager.add_computation(event_type, frame, arg)
# https://docs.python.org/3/library/inspect.html#the-interpreter-stack
import os
import hashlib
from absl import app, flags, logging
from absl.flags import FLAGS
import tensorflow as tf
import lxml.etree
import tqdm
flags.DEFINE_string(
"data_dir",
"./data/voc2012_raw/VOCdevkit/VOC2012/",
"path to raw PASCAL VOC dataset",
)
flags.DEFINE_enum("split", "train", ["train", "val"],
"specify train or val split")
flags.DEFINE_string("output_file", "./data/voc2012_train.tfrecord",
"output dataset")
flags.DEFINE_string("classes", "./data/voc2012.names", "classes file")
def build_example(annotation, class_map):
img_path = os.path.join(FLAGS.data_dir, "JPEGImages",
annotation["filename"])
img_raw = open(img_path, "rb").read()
key = hashlib.sha256(img_raw).hexdigest()
width = int(annotation["size"]["width"])
height = int(annotation["size"]["height"])
xmin = []
from absl import logging
import tensorflow as tf
from official.modeling import performance
from official.nlp import optimization
from official.nlp.bert import bert_models
from official.nlp.bert import common_flags
from official.nlp.bert import configs as bert_configs
from official.nlp.bert import input_pipeline
from official.nlp.bert import model_saving_utils
from official.nlp.bert import model_training_utils
from official.utils.misc import distribution_utils
from official.utils.misc import keras_utils
flags.DEFINE_enum(
'mode', 'train_and_eval', ['train_and_eval', 'export_only'],
'One of {"train_and_eval", "export_only"}. `train_and_eval`: '
'trains the model and evaluates in the meantime. '
'`export_only`: will take the latest checkpoint inside '
'model_dir and export a `SavedModel`.')
flags.DEFINE_string('train_data_path', None,
'Path to training data for BERT classifier.')
flags.DEFINE_string('eval_data_path', None,
'Path to evaluation data for BERT classifier.')
# Model training specific flags.
flags.DEFINE_string(
'input_meta_data_path', None,
'Path to file that contains meta data about input '
'to be used for training and evaluation.')
flags.DEFINE_integer('train_batch_size', 32, 'Batch size for training.')
flags.DEFINE_integer('eval_batch_size', 32, 'Batch size for evaluation.')
common_flags.define_common_bert_flags()
from kws_streaming.models import model_params
from kws_streaming.models import models
from kws_streaming.models import utils
FLAGS = flags.FLAGS
ALL_MODELS = list(model_params.HOTWORD_MODEL_PARAMS.keys())
MODELS_HELP = [f"'{name}'" for name in ALL_MODELS]
MODELS_HELP = f'{", ".join(MODELS_HELP[:-1])}, or {MODELS_HELP[-1]}'
flags.DEFINE_string(
'model', 'svdf', f'Name of the model to compile. Either {MODELS_HELP}.\n'
'See https://github.com/google-research/google-research/blob/master/kws_streaming/models/models.py#L38-L58'
)
flags.DEFINE_enum(
'mode', 'non_streaming',
['non_streaming', 'internal_streaming', 'external_streaming'],
'Mode to execute the model in.')
MODE_ENUM_TO_MODE = {
'non_streaming': modes.Modes.NON_STREAM_INFERENCE,
'internal_streaming': modes.Modes.STREAM_INTERNAL_STATE_INFERENCE,
'external_streaming': modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE,
}
class KeywordSpottingModule(tf.Module):
def __init__(self, model):
super().__init__()
self.m = model
self.m.predict = lambda x: self.m.call(x, training=False)
input_signature = [tf.TensorSpec(shape) for shape in self.input_shapes]
help='Project name for the Cloud TPU-enabled project. If not specified, we '
'will attempt to automatically detect the GCE project from metadata.')
flags.DEFINE_string(
'tpu_zone',
default=None,
help='GCE zone where the Cloud TPU is located in. If not specified, we '
'will attempt to automatically detect the GCE project from metadata.')
# Model specific paramenters
flags.DEFINE_string(
'eval_master',
default='',
help='GRPC URL of the eval master. Set to an appropriate value when running'
' on CPU/GPU')
flags.DEFINE_string('eval_name', default=None, help='Eval job name')
flags.DEFINE_enum('strategy', None, ['tpu', 'gpus', ''],
'Training: gpus for multi-gpu, if None, use TF default.')
flags.DEFINE_integer('num_cores',
default=8,
help='Number of TPU cores for training')
flags.DEFINE_bool('use_fake_data', False, 'Use fake input.')
flags.DEFINE_bool(
'use_xla', False,
'Use XLA even if strategy is not tpu. If strategy is tpu, always use XLA, '
'and this flag has no effect.')
flags.DEFINE_string('model_dir', None, 'Location of model_dir')
flags.DEFINE_string(
'hparams', '', 'Comma separated k=v pairs of hyperparameters or a module'
' containing attributes to use as hyperparameters.')
flags.DEFINE_integer('total_rounds', 200,
'Number of total training rounds.')
flags.DEFINE_integer(
'rounds_per_eval', 1,
'How often to evaluate the global model on the validation dataset.')
flags.DEFINE_integer('rounds_per_checkpoint', 50,
'How often to checkpoint the global model.')
with utils_impl.record_hparam_flags() as fed_pa_flags:
flags.DEFINE_integer(
'client_mixin_epochs_per_round', 1,
'The number of client epochs per federated round used for MCMC mixing.'
)
flags.DEFINE_enum(
'client_mixin_check_scheme', 'fixed_epochs', ['fixed_epochs'],
'The name of the scheme used for checking whether MCMC has mixed-in:\n'
'- fixed_epochs: assumes chains mix-in after a fixed number of epochs.'
)
flags.DEFINE_integer(
'client_mixin_check_start_round', 100,
'The round number starting which we start checking for MCMC mixin. '
'During all rounds before that we assume that clients are not mixed-in.'
)
flags.DEFINE_enum('client_update_delta_scheme', 'posterior_avg',
['simple_avg', 'posterior_avg'],
'The name of the scheme used to update weight deltas.')
flags.DEFINE_float(
'client_shrinkage_rho', 0.1,
'The hyperparameter of the shrinkage estimator of the posterior '
'covariance matrix.')
flags.DEFINE_boolean(
from absl import flags
from absl import logging
import tensorflow as tf
from sc2learner.agents.ppo_policies import LstmPolicy, MlpPolicy
from sc2learner.agents.ppo_agent import PPOActor, PPOLearner
from sc2learner.envs.raw_env import SC2RawEnv
from sc2learner.envs.rewards.reward_wrappers import KillingRewardWrapper
from sc2learner.envs.actions.zerg_action_wrappers import ZergActionWrapper
from sc2learner.envs.observations.zerg_observation_wrappers \
import ZergObservationWrapper
from sc2learner.utils.utils import print_arguments
FLAGS = flags.FLAGS
flags.DEFINE_enum("job_name", 'actor', ['actor', 'learner'], "Job type.")
flags.DEFINE_enum("policy", 'mlp', ['mlp', 'lstm'], "Job type.")
flags.DEFINE_integer("unroll_length", 128, "Length of rollout steps.")
flags.DEFINE_string("learner_ip", "localhost", "Learner IP address.")
flags.DEFINE_string("port_A", "5700", "Port for transporting model.")
flags.DEFINE_string("port_B", "5701", "Port for transporting data.")
flags.DEFINE_string("game_version", '4.6', "Game core version.")
flags.DEFINE_float("discount_gamma", 0.998, "Discount factor.")
flags.DEFINE_float("lambda_return", 0.95, "Lambda return factor.")
flags.DEFINE_float("clip_range", 0.1, "Clip range for PPO.")
flags.DEFINE_float("ent_coef", 0.01, "Coefficient for the entropy term.")
flags.DEFINE_float("vf_coef", 0.5, "Coefficient for the value loss.")
flags.DEFINE_float("learn_act_speed_ratio", 0, "Maximum learner/actor ratio.")
flags.DEFINE_integer("batch_size", 32, "Batch size.")
flags.DEFINE_integer("game_steps_per_episode", 43200, "Maximum steps per episode.")
flags.DEFINE_integer("learner_queue_size", 1024, "Size of learner's unroll queue.")
def define_flags():
"""Define the flags."""
# Cloud TPU Cluster Resolvers
flags.DEFINE_string(
'tpu',
default=None,
help='The Cloud TPU to use for training. This should be either the name '
'used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 '
'url.')
flags.DEFINE_string(
'gcp_project',
default=None,
help='Project name for the Cloud TPU-enabled project. If not specified, '
'we will attempt to automatically detect the GCE project from metadata.'
)
flags.DEFINE_string(
'tpu_zone',
default=None,
help='GCE zone where the Cloud TPU is located in. If not specified, we '
'will attempt to automatically detect the GCE project from metadata.')
# Model specific paramenters
flags.DEFINE_string(
'eval_master',
default='',
help='GRPC URL of the eval master. Set to an appropriate value when '
'running on CPU/GPU')
flags.DEFINE_string('eval_name', default=None, help='Eval job name')
flags.DEFINE_enum(
'strategy', '', ['tpu', 'gpus', ''],
'Training: gpus for multi-gpu, if None, use TF default.')
flags.DEFINE_integer('num_cores',
default=8,
help='Number of TPU cores for training')
flags.DEFINE_bool('use_fake_data', False, 'Use fake input.')
flags.DEFINE_bool(
'use_xla', False,
'Use XLA even if strategy is not tpu. If strategy is tpu, always use XLA,'
' and this flag has no effect.')
flags.DEFINE_string('model_dir', None, 'Location of model_dir')
flags.DEFINE_string('pretrained_ckpt', None,
'Start training from this EfficientDet checkpoint.')
flags.DEFINE_string(
'hparams', '',
'Comma separated k=v pairs of hyperparameters or a module'
' containing attributes to use as hyperparameters.')
flags.DEFINE_integer('batch_size', 64, 'training batch size')
flags.DEFINE_integer('eval_samples', 5000, 'The number of samples for '
'evaluation.')
flags.DEFINE_integer('steps_per_execution', 1,
'Number of steps per training execution.')
flags.DEFINE_string(
'train_file_pattern', None,
'Glob for train data files (e.g., COCO train - minival set)')
flags.DEFINE_string(
'val_file_pattern', None,
'Glob for evaluation tfrecords (e.g., COCO val2017 set)')
flags.DEFINE_string(
'val_json_file', None,
'COCO validation JSON containing golden bounding boxes. If None, use the '
'ground truth from the dataloader. Ignored if testdev_dir is not None.'
)
flags.DEFINE_enum('mode', 'traineval', ['train', 'traineval'],
'job mode: train, traineval.')
flags.DEFINE_string(
'hub_module_url', None, 'TF-Hub path/url to EfficientDet module.'
'If specified, pretrained_ckpt flag should not be used.')
flags.DEFINE_integer('num_examples_per_epoch', 120000,
'Number of examples in one epoch')
flags.DEFINE_integer('num_epochs', None, 'Number of epochs for training')
flags.DEFINE_string('model_name', 'efficientdet-d0', 'Model name.')
flags.DEFINE_bool('debug', False, 'Enable debug mode')
flags.DEFINE_integer(
'tf_random_seed', 111111,
'Fixed random seed for deterministic execution across runs for debugging.'
)
flags.DEFINE_bool('profile', False, 'Enable profile mode')
flags.DEFINE_string('training_data_filepattern', None,
'File pattern of the training data.')
flags.DEFINE_string('testing_data_filepattern', None,
'File pattern of the training data.')
flags.DEFINE_string('model_dir', None, 'Directory to store checkpoints.')
flags.DEFINE_string(
'params_path', None,
'Path to the json file containing params needed to run '
'p13n recommendation model.')
flags.DEFINE_integer('batch_size', 1, 'Training batch size.')
flags.DEFINE_float('learning_rate', 0.1, 'Learning rate.')
flags.DEFINE_integer('steps_per_epoch', 10,
'Number of steps to run in each epoch.')
flags.DEFINE_integer('num_epochs', 10000, 'Number of training epochs.')
flags.DEFINE_integer('num_eval_steps', 1000, 'Number of eval steps.')
flags.DEFINE_enum('run_mode', 'train_and_eval', ['train_and_eval', 'export'],
'Mode of the launcher, default value is: train_and_eval')
flags.DEFINE_float('gradient_clip_norm', 1.0,
'gradient_clip_norm <= 0 meaning no clip.')
flags.DEFINE_integer('max_history_length', 10, 'Max length of user history.')
flags.DEFINE_integer('num_predictions', 100,
'Num of top predictions to output.')
flags.DEFINE_string(
'encoder_type', 'bow', 'Type of the encoder for context'
'encoding, the value could be ["bow", "rnn", "cnn"].')
flags.DEFINE_string('checkpoint_path', '', 'Path to the checkpoint.')
CONTEXT = 'context'
LABEL = 'label'
class SimpleCheckpoint(tf.keras.callbacks.Callback):
import logging
from typing import Any, Dict, List
from absl import flags
from perfkitbenchmarker import benchmark_spec
from perfkitbenchmarker import configs
from perfkitbenchmarker import linux_packages
from perfkitbenchmarker import regex_util
from perfkitbenchmarker import sample
from perfkitbenchmarker import virtual_machine
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.linux_packages import cuda_toolkit
from perfkitbenchmarker.linux_packages import xgboost
_TREE_METHOD = flags.DEFINE_enum('xgboost_tree_method', 'gpu_hist',
['gpu_hist', 'hist'],
'XGBoost builtin tree methods.')
_SPARSITY = flags.DEFINE_float(
'xgboost_sparsity', 0.0, 'XGBoost sparsity-aware split finding algorithm.')
_ROWS = flags.DEFINE_integer('xgboost_rows', 1000000,
'The number of data rows.')
_COLUMNS = flags.DEFINE_integer('xgboost_columns', 50,
'The number of data columns.')
_ITERATIONS = flags.DEFINE_integer('xgboost_iterations', 500,
'The number of training iterations.')
_TEST_SIZE = flags.DEFINE_float(
'xgboost_test_size', 0.25,
'Train-test split for evaluating machine learning algorithms')
_PARAMS = flags.DEFINE_string(
'xgboost_params', None,
'Provide additional parameters as a Python dict string, '
from absl import flags
import os
import tensorflow as tf
from . import constants
from . import input_fn
from . import model
FLAGS = flags.FLAGS
# Model configurations.
flags.DEFINE_integer("embedding_size", 16, "Embedding size.")
flags.DEFINE_enum("encoder_cell_type", "LSTM",
["LSTM", "LayerNormBasicLSTM", "GRU"],
"Type of encoder cell to use.")
flags.DEFINE_integer("encoder_cell_size", 64, "Size of encoder cell.")
flags.DEFINE_enum("decoder_cell_type", "LSTM",
["LSTM", "LayerNormBasicLSTM", "GRU"],
"Type of decoder cell to use.")
flags.DEFINE_integer("decoder_cell_size", 64, "Size of decoder cell.")
flags.DEFINE_enum("optimizer_type", "Adam", ["Adam"],
"Type of optimizer to use.")
flags.DEFINE_float("learning_rate", 0.001, "Learning rate.")
flags.DEFINE_string(
"input_file", "../../data", "Path to raw input files."
"Assumes the filenames wiki.{train|valid|test}.raw")
flags.DEFINE_string("output_file", None, "Output TF example file.")
flags.DEFINE_string("vocab_file", "vocab.txt",
"The vocabulary file that the BERT model was trained on.")
flags.DEFINE_integer("max_seq_length", 512, "Maximum sequence length.")
flags.DEFINE_integer("max_sent_length", 70, "Maximum sequence length.")
flags.DEFINE_integer("max_para_length", 30, "Maximum sequence length.")
flags.DEFINE_integer("random_seed", 12345, "A random seed")
flags.DEFINE_bool(
"do_lower_case", True,
"Whether to lower case the input text. Should be True for uncased "
"models and False for cased models.")
flags.DEFINE_enum(
"format", FORMAT_BINARY, [FORMAT_BINARY, FORMAT_PARAGRAPH],
"Build a dataset of either binary order or paragraph reconstrucition")
FLAGS = flags.FLAGS
def read_file(filename):
"""Read the contents of filename (str) and split into documents by chapter."""
all_stories = []
tot = 0
with tf.gfile.GFile(filename, "r") as reader:
for line in reader:
line = line.strip()
if not line:
continue
def define_transformer_flags():
"""Add flags and flag validators for running transformer_main."""
# Add common flags (data_dir, model_dir, etc.).
flags_core.define_base(num_gpu=True, distribution_strategy=True)
flags_core.define_performance(num_parallel_calls=True,
inter_op=False,
intra_op=False,
synthetic_data=True,
max_train_steps=False,
dtype=True,
loss_scale=True,
all_reduce_alg=True,
num_packs=True,
tf_gpu_thread_mode=True,
datasets_num_private_threads=True,
enable_xla=True,
fp16_implementation=True)
flags_core.define_benchmark()
flags_core.define_device(tpu=True)
flags.DEFINE_integer(
name='train_steps',
short_name='ts',
default=300000,
help=flags_core.help_wrap('The number of steps used to train.'))
flags.DEFINE_integer(
name='steps_between_evals',
short_name='sbe',
default=5000,
help=flags_core.help_wrap(
'The Number of training steps to run between evaluations. This is '
'used if --train_steps is defined.'))
flags.DEFINE_boolean(name='enable_time_history',
default=True,
help='Whether to enable TimeHistory callback.')
flags.DEFINE_boolean(name='enable_tensorboard',
default=False,
help='Whether to enable Tensorboard callback.')
flags.DEFINE_boolean(name='enable_metrics_in_training',
default=False,
help='Whether to enable metrics during training.')
flags.DEFINE_boolean(name='enable_mlir_bridge',
default=False,
help='Whether to enable the TF to XLA bridge.')
# Set flags from the flags_core module as 'key flags' so they're listed when
# the '-h' flag is used. Without this line, the flags defined above are
# only shown in the full `--helpful` help text.
flags.adopt_module_key_flags(flags_core)
# Add transformer-specific flags
flags.DEFINE_enum(
name='param_set',
short_name='mp',
default='big',
enum_values=PARAMS_MAP.keys(),
help=flags_core.help_wrap(
'Parameter set to use when creating and training the model. The '
'parameters define the input shape (batch size and max length), '
'model configuration (size of embedding, # of hidden layers, etc.), '
'and various other settings. The big parameter set increases the '
'default batch size, embedding/hidden size, and filter size. For a '
'complete list of parameters, please see model/model_params.py.'))
flags.DEFINE_bool(
name='static_batch',
short_name='sb',
default=False,
help=flags_core.help_wrap(
'Whether the batches in the dataset should have static shapes. In '
'general, this setting should be False. Dynamic shapes allow the '
'inputs to be grouped so that the number of padding tokens is '
'minimized, and helps model training. In cases where the input shape '
'must be static (e.g. running on TPU), this setting will be ignored '
'and static batching will always be used.'))
flags.DEFINE_integer(
name='max_length',
short_name='ml',
default=256,
help=flags_core.help_wrap(
'Max sentence length for Transformer. Default is 256. Note: Usually '
'it is more effective to use a smaller max length if static_batch is '
'enabled, e.g. 64.'))
# Flags for training with steps (may be used for debugging)
flags.DEFINE_integer(
name='validation_steps',
short_name='vs',
default=64,
help=flags_core.help_wrap('The number of steps used in validation.'))
# BLEU score computation
flags.DEFINE_string(
name='bleu_source',
short_name='bls',
default=None,
help=flags_core.help_wrap(
'Path to source file containing text translate when calculating the '
'official BLEU score. Both --bleu_source and --bleu_ref must be set. '
))
flags.DEFINE_string(
name='bleu_ref',
short_name='blr',
default=None,
help=flags_core.help_wrap(
'Path to source file containing text translate when calculating the '
'official BLEU score. Both --bleu_source and --bleu_ref must be set. '
))
flags.DEFINE_string(
name='vocab_file',
short_name='vf',
default=None,
help=flags_core.help_wrap(
'Path to subtoken vocabulary file. If data_download.py was used to '
'download and encode the training data, look in the data_dir to find '
'the vocab file.'))
flags.DEFINE_string(
name='mode',
default='train',
help=flags_core.help_wrap('mode: train, eval, or predict'))
flags.DEFINE_bool(name='use_ctl',
default=False,
help=flags_core.help_wrap(
'Whether the model runs with custom training loop.'))
flags.DEFINE_integer(
name='decode_batch_size',
default=32,
help=flags_core.help_wrap(
'Global batch size used for Transformer autoregressive decoding on '
'TPU.'))
flags.DEFINE_integer(
name='decode_max_length',
default=97,
help=flags_core.help_wrap(
'Max sequence length of the decode/eval data. This is used by '
'Transformer autoregressive decoding on TPU to have minimum '
'paddings.'))
flags.DEFINE_bool(
name='padded_decode',
default=False,
help=flags_core.help_wrap(
'Whether the autoregressive decoding runs with input data padded to '
'the decode_max_length. For TPU/XLA-GPU runs, this flag has to be '
'set due the static shape requirement. Although CPU/GPU could also '
'use padded_decode, it has not been tested. In addition, this method '
'will introduce unnecessary overheads which grow quadratically with '
'the max sequence length.'))
flags.DEFINE_bool(
name='enable_checkpointing',
default=True,
help=flags_core.help_wrap(
'Whether to do checkpointing during training. When running under '
'benchmark harness, we will avoid checkpointing.'))
flags_core.set_defaults(data_dir='/tmp/translate_ende',
model_dir='/tmp/transformer_model',
batch_size=None)
# pylint: disable=unused-variable
@flags.multi_flags_validator(
['bleu_source', 'bleu_ref'],
message='Both or neither --bleu_source and --bleu_ref must be defined.'
)
def _check_bleu_files(flags_dict):
return (flags_dict['bleu_source'] is None) == (flags_dict['bleu_ref']
is None)
@flags.multi_flags_validator(
['bleu_source', 'bleu_ref', 'vocab_file'],
message='--vocab_file must be defined if --bleu_source and --bleu_ref '
'are defined.')
def _check_bleu_vocab_file(flags_dict):
if flags_dict['bleu_source'] and flags_dict['bleu_ref']:
return flags_dict['vocab_file'] is not None
return True
from tensorflow.python.tpu import feature_column
# pylint: enable=g-direct-tensorflow-import
from official.datasets import movielens
from official.recommendation import constants as rconst
from official.recommendation import data_pipeline
from official.recommendation import data_preprocessing
from official.recommendation import neumf_model
from official.utils.flags import core as flags_core
from official.utils.logs import hooks_helper
FLAGS = flags.FLAGS
flags.DEFINE_enum(
name="dataset", default="ml-20m",
enum_values=["ml-1m", "ml-20m"], case_sensitive=False,
help=flags_core.help_wrap(
"Dataset to be trained and evaluated."))
flags.DEFINE_string(
"data_dir", default=None,
help=("The directory where movielens data is stored."))
flags.DEFINE_integer(
"batch_size", default=2048*16, help="Batch size.")
flags.DEFINE_string(
"model_dir", default=None,
help=("The directory where the model and summaries are stored."))
flags.DEFINE_string(
import glob
import numpy as np
import skimage
from torch.utils.data import DataLoader
import configargparse
from functools import partial
import torch
import json
from losses import image_log_mse
import trainingMAML
# Define Flags.
flags.DEFINE_string('data_root', '/home/yannick', 'Root directory of data.')
flags.DEFINE_string('exp_root', 'exp/maml', 'Root directory of experiments.')
flags.DEFINE_enum('dataset', 'KODAK21', ['KODAK', 'KODAK21'],
'Dataset used during training.')
flags.DEFINE_enum('maml_dataset', 'KODAK', ['KODAK', 'KODAK21'],
'Dataset used for training MAML.')
flags.DEFINE_integer('batch_size',
1,
'Batch size used during training.',
lower_bound=1)
flags.DEFINE_integer('epochs',
10000,
'Maximum number of epochs.',
lower_bound=1)
flags.DEFINE_integer('maml_iterations',
1000,
'Maximum number of iterations for MAML training',
lower_bound=1)
flags.DEFINE_integer('maml_batch_size',
from absl import logging
import pandas as pd
import tensorflow.compat.v1 as tf
from pruning_identified_exemplars.utils import model_utils
FLAGS = flags.FLAGS
flags.DEFINE_integer("num_cores", default=8, help=("Number of cores."))
flags.DEFINE_string("data_directory", "",
"The location of the tfrecords used for training.")
flags.DEFINE_integer("batch_size", 1, "Batch size for creating new dataset.")
flags.DEFINE_string("output_path", "",
"Directory path to save the csv data to.")
flags.DEFINE_string("ckpt_dir", "", "Ckpt to extract predictions from.")
flags.DEFINE_enum("mode", "eval", ("eval", "train"),
"Mode designated as train or eval.")
flags.DEFINE_float("label_smoothing", 0.1,
"Relax confidence in the labels by (1-label_smoothing).")
# set this flag to true to do a test run of this code with synthetic data
flags.DEFINE_bool("test_small_sample", False,
"Boolean for whether to test internally.")
imagenet_params = {
"num_eval_images": 50000,
"num_label_classes": 1000,
"batch_size": 1,
"mean_rgb": [0.485 * 255, 0.456 * 255, 0.406 * 255],
"stddev_rgb": [0.229 * 255, 0.224 * 255, 0.225 * 255]
}
from absl import logging
from clu import platform
import jax
from lingvo.jax import eval as eval_lib
from lingvo.jax import py_utils
from lingvo.jax import train
import tensorflow.compat.v2 as tf
# Required import to setup work units when running through XManager.
FLAGS = flags.FLAGS
flags.DEFINE_string('model', None, 'Lingvo Jax model name.')
flags.DEFINE_string('job_log_dir', None,
'Directory where all experiment assets will be stored.')
flags.DEFINE_enum('mode', 'train', ['train', 'eval', 'decode', 'decode_once'],
'Flag to control which job is called.')
flags.DEFINE_bool(
'eval_on_test', False, 'If True, then the training loop '
'includes a full evaluation on all the test set splits. '
'This can be set to True if we do not want an additional job '
'to run continuous eval.')
flags.DEFINE_bool(
'multi_host_checkpointing', False,
'Whether to use multi-host checkpointing or not. Only useful for '
'multi-host SPMD models.')
flags.DEFINE_bool(
'maybe_use_persistence_checkpointing', False,
'If suitable, will try to rely on persistence-based checkpointing rather '
'than Flax-based checkpointing for SPMD models.')
flags.DEFINE_string(
'checkpoint_todelete_subdir', None,
from absl import app
from absl import flags
from icubam import config
import icubam.db.store as db_store
from icubam.db import synchronizer
flags.DEFINE_string("config", config.DEFAULT_CONFIG_PATH, "Config file.")
flags.DEFINE_string("dotenv_path", config.DEFAULT_DOTENV_PATH, "Config file.")
flags.DEFINE_enum("mode", "dev", ["prod", "dev"], "Run mode.")
flags.DEFINE_bool(
"force_update", False,
"Allow in-place modifications to already present elements."
)
flags.DEFINE_string("icus_csv", None, "Path to csv file containing ICU data.")
flags.DEFINE_string(
"users_csv", None, "Path to csv file containing user data."
)
FLAGS = flags.FLAGS
def main(args=None):
cfg = config.Config(
FLAGS.config, mode=FLAGS.mode, env_path=FLAGS.dotenv_path
)
store_factory = db_store.create_store_factory_for_sqlite_db(cfg)
store = store_factory.create()
csv = synchronizer.CSVSynchcronizer(store)
import orbit
from do_wide_and_deep_networks_learn_the_same_things.large_scale_training import single_task_evaluator
from do_wide_and_deep_networks_learn_the_same_things.large_scale_training import single_task_trainer
import scipy.special
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
FLAGS = flags.FLAGS
# Define training setup
flags.DEFINE_string('tpu', None, 'Address of TPU to use for training.')
flags.DEFINE_enum('mode', 'train_and_evaluate',
['train', 'evaluate', 'train_and_evaluate'],
'Execution mode.')
flags.DEFINE_integer(
'checkpoints_to_keep', 1,
'Number of checkpoints to keep, in addition to the best.')
EVAL_ACCURACY_KEY = 'accuracy/eval'
def get_summaries_from_dir(path):
"""Load summaries from event files in a directory."""
events_files = tf.io.gfile.glob(os.path.join(path,
'events.out.tfevents.*'))
values = collections.defaultdict(lambda: [])
global_step = collections.defaultdict(lambda: [])
for events_file in events_files:
