def main(unused_argv):
logging.set_verbosity(FLAGS.log)
if not os.path.exists(FLAGS.output_dir):
os.makedirs(FLAGS.output_dir)
for input_file in sorted(os.listdir(FLAGS.input_dir)):
if not input_file.endswith('.wav'):
continue
wav_filename = input_file
midi_filename = input_file.replace('.wav', '.mid')
logging.info('Aligning %s to %s', midi_filename, wav_filename)
samples = audio_io.load_audio(
os.path.join(FLAGS.input_dir, wav_filename), align_fine_lib.SAMPLE_RATE)
ns = midi_io.midi_file_to_sequence_proto(
os.path.join(FLAGS.input_dir, midi_filename))
aligned_ns, unused_stats = align_fine_lib.align_cpp(
samples,
align_fine_lib.SAMPLE_RATE,
ns,
align_fine_lib.CQT_HOP_LENGTH_FINE,
sf2_path=FLAGS.sf2_path,
penalty_mul=FLAGS.penalty_mul)
midi_io.sequence_proto_to_midi_file(
aligned_ns, os.path.join(FLAGS.output_dir, midi_filename))
logging.info('Done')
def main(argv):
del argv # Unused.
logging.set_verbosity(FLAGS.log_level)
flags.mark_flag_as_required('logdir')
if FLAGS.num_workers <= 0:
raise ValueError('num_workers flag must be greater than 0.')
if FLAGS.task_id < 0:
raise ValueError('task_id flag must be greater than or equal to 0.')
if FLAGS.task_id >= FLAGS.num_workers:
raise ValueError(
'task_id flag must be strictly less than num_workers flag.')
ns, _ = get_namespace(FLAGS.config)
ns.run_training(is_chief=FLAGS.task_id == 0)
def main(argv):
del argv
logging.set_verbosity(FLAGS.log_level)
if not FLAGS.logdir:
raise ValueError('logdir flag must be provided.')
if FLAGS.num_workers <= 0:
raise ValueError('num_workers flag must be greater than 0.')
if FLAGS.task_id < 0:
raise ValueError('task_id flag must be greater than or equal to 0.')
if FLAGS.task_id >= FLAGS.num_workers:
raise ValueError(
'task_id flag must be strictly less than num_workers flag.')
if FLAGS.num_tuners <= 0:
raise ValueError('num_tuners flag must be greater than 0.')
if FLAGS.tuner_id < 0:
raise ValueError('tuner_id flag must be greater than or equal to 0.')
if FLAGS.tuner_id >= FLAGS.num_tuners:
raise ValueError(
'tuner_id flag must be strictly less than num_tuners flag.')
ns, _ = run_lib.get_namespace(FLAGS.config)
run_tuner_loop(ns)
sleep_t *= 1.3
tot_sleep += sleep_t
step += 1
print('done')
sys.stdout.flush()
def main(_):
config = models.get_model_config(FLAGS.model, FLAGS.config_name)
model_class = models.get_model_class(FLAGS.model)
study = study_config(config)
client = initialize_client()
print('Study:')
pprint.pprint(create_study(client, study))
tune(client, model_class, config, FLAGS.ensemble_count)
print('All done. Study name:', study_name())
if __name__ == "__main__":
logger = logging.getLogger().setLevel(logging.WARNING)
absl_logging.set_verbosity(absl_logging.WARNING)
FLAGS, unparsed = parser.parse_known_args()
train.FLAGS = FLAGS
train.FLAGS.model_dir = ""
app.run(main=main, argv=[sys.argv[0]] + unparsed)
def main(_):
logging.set_verbosity(logging.INFO)
tf.enable_resource_variables()
train_eval(FLAGS.root_dir,
num_iterations=FLAGS.num_iterations,
eval_interval=FLAGS.eval_interval)
export_model,
FLAGS.saved_model_dir,
signatures=export_model.f.get_concrete_function(
tf.TensorSpec(shape=(None, None, None, 3), dtype=tf.uint8)))
export_model = tf.saved_model.load(FLAGS.saved_model_dir)
boxes, scores, classes, valid_len = export_model.f(imgs)
# Visualize results.
for i, img in enumerate(imgs):
length = valid_len[i]
img = inference.visualize_image(
img,
boxes[i].numpy()[:length],
classes[i].numpy().astype(np.int)[:length],
scores[i].numpy()[:length],
label_map=config.label_map,
min_score_thresh=config.nms_configs.score_thresh,
max_boxes_to_draw=config.nms_configs.max_output_size)
output_image_path = os.path.join(FLAGS.output_dir, str(i) + '.jpg')
Image.fromarray(img).save(output_image_path)
print('writing annotated image to %s' % output_image_path)
if __name__ == '__main__':
flags.mark_flag_as_required('image_path')
flags.mark_flag_as_required('output_dir')
flags.mark_flag_as_required('model_dir')
logging.set_verbosity(logging.ERROR)
app.run(main)
self.config = self.solver.config
self.mode = utils.EVAL
task_name = self.config['data']['task']['name']
self.task = registers.task[task_name](self.config, self.mode)
self.dataset = self.task.dataset(self.mode, 25, 0)
self.iterator = self.dataset.make_one_shot_iterator()
self.one_element = self.iterator.get_next()
'''
def tearDown(self):
''' tear down '''
def test_dataset(self):
''' dataset unittest'''
pass
'''
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
for _ in range(2):
output = sess.run(self.one_element)
logging.info(output)
logging.info("output: {} {}".format(output.shape, output.dtype))
'''
if __name__ == '__main__':
logging.set_verbosity(logging.DEBUG)
tf.test.main()
def _run_executor(args: argparse.Namespace, beam_args: List[str]) -> None:
"""Selects a particular executor and run it based on name.
Args:
args:
--executor_class_path: The import path of the executor class.
--json_serialized_invocation_args: Full JSON-serialized parameters for
this execution. See go/mp-alpha-placeholder for details.
beam_args: Optional parameter that maps to the optional_pipeline_args
parameter in the pipeline, which provides additional configuration options
for apache-beam and tensorflow.logging.
For more about the beam arguments please refer to:
https://cloud.google.com/dataflow/docs/guides/specifying-exec-params
"""
logging.set_verbosity(logging.INFO)
# Rehydrate inputs/outputs/exec_properties from the serialized metadata.
executor_input = pipeline_pb2.ExecutorInput()
json_format.Parse(args.json_serialized_invocation_args,
executor_input,
ignore_unknown_fields=True)
inputs_dict = executor_input.inputs.artifacts
outputs_dict = executor_input.outputs.artifacts
inputs_parameter = executor_input.inputs.parameters
name_from_id = {}
inputs = kubeflow_v2_entrypoint_utils.parse_raw_artifact_dict(
inputs_dict, name_from_id)
outputs = kubeflow_v2_entrypoint_utils.parse_raw_artifact_dict(
outputs_dict, name_from_id)
exec_properties = kubeflow_v2_entrypoint_utils.parse_execution_properties(
inputs_parameter)
logging.info(
'Executor %s do: inputs: %s, outputs: %s, exec_properties: %s',
args.executor_class_path, inputs, outputs, exec_properties)
executor_cls = import_utils.import_class_by_path(args.executor_class_path)
executor_context = base_executor.BaseExecutor.Context(
beam_pipeline_args=beam_args, unique_id='')
executor = executor_cls(executor_context)
logging.info('Starting executor')
executor.Do(inputs, outputs, exec_properties)
# TODO(b/169583143): Remove this workaround when TFX migrates to use str-typed
# id/name to identify artifacts.
# Convert ModelBlessing artifact to use managed MLMD resource name.
if (issubclass(executor_cls, evaluator_executor.Executor)
and constants.BLESSING_KEY in outputs):
# Parse the parent prefix for managed MLMD resource name.
kubeflow_v2_entrypoint_utils.refactor_model_blessing(
artifact_utils.get_single_instance(
outputs[constants.BLESSING_KEY]), name_from_id)
# Log the output metadata to a file. So that it can be picked up by MP.
metadata_uri = executor_input.outputs.output_file
executor_output = pipeline_pb2.ExecutorOutput()
for k, v in kubeflow_v2_entrypoint_utils.translate_executor_output(
outputs, name_from_id).items():
executor_output.artifacts[k].CopyFrom(v)
fileio.open(metadata_uri,
'wb').write(json_format.MessageToJson(executor_output))
def benchmark_log(self):
self._benchmark_np_and_tf_np_unary('log')
def benchmark_exp(self):
self._benchmark_np_and_tf_np_unary('exp')
def benchmark_tanh(self):
self._benchmark_np_and_tf_np_unary('tanh')
def benchmark_matmul(self):
sizes = [(2, 2), (10, 10), (100, 100), (200, 200), (1000, 1000)]
# Override repeat flag since this can be very slow.
repeats = [FLAGS.repeat] * 3 + [50, 10]
times = []
for size, repeat in zip(sizes, repeats):
x = np.random.uniform(size=size).astype(np.float32, copy=False)
name = '{}_{}'.format(self._get_name(), size)
times.append(
self._benchmark_np_and_tf_np(name,
'matmul', (x, x),
repeat=repeat))
self._print_times('matmul', sizes, times)
if __name__ == '__main__':
logging.set_verbosity(logging.WARNING)
tf.enable_v2_behavior()
tf.test.main()
def test_set_verbosity_strings(self):
old_level = logging.get_verbosity()
# Lowercase names.
logging.set_verbosity('debug')
self.assertEquals(logging.get_verbosity(), logging.DEBUG)
logging.set_verbosity('info')
self.assertEquals(logging.get_verbosity(), logging.INFO)
logging.set_verbosity('warning')
self.assertEquals(logging.get_verbosity(), logging.WARNING)
logging.set_verbosity('warn')
self.assertEquals(logging.get_verbosity(), logging.WARNING)
logging.set_verbosity('error')
self.assertEquals(logging.get_verbosity(), logging.ERROR)
logging.set_verbosity('fatal')
# Uppercase names.
self.assertEquals(logging.get_verbosity(), logging.FATAL)
logging.set_verbosity('DEBUG')
self.assertEquals(logging.get_verbosity(), logging.DEBUG)
logging.set_verbosity('INFO')
self.assertEquals(logging.get_verbosity(), logging.INFO)
logging.set_verbosity('WARNING')
self.assertEquals(logging.get_verbosity(), logging.WARNING)
logging.set_verbosity('WARN')
self.assertEquals(logging.get_verbosity(), logging.WARNING)
logging.set_verbosity('ERROR')
self.assertEquals(logging.get_verbosity(), logging.ERROR)
logging.set_verbosity('FATAL')
self.assertEquals(logging.get_verbosity(), logging.FATAL)
# Integers as strings.
logging.set_verbosity(str(logging.DEBUG))
self.assertEquals(logging.get_verbosity(), logging.DEBUG)
logging.set_verbosity(str(logging.INFO))
self.assertEquals(logging.get_verbosity(), logging.INFO)
logging.set_verbosity(str(logging.WARNING))
self.assertEquals(logging.get_verbosity(), logging.WARNING)
logging.set_verbosity(str(logging.ERROR))
self.assertEquals(logging.get_verbosity(), logging.ERROR)
logging.set_verbosity(str(logging.FATAL))
self.assertEquals(logging.get_verbosity(), logging.FATAL)
logging.set_verbosity(old_level)
from __future__ import absolute_import, division, print_function
import tensorflow as tf
from absl import logging
from tf_transformers.activations import get_activation
from tf_transformers.core import LegacyLayer
from tf_transformers.layers import OnDeviceEmbedding, SimplePositionEmbedding, T5LayerNormalization
from tf_transformers.layers.mask import CausalMask, SelfAttentionMask, prefix_mask
from tf_transformers.layers.transformer import TransformerT5
logging.set_verbosity("INFO")
class T5Encoder(LegacyLayer):
"""T5 based encoder / Decoder .
Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer
Authors: Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee,
Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu
Implementation of T5 in TF2.0
Paper: https://arxiv.org/abs/1910.10683
Official Code: https://github.com/google-research/text-to-text-transfer-transformer
"""
def __init__(
self,
config,
mask_mode="user_defined",
import os
import subprocess
import numpy as np
from absl import logging as alog
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
alog.set_verbosity(alog.ERROR)
print(tf.__version__)
fashion_mnist = keras.datasets.fashion_mnist
(train_images, train_labels), (test_images,
test_labels) = fashion_mnist.load_data()
# scale the values to 0.0 to 1.0
train_images = train_images / 255.0
test_images = test_images / 255.0
# reshape for feeding into the model
train_images = train_images.reshape(train_images.shape[0], 28, 28, 1)
test_images = test_images.reshape(test_images.shape[0], 28, 28, 1)
class_names = [
'T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt',
'Sneaker', 'Bag', 'Ankle boot'
]
print('\ntrain_images.shape: {}, of {}'.format(train_images.shape,
train_images.dtype))
print('test_images.shape: {}, of {}'.format(test_images.shape,
def test_batch_into_prototype(self):
# Silence expected warning for this test.
logging.set_verbosity(logging.ERROR)
values = [{
"v1": np.arange(i),
"v2": np.arange(10 - i)
} for i in range(10)]
batched = list(
data_loading.batch_and_pad_to_prototype(
values, (1, 2), {
"v1": jax.ShapeDtypeStruct((8, ), np.int32),
"v2": jax.ShapeDtypeStruct((8, ), np.int32)
},
remainder_behavior=data_loading.BatchRemainderBehavior.
PAD_ZERO,
drop_too_large=True))
expected = [
{
"v1":
np.array([[
[0, 1, 0, 0, 0, 0, 0, 0],
[0, 1, 2, 0, 0, 0, 0, 0],
]]),
"v2":
np.array([[
[0, 1, 2, 3, 4, 5, 6, 7],
[0, 1, 2, 3, 4, 5, 6, 0],
]]),
},
{
"v1":
np.array([[
[0, 1, 2, 3, 0, 0, 0, 0],
[0, 1, 2, 3, 4, 0, 0, 0],
]]),
"v2":
np.array([[
[0, 1, 2, 3, 4, 5, 0, 0],
[0, 1, 2, 3, 4, 0, 0, 0],
]]),
},
{
"v1":
np.array([[
[0, 1, 2, 3, 4, 5, 0, 0],
[0, 1, 2, 3, 4, 5, 6, 0],
]]),
"v2":
np.array([[
[0, 1, 2, 3, 0, 0, 0, 0],
[0, 1, 2, 0, 0, 0, 0, 0],
]]),
},
{
"v1":
np.array([[
[0, 1, 2, 3, 4, 5, 6, 7],
[0, 0, 0, 0, 0, 0, 0, 0],
]]),
"v2":
np.array([[
[0, 1, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0],
]]),
},
]
jax.test_util.check_eq(batched, expected)
def main(_):
os.environ['SDL_VIDEODRIVER'] = 'dummy'
tf.compat.v1.enable_v2_behavior()
logging.set_verbosity(logging.INFO)
root_dir = os.path.expanduser(FLAGS.root_dir)
def _process_list_flag(flag_val):
if 'xm_parameters' in FLAGS and isinstance(flag_val, str):
return flag_val.split(',')
return flag_val
actor_net_layer_sizes = [
int(lsize) for lsize in _process_list_flag(FLAGS.actor_net_size)
]
if FLAGS.critic_net_size == ['-1'] or FLAGS.critic_net_size == '-1':
critic_net_layer_sizes = actor_net_layer_sizes.copy()
else:
critic_net_layer_sizes = [
int(lsize) for lsize in _process_list_flag(FLAGS.critic_net_size)
]
if FLAGS.env_name == 'point_mass':
env_load_fn = functools.partial(
get_env,
env_type=FLAGS.point_mass_env_type,
reward_type=FLAGS.reward_type)
elif FLAGS.env_name == 'sawyer_door':
env_load_fn = functools.partial(get_env, reward_type=FLAGS.reward_type)
if FLAGS.agent_type == 'sac':
initial_collect_steps = 10000
replay_buffer_capacity = FLAGS.replay_buffer_capacity
target_update_tau = 0.005
target_update_period = 1
td_errors_loss_fn = tf.math.squared_difference
elif FLAGS.agent_type == 'td3':
initial_collect_steps = 10000
replay_buffer_capacity = FLAGS.replay_buffer_capacity
target_update_tau = 0.05
target_update_period = 5
td_errors_loss_fn = tf.compat.v1.losses.huber_loss
train_eval(
root_dir,
random_seed=FLAGS.random_seed,
env_name=FLAGS.env_name,
env_load_fn=env_load_fn,
max_episode_steps=FLAGS.max_episode_steps,
eval_episode_steps=FLAGS.eval_episode_steps,
video_record_interval=FLAGS.video_record_interval,
num_videos=FLAGS.num_videos_per_interval,
num_iterations=FLAGS.num_iterations,
actor_fc_layers=tuple(actor_net_layer_sizes),
critic_joint_fc_layers=tuple(critic_net_layer_sizes),
reward_scale_factor=FLAGS.reward_scale_factor,
actor_learning_rate=FLAGS.actor_learning_rate,
critic_learning_rate=FLAGS.critic_learning_rate,
alpha_learning_rate=FLAGS.alpha_learning_rate,
use_minimum=bool(FLAGS.use_minimum),
reset_lagrange_learning_rate=FLAGS.reset_lagrange_learning_rate,
value_threshold=FLAGS.value_threshold,
reset_goal_frequency=FLAGS.reset_goal_frequency,
batch_size=FLAGS.batch_size,
initial_collect_steps=initial_collect_steps,
collect_steps_per_iteration=FLAGS.collect_steps_per_iteration,
train_steps_per_iteration=FLAGS.train_steps_per_iteration,
replay_buffer_capacity=replay_buffer_capacity,
exploration_noise_std=FLAGS.exploration_noise_std,
target_update_tau=target_update_tau,
target_update_period=target_update_period,
gamma=FLAGS.discount_factor,
td_errors_loss_fn=td_errors_loss_fn,
)
def main(_):
logging.set_verbosity(logging.INFO)
processors = {
"mnli_matched": MnliMatchedProcessor,
"mnli_mismatched": MnliMismatchedProcessor,
"sts-b": StsbProcessor,
"imdb": ImdbProcessor,
"yelp5": Yelp5Processor
}
task_name = FLAGS.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels() if not FLAGS.is_regression else None
sp = spm.SentencePieceProcessor()
sp.Load(FLAGS.spiece_model_file)
def tokenize_fn(text):
text = preprocess_utils.preprocess_text(text, lower=FLAGS.uncased)
return preprocess_utils.encode_ids(sp, text)
spm_basename = os.path.basename(FLAGS.spiece_model_file)
train_file_base = "{}.len-{}.train.tf_record".format(spm_basename,
FLAGS.max_seq_length)
train_file = os.path.join(FLAGS.output_dir, train_file_base)
logging.info("Use tfrecord file %s", train_file)
train_examples = processor.get_train_examples(FLAGS.data_dir)
np.random.shuffle(train_examples)
logging.info("Num of train samples: %d", len(train_examples))
file_based_convert_examples_to_features(train_examples, label_list,
FLAGS.max_seq_length, tokenize_fn,
train_file, FLAGS.num_passes)
if FLAGS.eval_split == "dev":
eval_examples = processor.get_dev_examples(FLAGS.data_dir)
else:
eval_examples = processor.get_test_examples(FLAGS.data_dir)
logging.info("Num of eval samples: %d", len(eval_examples))
# TPU requires a fixed batch size for all batches, therefore the number
# of examples must be a multiple of the batch size, or else examples
# will get dropped. So we pad with fake examples which are ignored
# later on. These do NOT count towards the metric (all tf.metrics
# support a per-instance weight, and these get a weight of 0.0).
#
# Modified in XL: We also adopt the same mechanism for GPUs.
while len(eval_examples) % FLAGS.eval_batch_size != 0:
eval_examples.append(classifier_utils.PaddingInputExample())
eval_file_base = "{}.len-{}.{}.eval.tf_record".format(spm_basename,
FLAGS.max_seq_length,
FLAGS.eval_split)
eval_file = os.path.join(FLAGS.output_dir, eval_file_base)
file_based_convert_examples_to_features(eval_examples, label_list,
FLAGS.max_seq_length, tokenize_fn,
eval_file)
def main(_):
logging.set_verbosity(logging.INFO)
args = flags.FLAGS.flag_values_dict()
if (
os.path.exists(args["output_dir"])
and os.listdir(args["output_dir"])
and args["do_train"]
and not args["overwrite_output_dir"]
):
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
args["output_dir"]
)
)
if args["fp16"]:
tf.config.optimizer.set_experimental_options({"auto_mixed_precision": True})
if args["tpu"]:
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=args["tpu"])
tf.config.experimental_connect_to_cluster(resolver)
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.experimental.TPUStrategy(resolver)
args["n_device"] = args["num_tpu_cores"]
elif len(args["gpus"].split(",")) > 1:
args["n_device"] = len([f"/gpu:{gpu}" for gpu in args["gpus"].split(",")])
strategy = tf.distribute.MirroredStrategy(devices=[f"/gpu:{gpu}" for gpu in args["gpus"].split(",")])
elif args["no_cuda"]:
args["n_device"] = 1
strategy = tf.distribute.OneDeviceStrategy(device="/cpu:0")
else:
args["n_device"] = len(args["gpus"].split(","))
strategy = tf.distribute.OneDeviceStrategy(device="/gpu:" + args["gpus"].split(",")[0])
logging.warning(
"n_device: %s, distributed training: %s, 16-bits training: %s",
args["n_device"],
bool(args["n_device"] > 1),
args["fp16"],
)
labels = get_labels(args["labels"])
num_labels = len(labels)
pad_token_label_id = -1
config = AutoConfig.from_pretrained(
args["config_name"] if args["config_name"] else args["model_name_or_path"],
num_labels=num_labels,
cache_dir=args["cache_dir"] if args["cache_dir"] else None,
)
logging.info("Training/evaluation parameters %s", args)
# Training
if args["do_train"]:
tokenizer = AutoTokenizer.from_pretrained(
args["tokenizer_name"] if args["tokenizer_name"] else args["model_name_or_path"],
do_lower_case=args["do_lower_case"],
cache_dir=args["cache_dir"] if args["cache_dir"] else None,
)
with strategy.scope():
model = TFAutoModelForTokenClassification.from_pretrained(
args["model_name_or_path"],
from_pt=bool(".bin" in args["model_name_or_path"]),
config=config,
cache_dir=args["cache_dir"] if args["cache_dir"] else None,
)
train_batch_size = args["per_device_train_batch_size"] * args["n_device"]
print('** train_batch_size = %s' % train_batch_size)
train_dataset, num_train_examples = load_and_cache_examples(
args, tokenizer, labels, pad_token_label_id, train_batch_size, mode="train"
)
train_dataset = strategy.experimental_distribute_dataset(train_dataset)
print('** num_train_examples = %s' % num_train_examples)
train(
args,
strategy,
train_dataset,
tokenizer,
model,
num_train_examples,
labels,
train_batch_size,
pad_token_label_id,
)
if not os.path.exists(args["output_dir"]):
os.makedirs(args["output_dir"])
logging.info("Saving model to %s", args["output_dir"])
model.save_pretrained(args["output_dir"])
tokenizer.save_pretrained(args["output_dir"])
# Evaluation
if args["do_eval"]:
tokenizer = AutoTokenizer.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
checkpoints = []
results = []
if args["eval_all_checkpoints"]:
checkpoints = list(
os.path.dirname(c)
for c in sorted(
glob.glob(args["output_dir"] + "/**/" + TF2_WEIGHTS_NAME, recursive=True),
key=lambda f: int("".join(filter(str.isdigit, f)) or -1),
)
)
logging.info("Evaluate the following checkpoints: %s", checkpoints)
if len(checkpoints) == 0:
checkpoints.append(args["output_dir"])
for checkpoint in checkpoints:
global_step = checkpoint.split("-")[-1] if re.match(".*checkpoint-[0-9]", checkpoint) else "final"
with strategy.scope():
model = TFAutoModelForTokenClassification.from_pretrained(checkpoint)
y_true, y_pred, eval_loss = evaluate(
args, strategy, model, tokenizer, labels, pad_token_label_id, mode="dev"
)
report = metrics.classification_report(y_true, y_pred, digits=4)
if global_step:
results.append({global_step + "_report": report, global_step + "_loss": eval_loss})
output_eval_file = os.path.join(args["output_dir"], "eval_results.txt")
with tf.io.gfile.GFile(output_eval_file, "w") as writer:
for res in results:
for key, val in res.items():
if "loss" in key:
logging.info(key + " = " + str(val))
writer.write(key + " = " + str(val))
writer.write("\n")
else:
logging.info(key)
logging.info("\n" + report)
writer.write(key + "\n")
writer.write(report)
writer.write("\n")
if args["do_predict"]:
tokenizer = AutoTokenizer.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
model = TFAutoModelForTokenClassification.from_pretrained(args["output_dir"])
eval_batch_size = args["per_device_eval_batch_size"] * args["n_device"]
predict_dataset, _ = load_and_cache_examples(
args, tokenizer, labels, pad_token_label_id, eval_batch_size, mode="test"
)
y_true, y_pred, pred_loss = evaluate(args, strategy, model, tokenizer, labels, pad_token_label_id, mode="test")
output_test_results_file = os.path.join(args["output_dir"], "test_results.txt")
output_test_predictions_file = os.path.join(args["output_dir"], "test_predictions.txt")
report = metrics.classification_report(y_true, y_pred, digits=4)
with tf.io.gfile.GFile(output_test_results_file, "w") as writer:
report = metrics.classification_report(y_true, y_pred, digits=4)
logging.info("\n" + report)
writer.write(report)
writer.write("\n\nloss = " + str(pred_loss))
with tf.io.gfile.GFile(output_test_predictions_file, "w") as writer:
with tf.io.gfile.GFile(os.path.join(args["data_dir"], "test.txt"), "r") as f:
example_id = 0
for line in f:
if line.startswith("-DOCSTART-") or line == "" or line == "\n":
writer.write(line)
if not y_pred[example_id]:
example_id += 1
elif y_pred[example_id]:
output_line = line.split()[0] + " " + y_pred[example_id].pop(0) + "\n"
writer.write(output_line)
else:
logging.warning("Maximum sequence length exceeded: No prediction for '%s'.", line.split()[0])
# -*- coding: utf-8 -*-
from textqa.model.CQA.tbqa import TBQA
from textqa.model.CQA.scripts.evaluate import Evaluate
from textqa.model.CQA import args
from absl import logging
import warnings
if __name__ == '__main__':
# 设置日志级别
logging.set_verbosity('error')
warnings.filterwarnings('ignore')
args.cat_threshold = 0.8
while args.cat_threshold > 0:
print('cat_threshold:', args.cat_threshold)
tbqa = TBQA()
tbqa.get_multi_answers()
evaluator = Evaluate()
evaluator.evaluate()
args.cat_threshold -= 0.1
def main(_):
logging.set_verbosity(logging.INFO)
args = flags.FLAGS.flag_values_dict()
if args["fp16"]:
tf.config.optimizer.set_experimental_options(
{"auto_mixed_precision": True})
if args["tpu"]:
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
tpu=args["tpu"])
tf.config.experimental_connect_to_cluster(resolver)
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.experimental.TPUStrategy(resolver)
args["n_device"] = args["num_tpu_cores"]
elif len(args["gpus"].split(",")) > 1:
args["n_device"] = len(
[f"/gpu:{gpu}" for gpu in args["gpus"].split(",")])
strategy = tf.distribute.MirroredStrategy(
devices=[f"/gpu:{gpu}" for gpu in args["gpus"].split(",")])
elif args["no_cuda"]:
args["n_device"] = 1
strategy = tf.distribute.OneDeviceStrategy(device="/cpu:0")
else:
args["n_device"] = len(args["gpus"].split(","))
strategy = tf.distribute.OneDeviceStrategy(device="/gpu:" +
args["gpus"].split(",")[0])
logging.warning(
"n_device: %s, distributed training: %s, 16-bits training: %s",
args["n_device"],
bool(args["n_device"] > 1),
args["fp16"],
)
labels = get_labels(args["labels"])
pad_token_label_id = -1
logging.info("predict parameters %s", args)
tokenizer = AutoTokenizer.from_pretrained(
args["output_dir"], do_lower_case=args["do_lower_case"])
model = TFAutoModelForTokenClassification.from_pretrained(
args["output_dir"])
while True:
print('Input chinese sentence:')
line = str(input())
if line == 'quit':
break
if len(line) < 1:
print(
'Please input a chinese sentence or "quit" to break this loop:'
)
continue
examples = read_examples_from_line(line)
features = convert_examples_to_features(
examples,
labels,
args["max_seq_length"],
tokenizer,
cls_token_at_end=bool(args["model_type"] in ["xlnet"]),
# xlnet has a cls token at the end
cls_token=tokenizer.cls_token,
cls_token_segment_id=2 if args["model_type"] in ["xlnet"] else 0,
sep_token=tokenizer.sep_token,
sep_token_extra=bool(args["model_type"] in ["roberta"]),
# roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805
pad_on_left=bool(args["model_type"] in ["xlnet"]),
# pad on the left for xlnet
pad_token=tokenizer.pad_token_id,
pad_token_segment_id=tokenizer.pad_token_type_id,
pad_token_label_id=pad_token_label_id,
)
feature = features[0]
X = collections.OrderedDict()
X["input_ids"] = tf.train.Feature(int64_list=tf.train.Int64List(
value=list(feature.input_ids)))
X["input_mask"] = tf.train.Feature(int64_list=tf.train.Int64List(
value=list(feature.input_mask)))
X["segment_ids"] = tf.train.Feature(int64_list=tf.train.Int64List(
value=list(feature.segment_ids)))
X["label_ids"] = tf.train.Feature(int64_list=tf.train.Int64List(
value=list(feature.label_ids)))
tf_example = tf.train.Example(features=tf.train.Features(feature=X))
tf_example = tf_example.SerializeToString()
max_seq_length = args["max_seq_length"]
name_to_features = {
"input_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"label_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64),
}
def _decode_record(record):
example = tf.io.parse_single_example(record, name_to_features)
features = {}
features["input_ids"] = example["input_ids"]
features["input_mask"] = example["input_mask"]
features["segment_ids"] = example["segment_ids"]
return features, example["label_ids"]
dataset = []
dataset.append(tf_example)
dataset = tf.data.Dataset.from_tensor_slices(dataset)
dataset = dataset.map(_decode_record)
batch_size = 1
dataset = dataset.batch(batch_size)
eval_features, eval_labels = iter(dataset).next()
inputs = {
"attention_mask": eval_features["input_mask"],
"training": False
}
if args["model_type"] != "distilbert":
inputs["token_type_ids"] = (eval_features["segment_ids"]
if args["model_type"]
in ["bert", "xlnet"] else None)
with strategy.scope():
logits = model(eval_features["input_ids"], **inputs)[0]
active_loss = tf.reshape(eval_labels, (-1, )) != pad_token_label_id
preds = logits.numpy()
label_ids = eval_labels.numpy()
preds = np.argmax(preds, axis=2)
y_pred = [[] for _ in range(label_ids.shape[0])]
for i in range(label_ids.shape[0]):
for j in range(label_ids.shape[1]):
if label_ids[i, j] != pad_token_label_id:
y_pred[i].append(labels[preds[i, j]])
tokens = tokenizer.tokenize(line)
print('## tokens = %s' % tokens)
print('## y_pred = %s' % y_pred)
print('## %s = %s' % (len(tokens), len(y_pred[0])))
word_group = []
subword = {}
def _add_word(subword):
word_group.append(subword['token'] + '/' + subword['flag'])
subword.clear()
for i, token in enumerate(tokens):
flag = y_pred[0][i]
print('## %s = %s' % (token, flag))
if flag.startswith('B'):
if len(subword) > 0:
_add_word(subword)
subword['token'] = token
subword['flag'] = flag
elif flag.startswith('I'):
if (len(subword) > 0 and (y_pred[0][i - 1].startswith('I')
or y_pred[0][i - 1].startswith('B'))
and (y_pred[0][i - 1][1:] == flag[1:])):
subword['token'] = subword['token'] + token
continue
elif len(subword) > 0:
_add_word(subword)
subword['token'] = token
subword['flag'] = flag
else:
if len(subword) > 0:
_add_word(subword)
subword['token'] = token
subword['flag'] = flag
_add_word(subword)
if len(subword) > 0:
_add_word(subword)
print('## word_group = %s' % word_group)
def __init__(self,
model_name: str,
uri: str,
hparams: str = '',
model_dir: Optional[str] = None,
epochs: int = 50,
batch_size: int = 64,
steps_per_execution: int = 1,
moving_average_decay: int = 0,
var_freeze_expr: str = '(efficientnet|fpn_cells|resample_p6)',
tflite_max_detections: int = 25,
strategy: Optional[str] = None,
tpu: Optional[str] = None,
gcp_project: Optional[str] = None,
tpu_zone: Optional[str] = None,
use_xla: bool = False,
profile: bool = False,
debug: bool = False,
tf_random_seed: int = 111111) -> None:
"""Initialze an instance with model paramaters.
Args:
model_name: Model name.
uri: TF-Hub path/url to EfficientDet module.
hparams: Hyperparameters used to overwrite default configuration. Can be
1) Dict, contains parameter names and values; 2) String, Comma separated
k=v pairs of hyperparameters; 3) String, yaml filename which's a module
containing attributes to use as hyperparameters.
model_dir: The location to save the model checkpoint files.
epochs: Default training epochs.
batch_size: Training & Evaluation batch size.
steps_per_execution: Number of steps per training execution.
moving_average_decay: Float. The decay to use for maintaining moving
averages of the trained parameters.
var_freeze_expr: Expression to freeze variables.
tflite_max_detections: The max number of output detections in the TFLite
model.
strategy: A string specifying which distribution strategy to use.
Accepted values are 'tpu', 'gpus', None. tpu' means to use TPUStrategy.
'gpus' mean to use MirroredStrategy for multi-gpus. If None, use TF
default with OneDeviceStrategy.
tpu: The Cloud TPU to use for training. This should be either the name
used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470
url.
gcp_project: Project name for the Cloud TPU-enabled project. If not
specified, we will attempt to automatically detect the GCE project from
metadata.
tpu_zone: GCE zone where the Cloud TPU is located in. If not specified, we
will attempt to automatically detect the GCE project from metadata.
use_xla: Use XLA even if strategy is not tpu. If strategy is tpu, always
use XLA, and this flag has no effect.
profile: Enable profile mode.
debug: Enable debug mode.
tf_random_seed: Fixed random seed for deterministic execution across runs
for debugging.
"""
self.model_name = model_name
self.uri = uri
self.batch_size = batch_size
config = hparams_config.get_efficientdet_config(model_name)
config.override(hparams)
config.image_size = utils.parse_image_size(config.image_size)
config.var_freeze_expr = var_freeze_expr
config.moving_average_decay = moving_average_decay
config.tflite_max_detections = tflite_max_detections
if epochs:
config.num_epochs = epochs
if use_xla and strategy != 'tpu':
tf.config.optimizer.set_jit(True)
for gpu in tf.config.list_physical_devices('GPU'):
tf.config.experimental.set_memory_growth(gpu, True)
if debug:
tf.config.experimental_run_functions_eagerly(True)
tf.debugging.set_log_device_placement(True)
os.environ['TF_DETERMINISTIC_OPS'] = '1'
tf.random.set_seed(tf_random_seed)
logging.set_verbosity(logging.DEBUG)
if strategy == 'tpu':
tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
tpu, zone=tpu_zone, project=gcp_project)
tf.config.experimental_connect_to_cluster(tpu_cluster_resolver)
tf.tpu.experimental.initialize_tpu_system(tpu_cluster_resolver)
ds_strategy = tf.distribute.TPUStrategy(tpu_cluster_resolver)
logging.info('All devices: %s', tf.config.list_logical_devices('TPU'))
tf.config.set_soft_device_placement(True)
elif strategy == 'gpus':
ds_strategy = tf.distribute.MirroredStrategy()
logging.info('All devices: %s', tf.config.list_physical_devices('GPU'))
else:
if tf.config.list_physical_devices('GPU'):
ds_strategy = tf.distribute.OneDeviceStrategy('device:GPU:0')
else:
ds_strategy = tf.distribute.OneDeviceStrategy('device:CPU:0')
self.ds_strategy = ds_strategy
if model_dir is None:
model_dir = tempfile.mkdtemp()
params = dict(
profile=profile,
model_name=model_name,
steps_per_execution=steps_per_execution,
model_dir=model_dir,
strategy=strategy,
batch_size=batch_size,
tf_random_seed=tf_random_seed,
debug=debug)
config.override(params, True)
self.config = config
# set mixed precision policy by keras api.
precision = utils.get_precision(config.strategy, config.mixed_precision)
policy = tf.keras.mixed_precision.experimental.Policy(precision)
tf.keras.mixed_precision.experimental.set_policy(policy)
def main(_):
logging.set_verbosity(logging.INFO)
data_dir = download_demo_data()
run(data_dir, FLAGS.export_dir, spec=FLAGS.spec)
def main(_):
tf.compat.v1.enable_v2_behavior()
logging.set_verbosity(logging.INFO)
gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_param)
train_eval(FLAGS.root_dir)
def main(_): logging.set_verbosity(logging.INFO) data_dir = download_demo_data() run(data_dir, FLAGS.tflite_filename, FLAGS.label_filename, epochs=10)
parser.add_argument("--logdir", default=None, help="Path to log directory")
parser.add_argument("--export_only",
default=False,
action="store_true",
help="Do not train, only export the model")
parser.add_argument("--config",
default="config/config.yaml",
help="path to config file")
parser.add_argument(
"--datadir",
default=None,
help="Path to custom data directory containing sharded TFRecords")
parser.add_argument("--modeldir",
default=None,
help="directory to store checkpoints and SavedModel")
parser.add_argument(
"--type",
default=None,
help="Train Student 'adversarial'-ly / 'comparative'-ly")
parser.add_argument("--verbose",
"-v",
default=0,
action="count",
help="Increases Verbosity. Repeat to increase more")
FLAGS, unparsed = parser.parse_known_args()
log_levels = [logging.FATAL, logging.WARNING, logging.INFO, logging.DEBUG]
log_level = log_levels[min(FLAGS.verbose, len(log_levels) - 1)]
logging.set_verbosity(log_level)
train_and_export(**vars(FLAGS))
def main(_):
logging.set_verbosity(logging.INFO)
agent_class = dqn_agent.DdqnAgent if FLAGS.use_ddqn else dqn_agent.DqnAgent
train_eval(FLAGS.root_dir,
agent_class=agent_class,
num_iterations=FLAGS.num_iterations)
def main(_):
logging.set_verbosity(logging.INFO)
tf.compat.v1.enable_resource_variables()
train_eval(FLAGS.root_dir, num_iterations=FLAGS.num_iterations)
def main(_):
logging.set_verbosity(logging.INFO)
validate_flags()
tf.io.gfile.makedirs(FLAGS.output_dir)
model_config = config.get_model_config(
model_dir=FLAGS.output_dir,
source_file=FLAGS.source_model_config_file,
source_base64=FLAGS.source_model_config_base64,
write_from_source=FLAGS.do_train)
input_files = []
for input_pattern in FLAGS.input_file.split(","):
input_files.extend(tf.io.gfile.glob(input_pattern))
num_blocks_per_example, block_length = input_utils.get_block_params_from_input_file(
input_files[0])
max_num_annotations = None
if FLAGS.mlm_entity_fraction_to_mask is not None:
max_num_annotations = input_utils.get_num_annotations_from_input_file(
input_files[0])
# TODO(urikz): Define `pretrain_input_utils`.
input_config = pretrain_input_utils.PretrainInputConfig( # pylint: disable=undefined-variable
num_blocks_per_example=num_blocks_per_example,
block_length=block_length,
mlm_fraction_to_mask=FLAGS.mlm_fraction_to_mask,
mlm_max_consecutive_masks=FLAGS.mlm_max_consecutive_masks,
mlm_use_whole_word=FLAGS.mlm_use_whole_word,
mask_token_id=FLAGS.mask_token_id,
padding_token_id=FLAGS.padding_token_id,
max_num_annotations=max_num_annotations,
mlm_entity_fraction_to_mask=FLAGS.mlm_entity_fraction_to_mask,
mention_mask_mode=FLAGS.mention_mask_mode)
logging.info("*** Input Files ***")
for input_file in input_files:
logging.info(" %s", input_file)
tpu_cluster_resolver, num_tpu_tasks, num_tpu_cores = None, None, None
if FLAGS.use_tpu and FLAGS.tpu_name:
tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
tpu_system_metadata = tpu_cluster_resolver.get_tpu_system_metadata()
num_tpu_tasks = tpu_cluster_resolver.get_tpu_system_metadata(
).num_hosts
num_tpu_cores = tpu_system_metadata.num_cores
# PER_HOST_V1: iterator.get_next() is called 1 time with per_worker_batch_size
# PER_HOST_V2: iterator.get_next() is called 8 times with per_core_batch_size
# pylint: enable=line-too-long
is_per_host = tf_estimator.tpu.InputPipelineConfig.PER_HOST_V1
run_config = tf_estimator.tpu.RunConfig(
cluster=tpu_cluster_resolver,
master=FLAGS.master,
model_dir=FLAGS.output_dir,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
tpu_config=tf_estimator.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
tpu_job_name=FLAGS.tpu_job_name,
per_host_input_for_training=is_per_host,
experimental_host_call_every_n_steps=FLAGS.iterations_per_loop))
batch_size = (num_tpu_tasks or 1) * num_blocks_per_example
logging.info("***** Input configuration *****")
logging.info(" Number of blocks per example = %d", num_blocks_per_example)
logging.info(" Block length = %d", block_length)
logging.info(" Number of TPU tasks = %d", num_tpu_tasks or 0)
logging.info(" Batch size = %d", batch_size)
logging.info(" Number of TPU cores = %d", num_tpu_cores or 0)
logging.info(" Number of annotations per example = %d",
input_config.max_num_annotations or 0)
model_fn = model_fn_builder(
model_config=model_config,
padding_token_id=FLAGS.padding_token_id,
enable_side_inputs=FLAGS.enable_side_inputs,
num_replicas_concat=num_tpu_cores,
cross_block_attention_mode=FLAGS.cross_block_attention_mode,
extra_loss=parse_extra_loss_flag(FLAGS.extra_loss),
summary_num_layers=FLAGS.summary_num_layers,
summary_num_cross_attention_heads=FLAGS.
summary_num_cross_attention_heads,
summary_enable_default_side_input=FLAGS.
summary_enable_default_side_input,
init_checkpoint=FLAGS.init_checkpoint,
learning_rate=FLAGS.learning_rate,
num_train_steps=FLAGS.num_train_steps,
num_warmup_steps=FLAGS.num_warmup_steps,
use_tpu=FLAGS.use_tpu,
use_one_hot_embeddings=FLAGS.use_one_hot_embeddings,
optimizer=FLAGS.optimizer,
poly_power=FLAGS.poly_power,
start_warmup_step=FLAGS.start_warmup_step,
learning_rate_schedule=FLAGS.learning_rate_schedule,
metrics_name=FLAGS.metrics_name)
# If TPU is not available, this will fall back to normal Estimator on CPU
# or GPU.
estimator = tf_estimator.tpu.TPUEstimator(use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=batch_size,
eval_batch_size=batch_size)
if FLAGS.do_train:
logging.info("***** Running training *****")
train_input_fn = input_fn_builder(input_files=input_files,
input_config=input_config,
model_config=model_config,
is_training=True)
estimator.train(input_fn=train_input_fn,
max_steps=FLAGS.num_train_steps)
if FLAGS.do_eval:
logging.info("***** Running evaluation *****")
num_eval_examples = input_utils.get_num_examples_in_tf_records(
input_files)
eval_steps_per_epoch = num_eval_examples
max_eval_steps = eval_steps_per_epoch * FLAGS.num_eval_epochs
logging.info(" Number of eval examples = %d", num_eval_examples)
logging.info(" Number of TPU tasks = %d", num_tpu_tasks or 1)
logging.info(" Number of eval steps per epoch = %d",
eval_steps_per_epoch)
logging.info(" Eval steps = %d", max_eval_steps)
eval_input_fn = input_fn_builder(input_files=input_files,
input_config=input_config,
model_config=model_config,
is_training=False)
# Run evaluation for each new checkpoint.
for ckpt in tf.train.checkpoints_iterator(FLAGS.output_dir):
logging.info("Starting eval on new checkpoint: %s", ckpt)
try:
start_timestamp = time.time(
) # This time will include compilation time
eval_results = estimator.evaluate(input_fn=eval_input_fn,
steps=max_eval_steps,
checkpoint_path=ckpt)
elapsed_time = int(time.time() - start_timestamp)
logging.info("Eval results: %s. Elapsed seconds: %d",
eval_results, elapsed_time)
# Terminate eval job when final checkpoint is reached.
current_step = int(os.path.basename(ckpt).split("-")[1])
if current_step >= FLAGS.num_train_steps:
logging.info("Evaluation finished after training step %d",
current_step)
break
except tf.errors.NotFoundError:
# Since the coordinator is on a different job than the TPU worker,
# sometimes the TPU worker does not finish initializing until long after
# the CPU job tells it to start evaluating. In this case, the checkpoint
# file could have been deleted already.
logging.info(
"Checkpoint %s no longer exists, skipping checkpoint",
ckpt)
def main(_):
logging.get_absl_handler().use_absl_log_file('log-sparse.txt',
FLAGS.output_dir)
logging.set_verbosity(logging.INFO)
processors = {"ner": NerProcessor}
if not FLAGS.do_train and not FLAGS.do_eval:
raise ValueError(
"At least one of `do_train` or `do_eval` must be True.")
bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
if FLAGS.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
"Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" %
(FLAGS.max_seq_length, bert_config.max_position_embeddings))
task_name = FLAGS.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file,
do_lower_case=FLAGS.do_lower_case)
tpu_cluster_resolver = None
if FLAGS.use_tpu and FLAGS.tpu_name:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
master=FLAGS.master,
model_dir=FLAGS.output_dir,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
tpu_config=tf.contrib.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_tpu_cores,
per_host_input_for_training=is_per_host))
train_examples = None
num_train_steps = None
num_warmup_steps = None
if FLAGS.do_train:
train_examples = processor.get_train_examples(FLAGS.data_dir)
num_train_steps = int(
len(train_examples) / FLAGS.train_batch_size *
FLAGS.num_train_epochs)
num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)
model_fn = model_fn_builder(bert_config=bert_config,
num_labels=len(label_list),
init_checkpoint=FLAGS.init_checkpoint,
learning_rate=FLAGS.learning_rate,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
use_tpu=FLAGS.use_tpu,
use_one_hot_embeddings=FLAGS.use_tpu)
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.train_batch_size,
eval_batch_size=FLAGS.eval_batch_size,
predict_batch_size=FLAGS.predict_batch_size)
if FLAGS.do_train:
train_file = os.path.join(FLAGS.output_dir, "train.tf_record")
_, _ = filed_based_convert_examples_to_features(
train_examples, label_list, FLAGS.max_seq_length, tokenizer,
train_file)
logging.info("***** Running training *****")
logging.info(" Num examples = %d", len(train_examples))
logging.info(" Batch size = %d", FLAGS.train_batch_size)
logging.info(" Num steps = %d", num_train_steps)
train_input_fn = file_based_input_fn_builder(
input_file=train_file,
seq_length=FLAGS.max_seq_length,
is_training=True,
drop_remainder=True)
start_time = time.time()
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
logging.info("--- %s seconds ---", (time.time() - start_time))
if FLAGS.do_eval:
eval_examples = processor.get_dev_examples(FLAGS.data_dir)
eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record")
batch_tokens, batch_labels = filed_based_convert_examples_to_features(
eval_examples, label_list, FLAGS.max_seq_length, tokenizer,
eval_file)
logging.info("***** Running evaluation *****")
logging.info(" Num examples = %d", len(eval_examples))
logging.info(" Batch size = %d", FLAGS.eval_batch_size)
# if FLAGS.use_tpu:
# eval_steps = int(len(eval_examples) / FLAGS.eval_batch_size)
# eval_drop_remainder = True if FLAGS.use_tpu else False
eval_input_fn = file_based_input_fn_builder(
input_file=eval_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
result = estimator.evaluate(input_fn=eval_input_fn)
output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as wf:
wf.write("***** Eval results *****\n")
confusion_matrix = result["confusion_matrix"]
p, r, f = metrics.calculate(confusion_matrix, len(label_list) - 1)
wf.write("***********************************************\n")
wf.write(
"********************P = {}*********************\n".format(
str(p)))
wf.write(
"********************R = {}*********************\n".format(
str(r)))
wf.write(
"********************F = {}*********************\n".format(
str(f)))
wf.write("***********************************************\n")
if FLAGS.do_predict:
with open(FLAGS.middle_output + '/label2id.pkl', 'rb') as rf:
label2id = pickle.load(rf)
id2label = {value: key for key, value in label2id.items()}
predict_examples = processor.get_test_examples(FLAGS.data_dir)
predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record")
batch_tokens, batch_labels = filed_based_convert_examples_to_features(
predict_examples, label_list, FLAGS.max_seq_length, tokenizer,
predict_file)
logging.info("***** Running prediction*****")
logging.info(" Num examples = %d", len(predict_examples))
logging.info(" Batch size = %d", FLAGS.predict_batch_size)
predict_input_fn = file_based_input_fn_builder(
input_file=predict_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
result = estimator.predict(input_fn=predict_input_fn)
output_predict_file = os.path.join(FLAGS.output_dir, "label_test.txt")
# here if the tag is "X" means it belong to its before token, here for convenient evaluate use
# conlleval.pl we discarding it directly
Writer(output_predict_file, result, batch_tokens, batch_labels,
id2label)
FLAGS = flags.FLAGS
def main(*args, **kwargs):
nasbench = nasbench_api.NASBench(FLAGS.path_to_nasbench)
module = nasbench.fixed_statistics[FLAGS.hash_key]
spec = model_spec.ModelSpec(module['module_adjacency'], module['module_operations'])
config = nasbench_config.build_config()
for flag in FLAGS.flags_by_module_dict()[args[0][0]]:
config[flag.name] = flag.value
config['use_tpu'] = False
config['use_KD'] = False
config['intermediate_evaluations'] = ['1.0']
trainset_multipier = FLAGS.trainset_part_percentage / 100.0
config['num_train'] = int(config['num_train'] * trainset_multipier)
config['num_train_eval'] = int(config['num_train_eval'] * trainset_multipier)
config['num_augment'] = int(config['num_augment'] * trainset_multipier)
logging.info("Train and evaluate with config\n{}\n and spec\n{}".format(config, spec))
train(spec, config, FLAGS.save_path)
if __name__ == "__main__":
logging.set_verbosity('info')
logging.set_stderrthreshold('info')
app.run(main)
training_accuracy.reset_states()
with test_summary_writer.as_default():
test_iterator = iter(test_dataset)
for step in range(steps_per_eval):
if step % 20 == 0:
logging.info(
'Starting to run eval step %s of epoch: %s', step,
epoch)
test_step(test_iterator)
tf.summary.scalar('loss',
test_loss.result(),
step=optimizer.iterations)
tf.summary.scalar('accuracy',
test_accuracy.result(),
step=optimizer.iterations)
logging.info('Test loss: %s, accuracy: %s%%',
round(test_loss.result(), 4),
round(test_accuracy.result() * 100, 2))
test_loss.reset_states()
test_accuracy.reset_states()
checkpoint_name = checkpoint.save(
os.path.join(model_dir, 'checkpoint'))
logging.info('Saved checkpoint to %s', checkpoint_name)
if __name__ == '__main__':
logging.set_verbosity(logging.INFO)
app.run(main)
def main(_):
logging.set_verbosity(logging.INFO)
# Create the path for the serialized collect policy.
collect_policy_saved_model_path = os.path.join(
FLAGS.root_dir, learner.POLICY_SAVED_MODEL_DIR,
learner.COLLECT_POLICY_SAVED_MODEL_DIR)
saved_model_pb_path = os.path.join(collect_policy_saved_model_path,
'saved_model.pb')
samples_per_insert = FLAGS.samples_per_insert
min_table_size_before_sampling = FLAGS.min_table_size_before_sampling
try:
# Wait for the collect policy to be outputed by learner (timeout after 2
# days), then load it.
train_utils.wait_for_file(
saved_model_pb_path, sleep_time_secs=2, num_retries=86400)
collect_policy = py_tf_eager_policy.SavedModelPyTFEagerPolicy(
collect_policy_saved_model_path, load_specs_from_pbtxt=True)
except TimeoutError as e:
# If the collect policy does not become available during the wait time of
# the call `wait_for_file`, that probably means the learner is not running.
logging.error('Could not get the file %s. Exiting.', saved_model_pb_path)
raise e
# Create the signature for the variable container holding the policy weights.
train_step = train_utils.create_train_step()
variables = {
reverb_variable_container.POLICY_KEY: collect_policy.variables(),
reverb_variable_container.TRAIN_STEP_KEY: train_step
}
variable_container_signature = tf.nest.map_structure(
lambda variable: tf.TensorSpec(variable.shape, dtype=variable.dtype),
variables)
logging.info('Signature of variables: \n%s', variable_container_signature)
# Create the signature for the replay buffer holding observed experience.
replay_buffer_signature = tensor_spec.from_spec(
collect_policy.collect_data_spec)
logging.info('Signature of experience: \n%s', replay_buffer_signature)
if samples_per_insert is not None:
# Use SamplesPerInsertRatio limiter
samples_per_insert_tolerance = _SAMPLES_PER_INSERT_TOLERANCE_RATIO * samples_per_insert
error_buffer = min_table_size_before_sampling * samples_per_insert_tolerance
experience_rate_limiter = reverb.rate_limiters.SampleToInsertRatio(
min_size_to_sample=min_table_size_before_sampling,
samples_per_insert=samples_per_insert,
error_buffer=error_buffer)
else:
# Use MinSize limiter
experience_rate_limiter = reverb.rate_limiters.MinSize(
min_table_size_before_sampling)
# Crete and start the replay buffer and variable container server.
server = reverb.Server(
tables=[
reverb.Table( # Replay buffer storing experience.
name=reverb_replay_buffer.DEFAULT_TABLE,
sampler=reverb.selectors.Uniform(),
remover=reverb.selectors.Fifo(),
rate_limiter=experience_rate_limiter,
max_size=FLAGS.replay_buffer_capacity,
max_times_sampled=0,
signature=replay_buffer_signature,
),
reverb.Table( # Variable container storing policy parameters.
name=reverb_variable_container.DEFAULT_TABLE,
sampler=reverb.selectors.Uniform(),
remover=reverb.selectors.Fifo(),
rate_limiter=reverb.rate_limiters.MinSize(1),
max_size=1,
max_times_sampled=0,
signature=variable_container_signature,
),
],
port=FLAGS.port)
server.wait()
learning_rate=0.0001,
global_step=tf.train.get_or_create_global_step(),
decay_steps=100000,
decay_rate=0.9,
staircase=True)
discriminator_lr = 0.001
return generator_lr, discriminator_lr
def _optimizer(gen_lr, dis_lr, use_sync_replicas):
"""Get an optimizer, that's optionally synchronous."""
generator_opt = tf.train.RMSPropOptimizer(gen_lr, decay=.9, momentum=0.1)
discriminator_opt = tf.train.RMSPropOptimizer(dis_lr, decay=.95, momentum=0.1)
def _make_sync(opt):
return tf.train.SyncReplicasOptimizer(
opt,
replicas_to_aggregate=FLAGS.worker_replicas-FLAGS.backup_workers,
total_num_replicas=FLAGS.worker_replicas)
if use_sync_replicas:
generator_opt = _make_sync(generator_opt)
discriminator_opt = _make_sync(discriminator_opt)
return generator_opt, discriminator_opt
if __name__ == '__main__':
logging.set_verbosity(logging.INFO)
tf.app.run()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
logging.info("tensorflow version: %s", tf.__version__)
evaluate()
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
tf.compat.v1.enable_v2_behavior()
logging.set_verbosity(logging.INFO)
# Flags.
hparam_dict = collections.OrderedDict([(name, FLAGS[name].value)
for name in hparam_flags])
for k in hparam_dict.keys():
if hparam_dict[k] is None:
hparam_dict[k] = 'None'
for k, v in hparam_dict.items():
print('{} : {} '.format(k, v))
tff.framework.set_default_executor(
tff.framework.create_local_executor(
num_clients=FLAGS.num_clients_per_round))
# Trained classifier model.
classifier_model = ecm.get_trained_emnist_classifier_model()
# GAN Models.
disc_model_fn, gen_model_fn = _get_gan_network_models(FLAGS.noise_dim)
# Training datasets.
server_gen_inputs_dataset = _create_gen_inputs_dataset(
batch_size=FLAGS.server_train_batch_size, noise_dim=FLAGS.noise_dim)
client_gen_inputs_dataset = _create_gen_inputs_dataset(
batch_size=CLIENT_TRAIN_BATCH_SIZE, noise_dim=FLAGS.noise_dim)
if FLAGS.filtering == 'by_user':
client_real_images_train_tff_data = (
fedu.get_filtered_by_user_client_data_for_training(
invert_imagery_probability=FLAGS.invert_imagery_probability,
accuracy_threshold=FLAGS.accuracy_threshold,
batch_size=CLIENT_TRAIN_BATCH_SIZE))
elif FLAGS.filtering == 'by_example':
client_real_images_train_tff_data = (
fedu.get_filtered_by_example_client_data_for_training(
invert_imagery_probability=FLAGS.invert_imagery_probability,
min_num_examples=FLAGS.min_num_examples,
example_class_selection=FLAGS.example_class_selection,
batch_size=CLIENT_TRAIN_BATCH_SIZE))
else:
client_real_images_train_tff_data = (
fedu.get_unfiltered_client_data_for_training(
batch_size=CLIENT_TRAIN_BATCH_SIZE))
print('There are %d unique clients that will be used for GAN training.' %
len(client_real_images_train_tff_data.client_ids))
# Training: GAN Losses and Optimizers.
gan_loss_fns = gan_losses.WassersteinGanLossFns(
grad_penalty_lambda=FLAGS.wass_gp_lambda)
disc_optimizer = tf.keras.optimizers.SGD(lr=0.0005)
gen_optimizer = tf.keras.optimizers.SGD(lr=0.005)
# Eval datasets.
gen_inputs_eval_dataset = _create_gen_inputs_dataset(
batch_size=EVAL_BATCH_SIZE, noise_dim=FLAGS.noise_dim)
real_images_eval_dataset = _create_real_images_dataset_for_eval()
# Eval hook.
path_to_output_images = _get_path_to_output_image(FLAGS.root_output_dir,
FLAGS.exp_name)
logging.info('path_to_output_images is %s', path_to_output_images)
eval_hook_fn = _get_emnist_eval_hook_fn(
FLAGS.exp_name, FLAGS.root_output_dir, hparam_dict, gan_loss_fns,
gen_inputs_eval_dataset, real_images_eval_dataset,
FLAGS.num_rounds_per_save_images, path_to_output_images,
classifier_model)
# Form the GAN.
gan = _get_gan(gen_model_fn,
disc_model_fn,
gan_loss_fns,
gen_optimizer,
disc_optimizer,
server_gen_inputs_dataset,
client_real_images_train_tff_data,
use_dp=FLAGS.use_dp,
dp_l2_norm_clip=FLAGS.dp_l2_norm_clip,
dp_noise_multiplier=FLAGS.dp_noise_multiplier,
clients_per_round=FLAGS.num_clients_per_round)
# Training.
_, tff_time = _train(gan,
server_gen_inputs_dataset,
client_gen_inputs_dataset,
client_real_images_train_tff_data,
FLAGS.num_client_disc_train_steps,
FLAGS.num_server_gen_train_steps,
FLAGS.num_clients_per_round,
FLAGS.num_rounds,
FLAGS.num_rounds_per_eval,
eval_hook_fn,
FLAGS.num_rounds_per_checkpoint,
output_dir=FLAGS.root_output_dir,
exp_name=FLAGS.exp_name)
logging.info('Total training time was %4.3f seconds.', tff_time)
print('\nTRAINING COMPLETE.')
def run_decoding_experiment(my_flags):
"""Runs one decoding experiment: assemble data, train, evaluate.
Args:
my_flags: A decoding flags object telling how to do the decoding.
"""
if my_flags.debug:
logging.set_verbosity(logging.DEBUG)
if my_flags.pre_context + 1 + my_flags.post_context < my_flags.min_context:
my_flags.post_context = my_flags.min_context - (my_flags.pre_context +
1)
if not my_flags.summary_dir.endswith('/'):
my_flags.summary_dir = my_flags.summary_dir + '/'
params = my_flags.experiment_parameters()
logging.info('Params string is: %s', params)
logging.info('TFRecord data from: %s with %s', my_flags.tfexample_dir,
my_flags.tfexample_pattern)
if my_flags.check_file_pattern:
check_files(my_flags.tfexample_dir, my_flags.tfexample_pattern)
return
# Create the dataset we'll use for this experiment.
test_brain_data = brain_data.create_brain_dataset(
my_flags.data,
my_flags.input_field,
my_flags.output_field,
frame_rate=my_flags.frame_rate,
pre_context=my_flags.pre_context,
post_context=my_flags.post_context,
in2_fields=my_flags.input2_field,
in2_pre_context=my_flags.input2_pre_context,
in2_post_context=my_flags.input2_post_context,
final_batch_size=my_flags.batch_size,
shuffle_buffer_size=my_flags.shuffle_buffer_size,
data_dir=my_flags.tfexample_dir,
data_pattern=my_flags.tfexample_pattern,
train_file_pattern=my_flags.train_file_pattern,
validate_file_pattern=my_flags.validate_file_pattern,
test_file_pattern=my_flags.test_file_pattern)
some_dataset = test_brain_data.create_dataset('train')
tensorboard_dir = my_flags.tensorboard_dir
if tensorboard_dir and 'PARAMS' in tensorboard_dir:
tensorboard_dir = tensorboard_dir.replace(
'PARAMS', my_flags.experiment_parameters(','))
test_model = create_brain_model(my_flags, some_dataset)
test_model.add_tensorboard_summary('Parameters',
my_flags.experiment_parameters(' '))
train_results, test_results = train_and_test(my_flags,
test_brain_data,
test_model,
epochs=my_flags.epoch_count)
print() # Needed to finish off line after TF training status updates.
test_model.summary()
test_model.add_metadata(attr.asdict(my_flags), dataset=some_dataset)
dprime, final_decoder = train_lda_model(test_brain_data, test_model,
my_flags)
logging.info('train_and_test got these results: %s and test %s',
train_results, test_results)
print('train_and_test got these results: %s and test %s' %
(train_results, test_results))
logging.info('Calculated dprime is %g.', dprime)
print('Calculated dprime is %g.' % dprime)
if my_flags.summary_dir:
logging.info('Writing train/test results to %s', my_flags.summary_dir)
write_experiment_summary(my_flags, train_results, test_results, dprime)
print('Wrote train/test results to %s.' % my_flags.summary_dir)
if my_flags.tensorboard_dir:
# Note: Keras' evaluate method doesn't write out results. So
# instead, we have to write out our own version of the final results.
logdir = os.path.join(test_model.tensorboard_dir, 'dprime')
writer = tf.summary.create_file_writer(logdir=logdir)
with writer.as_default():
tf.summary.scalar('dprime', dprime, step=my_flags.epoch_count)
if my_flags.saved_model_dir:
logging.info('Writing saved model to %s', my_flags.saved_model_dir)
test_model.save(my_flags.saved_model_dir)
final_decoder.save_parameters(
os.path.join(my_flags.saved_model_dir, 'decoder_model.json'))
print('Wrote saved model to %s.' % my_flags.saved_model_dir)
