def train(hparams, output_dir): prefix = output_dir #remove trash # prefix = "~/trash/loop_{}".format(random.randint(10000, 99999)) data_dir = os.path.expanduser(prefix + "/data") tmp_dir = os.path.expanduser(prefix + "/tmp") output_dir = os.path.expanduser(prefix + "/output") tf.gfile.MakeDirs(data_dir) tf.gfile.MakeDirs(tmp_dir) tf.gfile.MakeDirs(output_dir) last_model = "" start_time = time.time() line = ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> " for iloop in range(hparams.epochs): time_delta = time.time() - start_time print(line+"Step {}.1. - generate data from policy. " "Time: {}".format(iloop, str(datetime.timedelta(seconds=time_delta)))) FLAGS.problems = "gym_discrete_problem" FLAGS.agent_policy_path = last_model gym_problem = problems.problem(FLAGS.problems) gym_problem.num_steps = hparams.true_env_generator_num_steps iter_data_dir = os.path.join(data_dir, str(iloop)) tf.gfile.MakeDirs(iter_data_dir) gym_problem.generate_data(iter_data_dir, tmp_dir) time_delta = time.time() - start_time print(line+"Step {}.2. - generate env model. " "Time: {}".format(iloop, str(datetime.timedelta(seconds=time_delta)))) # 2. generate env model FLAGS.data_dir = iter_data_dir FLAGS.output_dir = output_dir FLAGS.model = hparams.generative_model FLAGS.hparams_set = hparams.generative_model_params FLAGS.train_steps = hparams.model_train_steps FLAGS.eval_steps = 1 t2t_trainer.main([]) time_delta = time.time() - start_time print(line+"Step {}.3. - evalue env model. " "Time: {}".format(iloop, str(datetime.timedelta(seconds=time_delta)))) gym_simulated_problem = problems.problem("gym_simulated_discrete_problem") gym_simulated_problem.num_steps = hparams.simulated_env_generator_num_steps gym_simulated_problem.generate_data(iter_data_dir, tmp_dir) # time_delta = time.time() - start_time print(line+"Step {}.4. - train PPO in model env." " Time: {}".format(iloop, str(datetime.timedelta(seconds=time_delta)))) ppo_epochs_num=hparams.ppo_epochs_num ppo_hparams = trainer_lib.create_hparams("atari_base", "epochs_num={},simulated_environment=True,eval_every_epochs=0,save_models_every_epochs={}".format(ppo_epochs_num+1, ppo_epochs_num), data_dir=output_dir) ppo_hparams.epoch_length = hparams.ppo_epoch_length ppo_dir = tempfile.mkdtemp(dir=data_dir, prefix="ppo_") in_graph_wrappers = [(TimeLimitWrapper, {"timelimit": 150}), (PongT2TGeneratorHackWrapper, {"add_value": -2})] + gym_problem.in_graph_wrappers ppo_hparams.add_hparam("in_graph_wrappers", in_graph_wrappers) rl_trainer_lib.train(ppo_hparams, "PongNoFrameskip-v4", ppo_dir) last_model = ppo_dir + "/model{}.ckpt".format(ppo_epochs_num)
def test_glow_inference(self): hparams = glow.glow_hparams() hparams.depth = 15 hparams.n_levels = 2 hparams.data_dir = '' curr_dir = tempfile.mkdtemp() # Training pipeline with tf.Graph().as_default(): cifar_problem = problems.problem( 'image_cifar10_plain_random_shift') hparams.problem = cifar_problem model = glow.Glow(hparams, tf_estimator.ModeKeys.TRAIN) train_dataset = cifar_problem.dataset(MODES.TRAIN) one_shot = train_dataset.make_one_shot_iterator() x_batch, y_batch = self.batch(one_shot) features = {'inputs': x_batch, 'targets': y_batch} model_path = os.path.join(curr_dir, 'model') model(features) with tf.Session() as session: saver = tf.train.Saver() session.run(tf.global_variables_initializer()) init_op = tf.get_collection('glow_init_op') session.run(init_op) z = session.run([model.z]) mean_z = np.mean(z) is_undefined = np.isnan(mean_z) or np.isinf(mean_z) self.assertTrue(not is_undefined) saver.save(session, model_path) # Inference pipeline with tf.Graph().as_default(): cifar_problem = problems.problem( 'image_cifar10_plain_random_shift') hparams.problem = cifar_problem model = glow.Glow(hparams, tf_estimator.ModeKeys.PREDICT) test_dataset = cifar_problem.dataset(MODES.EVAL) one_shot = test_dataset.make_one_shot_iterator() x_batch, y_batch = self.batch(one_shot) features = {'inputs': x_batch, 'targets': y_batch} model_path = os.path.join(curr_dir, 'model') predictions = model.infer(features) with tf.Session() as session: saver = tf.train.Saver() saver.restore(session, model_path) predictions_np = session.run(predictions) self.assertTrue(np.all(predictions_np <= 255)) self.assertTrue(np.all(predictions_np >= 0))
def test_glow_inference(self): hparams = glow.glow_hparams() hparams.depth = 15 hparams.n_levels = 2 hparams.data_dir = '' curr_dir = tempfile.mkdtemp() # Training pipeline with tf.Graph().as_default(): cifar_problem = problems.problem('image_cifar10_plain_random_shift') hparams.problem = cifar_problem model = glow.Glow(hparams, tf.estimator.ModeKeys.TRAIN) train_dataset = cifar_problem.dataset(MODES.TRAIN) one_shot = train_dataset.make_one_shot_iterator() x_batch, y_batch = self.batch(one_shot) features = {'inputs': x_batch, 'targets': y_batch} model_path = os.path.join(curr_dir, 'model') model(features) with tf.Session() as session: saver = tf.train.Saver() session.run(tf.global_variables_initializer()) init_op = tf.get_collection('glow_init_op') session.run(init_op) z = session.run([model.z]) mean_z = np.mean(z) is_undefined = np.isnan(mean_z) or np.isinf(mean_z) self.assertTrue(not is_undefined) saver.save(session, model_path) # Inference pipeline with tf.Graph().as_default(): cifar_problem = problems.problem('image_cifar10_plain_random_shift') hparams.problem = cifar_problem model = glow.Glow(hparams, tf.estimator.ModeKeys.PREDICT) test_dataset = cifar_problem.dataset(MODES.EVAL) one_shot = test_dataset.make_one_shot_iterator() x_batch, y_batch = self.batch(one_shot) features = {'inputs': x_batch, 'targets': y_batch} model_path = os.path.join(curr_dir, 'model') predictions = model.infer(features) with tf.Session() as session: saver = tf.train.Saver() saver.restore(session, model_path) predictions_np = session.run(predictions) self.assertTrue(np.all(predictions_np <= 255)) self.assertTrue(np.all(predictions_np >= 0))
def _train_and_eval_dataset_v1(problem_name, data_dir): """Return train and evaluation datasets, feature info and supervised keys.""" from tensor2tensor import problems # pylint: disable=g-import-not-at-top problem = problems.problem(problem_name) train_dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN, data_dir) train_dataset = train_dataset.map(_select_features) eval_dataset = problem.dataset(tf.estimator.ModeKeys.EVAL, data_dir) eval_dataset = eval_dataset.map(_select_features) hparams = problem.get_hparams() # We take a few training examples to guess the shapes. input_shapes, target_shapes = [], [] example_tensor = train_dataset.make_one_shot_iterator().get_next() sess = tf.Session() example1 = sess.run(example_tensor) example2 = sess.run(example_tensor) example3 = sess.run(example_tensor) # We use "inputs" as input except for purely auto-regressive tasks like # language models where "targets" are used as input_key. input_key = "inputs" if "inputs" in example1 else "targets" supervised_keys = ([input_key], ["targets"]) for example in [example1, example2, example3]: input_shapes.append(list(example[input_key].shape)) target_shapes.append(list(example["targets"].shape)) input_vocab_size = hparams.vocab_size[input_key] target_vocab_size = hparams.vocab_size["targets"] input_info = _make_info(input_shapes, input_vocab_size) target_info = _make_info(target_shapes, target_vocab_size) info = {input_key: input_info, "targets": target_info} return train_dataset, eval_dataset, info, supervised_keys
def testBuildDataset(self): # See all the available problems self.assertTrue(len(problems.available()) > 10) # Retrieve a problem by name print(problems.available()) exit(0) problem = problems.problem("translate_ende_wmt8k") # Access train and dev datasets through Problem train_dataset = problem.dataset(MODES.TRAIN) dev_dataset = problem.dataset(MODES.EVAL) # Access vocab size and other info (e.g. the data encoders used to # encode/decode data for the feature, used below) through feature_info. feature_info = problem.feature_info self.assertTrue(feature_info["inputs"].vocab_size > 0) self.assertTrue(feature_info["targets"].vocab_size > 0) train_example = train_dataset.make_one_shot_iterator().get_next() dev_example = dev_dataset.make_one_shot_iterator().get_next() with tf.Session() as sess: train_ex_val, _ = sess.run([train_example, dev_example]) _ = feature_info["inputs"].encoder.decode(train_ex_val["inputs"]) _ = feature_info["targets"].encoder.decode(train_ex_val["targets"])
def _train_and_eval_dataset_v1(problem_name, data_dir): """Return train and evaluation datasets, feature info and supervised keys.""" problem = problems.problem(problem_name) train_dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN, data_dir) train_dataset = train_dataset.map(_select_features) eval_dataset = problem.dataset(tf.estimator.ModeKeys.EVAL, data_dir) eval_dataset = eval_dataset.map(_select_features) supervised_keys = (["inputs"], ["targets"]) hparams = problem.get_hparams() # We take a few training examples to guess the shapes. input_shapes, target_shapes = [], [] example_tensor = train_dataset.make_one_shot_iterator().get_next() sess = tf.Session() example1 = sess.run(example_tensor) example2 = sess.run(example_tensor) example3 = sess.run(example_tensor) for example in [example1, example2, example3]: input_shapes.append(list(example["inputs"].shape)) target_shapes.append(list(example["targets"].shape)) input_vocab_size = hparams.vocab_size["inputs"] target_vocab_size = hparams.vocab_size["targets"] input_info = _make_info(input_shapes, input_vocab_size) target_info = _make_info(target_shapes, target_vocab_size) info = {"inputs": input_info, "targets": target_info} return train_dataset, eval_dataset, info, supervised_keys
def test_glow(self): with tf.Graph().as_default(): hparams = glow.glow_hparams() hparams.depth = 15 hparams.n_levels = 2 hparams.init_batch_size = 256 hparams.batch_size = 1 cifar_problem = problems.problem( 'image_cifar10_plain_random_shift') hparams.problem = cifar_problem model = glow.Glow(hparams, tf.estimator.ModeKeys.TRAIN) train_dataset = cifar_problem.dataset(MODES.TRAIN) one_shot = train_dataset.make_one_shot_iterator() x_batch, y_batch = self.batch(one_shot) features = {'inputs': x_batch, 'targets': y_batch} _, obj_dict = model.body(features) objective = obj_dict['training'] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) # Run initialization. init_op = tf.get_collection('glow_init_op') sess.run(init_op) # Run forward pass. obj_np = sess.run(objective) mean_obj = np.mean(obj_np) # Check that one forward-propagation does not NaN, i.e # initialization etc works as expected. self.assertTrue(mean_obj > 0 and mean_obj < 10.0)
def test_glow(self): with tf.Graph().as_default(): hparams = glow.glow_hparams() hparams.depth = 15 hparams.n_levels = 2 hparams.init_batch_size = 256 hparams.batch_size = 1 hparams.data_dir = '' cifar_problem = problems.problem('image_cifar10_plain_random_shift') hparams.problem = cifar_problem model = glow.Glow(hparams, tf.estimator.ModeKeys.TRAIN) train_dataset = cifar_problem.dataset(MODES.TRAIN) one_shot = train_dataset.make_one_shot_iterator() x_batch, y_batch = self.batch(one_shot) features = {'inputs': x_batch, 'targets': y_batch} _, obj_dict = model.body(features) objective = obj_dict['training'] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) # Run initialization. init_op = tf.get_collection('glow_init_op') sess.run(init_op) # Run forward pass. obj_np = sess.run(objective) mean_obj = np.mean(obj_np) # Check that one forward-propagation does not NaN, i.e # initialization etc works as expected. self.assertTrue(mean_obj > 0 and mean_obj < 10.0)
def __init__(self, config): self.translate_problem = problems.problem(config.PROBLEM) self.encoder = self.translate_problem.feature_encoders( config.VOCAB_DIR) self.hparams = trainer_lib.create_hparams(config.HPARAMS, data_dir=config.VOCAB_DIR, problem_name=config.PROBLEM) self.checkpoint_path = config.CHECKPOINT_PATH self.translate_model = registry.model(config.MODEL)(self.hparams, Modes.PREDICT)
def example_apply_model(ckpt_path, hparams_set="img2img_transformer2d_tiny", problem_name="img2img_allen_brain_dim8to32", model_name="img2img_transformer", data_dir="/mnt/nfs-east1-d/data", input_dim=8, output_dim=32): # HACK: Avoid re-instantiating the model which causes problems... # TODO: Better way to handle this, e.g. delete from globals. if 'model' not in globals(): hp = trainer_lib.create_hparams(hparams_set, data_dir=data_dir, problem_name=problem_name) model = registry.model(model_name)(hp, Modes.TRAIN) problem_object = problems.problem(problem_name) dataset = problem_object.dataset(Modes.TRAIN, data_dir) with tfe.restore_variables_on_create(ckpt_path): for count, example in enumerate(tfe.Iterator(dataset)): if count > 1234: break # Example input fig = plt.figure(figsize=(8, 8)) example["inputs"] = tf.reshape(example["inputs"], [1, input_dim, input_dim, 3]) fig.add_subplot(1, 3, 1) plt.imshow(example["inputs"].numpy()[0]) # Example target fig.add_subplot(1, 3, 2) example["targets"] = tf.reshape(example["targets"], [1, output_dim, output_dim, 3]) plt.imshow(example["targets"].numpy()[0]) # Dummy target (expected by model) example["targets"] = tf.reshape( tf.zeros((1, output_dim, output_dim, 3), dtype=np.uint8), [1, output_dim, output_dim, 3]) # Produce and display prediction predictions, _ = model(example) fig.add_subplot(1, 3, 3) inferred = demo.infer(predictions) plt.imshow(inferred) plt.show() return example, predictions, inferred
def __init__( self, hparams_set, model_name, data_dir, problem_name, beam_size=1): inputs, targets, samples, att_mats = build_model( hparams_set, model_name, data_dir, problem_name, beam_size=beam_size) # Fetch the problem ende_problem = problems.problem(problem_name) encoders = ende_problem.feature_encoders(data_dir) self.inputs = inputs self.targets = targets self.att_mats = att_mats self.samples = samples self.encoders = encoders
def __init__( self, hparams_set, model_name, data_dir, problem_name, beam_size=1): inputs, targets, samples, att_mats = build_model( hparams_set, model_name, data_dir, problem_name, beam_size=beam_size) # Fetch the problem ende_problem = problems.problem(problem_name) encoders = ende_problem.feature_encoders(data_dir) self.inputs = inputs self.targets = targets self.att_mats = att_mats self.samples = samples self.encoders = encoders
def input_data(): """Input function to be returned.""" prob = problems.problem(problem_name) if data == 'image_cifar100': dataset = prob.dataset(mode, preprocess=augmented) if not augmented: dataset = dataset.map(map_func=standardization) else: dataset = prob.dataset(mode) dataset = dataset.batch(batch_size) dataset = dataset.repeat(repeat_num) dataset = dataset.make_one_shot_iterator().get_next() if data_format == 'CHW': dataset['inputs'] = tf.transpose(dataset['inputs'], (0, 3, 1, 2)) return dataset['inputs'], tf.squeeze( tf.one_hot(dataset['targets'], class_num))
def _train_and_eval_dataset_v1(problem_name, data_dir): """Return train and evaluation datasets, feature info and supervised keys.""" problem = problems.problem(problem_name) train_dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN, data_dir) eval_dataset = problem.dataset(tf.estimator.ModeKeys.EVAL, data_dir) supervised_keys = ("inputs", "targets") hparams = problem.get_hparams() # We take a few training examples to guess the shapes. input_shapes, target_shapes = [], [] for example in train_dataset.take(3): input_shapes.append(example["inputs"].shape.as_list()) target_shapes.append(example["targets"].shape.as_list()) input_info = _make_info(input_shapes, hparams.modality["inputs"].top_dimensionality) target_info = _make_info(target_shapes, hparams.modality["targets"].top_dimensionality) info = {"inputs": input_info, "targets": target_info} return train_dataset, eval_dataset, info, supervised_keys
def __init__( self, hparams_set, model_name, data_dir, problem_name, return_beams,beam_size,custom_problem_type,force_decode_len): ### inputs, targets, input_extra_length_ph,samples, att_mats = build_model( hparams_set, model_name, data_dir, problem_name, return_beams,beam_size,custom_problem_type,force_decode_len) # Fetch the problem ende_problem = problems.problem(problem_name) encoders = ende_problem.feature_encoders(data_dir) self.return_beams=return_beams self.beam_size=beam_size self.inputs = inputs self.targets = targets self.att_mats = att_mats self.samples = samples self.encoders = encoders self.input_extra_length_ph = input_extra_length_ph
def _train_and_eval_dataset_v1(problem_name, data_dir): """Return train and evaluation datasets, feature info and supervised keys.""" problem = problems.problem(problem_name) train_dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN, data_dir) train_dataset = train_dataset.map(_select_features) eval_dataset = problem.dataset(tf.estimator.ModeKeys.EVAL, data_dir) eval_dataset = eval_dataset.map(_select_features) supervised_keys = (["inputs"], ["targets"]) hparams = problem.get_hparams() # We take a few training examples to guess the shapes. input_shapes, target_shapes = [], [] for example in train_dataset.take(3): input_shapes.append(example["inputs"].shape.as_list()) target_shapes.append(example["targets"].shape.as_list()) input_vocab_size = hparams.vocab_size["inputs"] target_vocab_size = hparams.vocab_size["targets"] input_info = _make_info(input_shapes, input_vocab_size) target_info = _make_info(target_shapes, target_vocab_size) info = {"inputs": input_info, "targets": target_info} return train_dataset, eval_dataset, info, supervised_keys
def __init__(self, root_dir, model_name="transformer", hparams_set="transformer_base_single_gpu"): data_dir = os.path.join(root_dir, 'data') train_dir = os.path.join(root_dir, 'train') problem_name_file = open(os.path.join(root_dir, 'problem_name')) problem_name = problem_name_file.readline()[:-1] problem_name_file.close() problem = problems.problem(problem_name) self.encoders = problem.feature_encoders(data_dir) hparams = trainer_lib.create_hparams(hparams_set, data_dir=data_dir, problem_name=problem_name) self.translate_model = registry.model(model_name)(hparams, Modes.EVAL) checkpoint_file = open(os.path.join(train_dir, 'checkpoint')) ckpt_name = checkpoint_file.readline()[24:-2] checkpoint_file.close() self.ckpt_path = os.path.join(train_dir, ckpt_name)
def test_glow(self): with tf.Graph().as_default(): hparams = glow.glow_hparams() model = glow.Glow(hparams, tf.estimator.ModeKeys.TRAIN) cifar_problem = problems.problem( 'image_cifar10_plain_random_shift') train_dataset = cifar_problem.dataset(MODES.TRAIN) one_shot = train_dataset.make_one_shot_iterator() x_batch, y_batch = self.batch(one_shot) features = {'inputs': x_batch, 'targets': y_batch} _, obj_dict = model.body(features) objective = obj_dict['training'] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) obj_np = sess.run(objective) mean_obj = np.mean(obj_np) # Check that one forward-propagation does not NaN, i.e # initialization etc works as expected. is_undefined = np.isnan(mean_obj) or np.isinf(mean_obj) self.assertTrue(not is_undefined)
def main(_): # Fetch the problem wmt_problem = problems.problem(FLAGS.problem) # Declare the path we need data_dir = FLAGS.data_dir checkpoint_dir = FLAGS.model_dir ckpt_name = FLAGS.problem # ckpt_dir = tf.train.latest_checkpoint(os.path.join(checkpoint_dir, ckpt_name)) ckpt_dir = tf.train.latest_checkpoint(checkpoint_dir) # Create hparams and the model model_name = FLAGS.model hparams_set = FLAGS.hparams_set hparams = trainer_lib.create_hparams(hparams_set, data_dir=data_dir, problem_name=FLAGS.problem) # Get the encoders from the problem encoders = wmt_problem.feature_encoders(data_dir) translate_model = registry.model(model_name)(hparams, Modes.EVAL) sys.stdout.write('> ') sys.stdout.flush() sentence_en = sys.stdin.readline().strip() while sentence_en: if sentence_en == 'q': print("Close this process") break outputs = translate(encoders, translate_model, ckpt_dir, sentence_en) print(outputs) print('> ', end='') sys.stdout.flush() sentence_en = sys.stdin.readline()
def __init__(self, hparams_set, use_bottom_up_features, model_name, data_dir, problem_name, beam_size=1): if use_bottom_up_features: inputs, bottom_up_features, objet_boxes, targets, samples, att_mats = build_model_object( hparams_set, model_name, data_dir, problem_name, beam_size=beam_size) else: inputs, targets, samples, att_mats = build_model( hparams_set, model_name, data_dir, problem_name, beam_size=beam_size) # Fetch the problem ende_problem = problems.problem(problem_name) encoders = ende_problem.feature_encoders(data_dir) self.inputs = inputs self.targets = targets self.att_mats = att_mats self.samples = samples self.encoders = encoders self.bottom_up_features = None self.object_boxes = None if use_bottom_up_features: self.bottom_up_features = bottom_up_features self.object_boxes = objet_boxes
def testBuildDataset(self): # See all the available problems self.assertTrue(len(problems.available()) > 10) # Retrieve a problem by name problem = problems.problem("translate_ende_wmt8k") # Access train and dev datasets through Problem train_dataset = problem.dataset(MODES.TRAIN) dev_dataset = problem.dataset(MODES.EVAL) # Access vocab size and other info (e.g. the data encoders used to # encode/decode data for the feature, used below) through feature_info. feature_info = problem.feature_info self.assertTrue(feature_info["inputs"].vocab_size > 0) self.assertTrue(feature_info["targets"].vocab_size > 0) train_example = train_dataset.make_one_shot_iterator().get_next() dev_example = dev_dataset.make_one_shot_iterator().get_next() with tf.Session() as sess: train_ex_val, _ = sess.run([train_example, dev_example]) _ = feature_info["inputs"].encoder.decode(train_ex_val["inputs"]) _ = feature_info["targets"].encoder.decode(train_ex_val["targets"])
'split': problem.DatasetSplit.TRAIN, 'shards': 200 }, { 'split': problem.DatasetSplit.EVAL, 'shards': 1 }, ] DATA_DIR = os.path.expanduser('t2t-tatabahasa/data') TMP_DIR = os.path.expanduser('t2t-tatabahasa/tmp') TRAIN_DIR = os.path.expanduser('t2t-tatabahasa/train-small') PROBLEM = 'grammar' t2t_problem = problems.problem(PROBLEM) train_steps = 200000 eval_steps = 20 batch_size = 1024 * 6 save_checkpoints_steps = 10000 ALPHA = 0.0005 schedule = 'continuous_train_and_eval' MODEL = 'transformer_tag' HPARAMS = 'transformer_base' from tensor2tensor.utils.trainer_lib import create_run_config, create_experiment from tensor2tensor.utils.trainer_lib import create_hparams from tensor2tensor.utils import registry from tensor2tensor import models from tensor2tensor import problems
tf.gfile.MakeDirs(checkpoint_dir) gs_data_dir = "./tensor2tensor-data" gs_ckpt_dir = "./tensor2tensor-checkpoints/" tf.gfile.MakeDirs(gs_data_dir) tf.gfile.MakeDirs(gs_ckpt_dir) """# Download MNIST and inspect it""" # A Problem is a dataset together with some fixed pre-processing. # It could be a translation dataset with a specific tokenization, # or an image dataset with a specific resolution. # # There are many problems available in Tensor2Tensor problems.available() # Fetch the MNIST problem mnist_problem = problems.problem("image_mnist") # The generate_data method of a problem will download data and process it into # a standard format ready for training and evaluation. mnist_problem.generate_data(data_dir, tmp_dir) # Now let's see the training MNIST data as Tensors. mnist_example = tfe.Iterator(mnist_problem.dataset(Modes.TRAIN, data_dir)).next() image = mnist_example["inputs"] label = mnist_example["targets"] plt.imshow(image.numpy()[:, :, 0].astype(np.float32), cmap=plt.get_cmap('gray')) print("Label: %d" % label.numpy()) """# Translate from English to German with a pre-trained model"""
from tensor2tensor.utils import registry from tensor2tensor.utils import metrics import os import numpy as np from zh_vi import translate_zhvi DATA_DIR = os.path.expanduser("C:\\Users\\NguyenPM\\PycharmProjects\\transformer\\testdata\\data") # This folder contain the data TRAIN_DIR = os.path.expanduser("C:\\Users\\NguyenPM\\PycharmProjects\\transformer\\testdata\\train") # This folder contain the model tf.io.gfile.makedirs(DATA_DIR) tf.io.gfile.makedirs(TRAIN_DIR) PROBLEM = "translate_zhvi" MODEL = "transformer" # Our model HPARAMS = "transformer_big" zhvi_problem = problems.problem(PROBLEM) # Copy the vocab file locally so we can encode inputs and decode model outputs vocab_name = "vocab.translate_zhvi.16384.subwords" vocab_file = os.path.join(DATA_DIR, vocab_name) # Get the encoders from the problem encoders = zhvi_problem.feature_encoders(DATA_DIR) ckpt_path = tf.train.latest_checkpoint(os.path.join(TRAIN_DIR)) print(ckpt_path) def translate(inputs): encoded_inputs = encode(inputs) with tfe.restore_variables_on_create(ckpt_path): model_output = translate_model.infer(encoded_inputs)["outputs"]
Modes = tf.estimator.ModeKeys # Setup some directories data_dir = os.path.expanduser("~/t2t/data") tmp_dir = os.path.expanduser("~/t2t/tmp") train_dir = os.path.expanduser("~/t2t/train") checkpoint_dir = os.path.expanduser("~/t2t/checkpoints") tf.gfile.MakeDirs(data_dir) tf.gfile.MakeDirs(tmp_dir) tf.gfile.MakeDirs(train_dir) tf.gfile.MakeDirs(checkpoint_dir) gs_data_dir = "gs://tensor2tensor-data" gs_ckpt_dir = "gs://tensor2tensor-checkpoints/" # Fetch the MNIST problem ptb_problem = problems.problem("languagemodel_ptb10k") # Setup helper functions for encoding and decoding def encode(input_str, output_str=None): """Input str to features dict, ready for inference""" inputs = encoders["targets"].encode(input_str) + [1] # add EOS id batch_inputs = tf.reshape(inputs, [1, -1, 1]) # Make it 3D. return {"targets": batch_inputs} def decode(integers): """List of ints to str""" integers = list(np.squeeze(integers)) if 1 in integers: integers = integers[:integers.index(1)]
train_dir = os.path.expanduser("./t2t/train") checkpoint_dir = os.path.expanduser("./t2t/checkpoints") tf.gfile.MakeDirs(data_dir) tf.gfile.MakeDirs(tmp_dir) tf.gfile.MakeDirs(train_dir) tf.gfile.MakeDirs(checkpoint_dir) gs_data_dir = "gs://tensor2tensor-data" gs_ckpt_dir = "gs://tensor2tensor-checkpoints" # A Problem is a dataset together with some fixed pre-processing # It could be a translation dataset with a specific tokenization # or an image dataset with a specific resolution # # There are many problems available in Tensor2Tensor # print(problems.available()) # Fetch the MNIST problem mnist_problem = problems.problem("image_mnist") # The generate_data method of a problem with download data and process it into # a standard format ready for training and evaluation. mnist_problem.generate_data(data_dir, tmp_dir) # Now let's see the training MNIST data as Tensors. mnist_example = tfe.Iterator(mnist_problem.dataset(Modes.TRAIN, data_dir)).next() image = mnist_example["inputs"] label = mnist_example["targets"] plt.imshow(image.numpy()[:, :, 0].astype(np.float32), cmap=plt.get_cmap('gray')) print("Label: %d" % label.numpy()) plt.show()
# Other setup Modes = tf.estimator.ModeKeys # Setup some directories data_dir = os.path.expanduser("data") tmp_dir = os.path.expanduser("tmp") train_dir = os.path.expanduser("train") checkpoint_dir = os.path.expanduser("output") # tf.gfile.MakeDirs(data_dir) # tf.gfile.MakeDirs(tmp_dir) #tf.gfile.MakeDirs(train_dir) # tf.gfile.MakeDirs(checkpoint_dir) gs_data_dir = "data" gs_ckpt_dir = "checkpoints" ende_problem = problems.problem("sentiment_imdb") # Fetch the problem # Copy the vocab file locally so we can encode inputs and decode model outputs # All vocabs are stored on GCS vocab_name = "vocab.sentiment_imdb.8192.subwords" vocab_file = os.path.join(gs_data_dir, vocab_name) # Get the encoders from the problem encoders = ende_problem.feature_encoders(data_dir) print encoders # Setup helper functions for encoding and decoding def encode(input_str, output_str=None): """Input str to features dict, ready for inference""" inputs = encoders["inputs"].encode(input_str) + [1] # add EOS id batch_inputs = tf.reshape(inputs, [1, -1, 1]) # Make it 3D.
def run(self): self.params.logger.info("begin DataReader") if self.device is None: device_id = "" else: device_id = self.device[len(self.device) - 1] os.environ["CUDA_VISIBLE_DEVICES"] = device_id # Enable TF Eager execution # config = tf.ConfigProto( # log_device_placement=self.params.log_device_placement, # allow_soft_placement=True, # device_count = {'GPU': 0} # ) #config.gpu_options.allow_growth = True #config.gpu_options.per_process_gpu_memory_fraction=0.5 tfe = tf.contrib.eager tfe.enable_eager_execution() # Setup the training data # Fetch the MNIST problem problem = problems.problem(self.params.problem) # The generate_data method of a problem will download data and process it into # a standard format ready for training and evaluation. if self.params.generate_data == True: problem.generate_data(self.params.data_dir, self.params.tmp_dir) Modes = tf.estimator.ModeKeys if self.params.mode == "train": mode = Modes.TRAIN max_epochs = self.params.max_epochs start_epoch = get_ckpt_iters( self.params ) * self.params.batch_size // self.params.train_dataset_size num_repeats = max_epochs - start_epoch elif self.params.mode == "predict": mode = Modes.EVAL max_epochs = self.params.max_epochs start_epoch = get_ckpt_iters( self.params ) * self.params.batch_size // self.params.train_dataset_size num_repeats = 1 self.params.logger.info("epoch #%d" % self.params.max_epochs) model_data = [] if num_repeats > 0: dataset = problem.dataset(mode, self.params.data_dir) dataset = dataset.shuffle( buffer_size=256, reshuffle_each_iteration=self.params.reshuffle_each_epoch) dataset = dataset.repeat(num_repeats).batch(self.params.batch_size) pre_r = -1 for count, example in enumerate(tfe.Iterator(dataset)): if self.params.mode == "train": r = start_epoch + count * self.params.batch_size // self.params.train_dataset_size elif self.params.mode == "predict": r = start_epoch if r > pre_r: self.params.logger.info("epoch #%d" % (r + 1)) pre_r = r inputs, targets = example["inputs"], example["targets"] model_data.append([inputs.numpy(), targets.numpy()]) self.params.logger.info("end DataReader") return model_data
Modes = tf.estimator.ModeKeys # Setup some directories data_dir = "./t2t/data/" tmp_dir = "./t2t/tmp/" train_dir = "./t2t/train/" checkpoint_dir = "./t2t/checkpoints/" tf.gfile.MakeDirs(data_dir) tf.gfile.MakeDirs(tmp_dir) tf.gfile.MakeDirs(train_dir) tf.gfile.MakeDirs(checkpoint_dir) #%% # Translate English -> German problem_name = "translate_ende_wmt32k" ende_problem = problems.problem(problem_name) # Get the encoders from the problem encoders = ende_problem.feature_encoders(data_dir) # Setup helper functions for encoding and decoding def encode(input_str, output_str=None): """Input str to features dict, ready for inference""" inputs = encoders["inputs"].encode(input_str) + [1] # add EOS id batch_inputs = tf.reshape(inputs, [1, -1, 1]) # Make it 3D. return {"inputs": batch_inputs} def decode(integers): """List of ints to str"""
def get_data(dataset_name, mode="train", batch_size=256, num_epochs=20, prep_fn=None, preprocess_batch=None, metadata=None): """ Construct a tf.data.Dataset for the specified dataset. Args: dataset: string representing the dataset to load mode: string ("train" or "test") representing mode in which to run batch_size: integer representing size of batch prep_fn: optional preprocessing function that takes a tf.data.Dataset and returns a preprocessed Dataset. Returns: A tf.data.Dataset to be consumed by training or eval loops """ dataset = None if metadata is None: metadata = {} if dataset_name == "cifar10": (x_train, y_train), (x_test, y_test) = cifar10.load_data() x_train = x_train.astype(np.float32) x_test = x_test.astype(np.float32) y_train = np.squeeze(y_train, axis=1).astype(np.int32) y_test = np.squeeze(y_test, axis=1).astype(np.int32) elif dataset_name == "cifar100": (x_train, y_train), (x_test, y_test) = cifar100.load_data() x_train = x_train.astype(np.float32) x_test = x_test.astype(np.float32) y_train = np.squeeze(y_train, axis=1).astype(np.int32) y_test = np.squeeze(y_test, axis=1).astype(np.int32) elif dataset_name in ["envi_iwslt32k", "enfr_wmt_small8k"]: data_dir = os.path.join("data", dataset_name) tmp_dir = os.path.join("data", dataset_name + "_tmp") t2t_name = "translate_%s" % dataset_name problem = problems.problem(t2t_name) problem.generate_data(data_dir, tmp_dir) dataset = problem.dataset(mode, data_dir) metadata["problem"] = problem metadata["max_length"] = 30 elif dataset_name == "anki_spaeng": path_to_zip = tf.keras.utils.get_file( 'spa-eng.zip', origin='http://download.tensorflow.org/data/spa-eng.zip', extract=True) path_to_file = os.path.dirname(path_to_zip) + "/spa-eng/spa.txt" raise RuntimeError("Not implemented") else: raise ValueError("Unknown dataset: %s" % dataset_name) if prep_fn: if dataset: dataset, metadata = prep_fn(dataset, metadata) else: x_train, y_train, x_test, y_test, metadata = prep_fn( x_train, y_train, x_test, y_test, metadata) if dataset is None: if mode == "train": x, y = x_train, y_train elif mode == "test": x, y = x_test, y_test else: ValueError("Invalid mode: %s" % mode) dataset = tensorflow.data.Dataset.from_tensor_slices({ "inputs": x, "targets": y }) dataset = dataset.repeat(num_epochs).shuffle(buffer_size=500) drop_remainder = mode == "train" dataset = dataset.batch(batch_size, drop_remainder=drop_remainder) if preprocess_batch: dataset = preprocess_batch(dataset, metadata) return dataset, metadata
from tensor2tensor.utils import registry from tensor2tensor.utils import metrics # Enable TF Eager execution from tensorflow.contrib.eager.python import tfe tfe.enable_eager_execution() # Other setup Modes = tf.estimator.ModeKeys ckpt_path = sys.argv[1] fin_name = sys.argv[2] fout_name = sys.argv[3] # Fetch the problem ende_problem = problems.problem("translate_ende_wmt32k") # Get the encoders from the problem encoders = ende_problem.feature_encoders(ckpt_path) # Setup helper functions for encoding and decoding def encode(input_str, output_str=None): """Input str to features dict, ready for inference""" inputs = encoders["inputs"].encode(input_str) + [1] # add EOS id batch_inputs = tf.reshape(inputs, [1, -1, 1]) # Make it 3D. return {"inputs": batch_inputs} def decode(integers): """List of ints to str""" integers = list(np.squeeze(integers)) if 1 in integers:
data_dir = os.path.expanduser( "/home/huhan/20g_disk_d/python/projects/t2t/data") tmp_dir = os.path.expanduser("/home/huhan/20g_disk_d/python/projects/t2t/tmp") train_dir = os.path.expanduser( "/home/huhan/20g_disk_d/python/projects/t2t/train") checkpoint_dir = os.path.expanduser( "/home/huhan/20g_disk_d/python/projects/t2t/checkpoints") tf.gfile.MakeDirs(data_dir) tf.gfile.MakeDirs(tmp_dir) tf.gfile.MakeDirs(train_dir) tf.gfile.MakeDirs(checkpoint_dir) gs_ckpt_dir = "gs://tensor2tensor-checkpoints/" problem_name = "librispeech_clean" asr_problem = problems.problem(problem_name) encoders = asr_problem.feature_encoders(None) model_name = "transformer" hparams_set = "transformer_librispeech_tpu" hparams = trainer_lib.create_hparams(hparams_set, data_dir=data_dir, problem_name=problem_name) asr_model = registry.model(model_name)(hparams, Modes.PREDICT) def encode(x): waveforms = encoders["waveforms"].encode(x) encoded_dict = asr_problem.preprocess_example( {
return inputs def decode(integers): """List of ints to str""" integers = list(np.squeeze(integers)) if 1 in integers: integers = integers[:integers.index(1)] # 1 is EOS id return encoders["inputs"].decode(np.squeeze(integers)) data_dir = '/media/satan/3e75fcb2-d0e7-4b40-9145-34692ff66ff1/sunmingsheng/DataSet/t2t/data' tmp_dir = '/media/satan/3e75fcb2-d0e7-4b40-9145-34692ff66ff1/sunmingsheng/DataSet/t2t/tmp' dataLen = {'train': 287113, 'test': 11490, 'dev': 13368} cnn_dailymail = problems.problem('summarize_cnn_dailymail32k') PADDING = 0 SOURCE_PADDING_LENGTH = 1000 TARGET_PADDING_LENGTH = 100 encoders = cnn_dailymail.feature_encoders(data_dir) kind = ['test', 'train', 'dev'] for k in kind: source_name = 'cnndm.{}.source'.format(k) target_name = 'cnndm.{}.target'.format(k) with open(os.path.join(tmp_dir, target_name), encoding='utf-8') as f0: targets = f0.readlines() targets = [t.split('\n')[0] for t in targets] # 去除换行符号 for i in tqdm(range(dataLen[k]), desc=target_name): targets[i] = encode(targets[i], max_length=TARGET_PADDING_LENGTH)
tfe = tf.contrib.eager tfe.enable_eager_execution() # Other setup Modes = tf.estimator.ModeKeys # Setup some directories data_dir = '/home/team55/notespace/data/t2t_big/data' # Create hparams and the model model_name = "transformer" hparams_set = "transformer_big" checkpoint_dir = '/home/team55/notespace/data/t2t_big/model' ckpt_path = tf.train.latest_checkpoint(checkpoint_dir) problem_name = 'jddc_big' jddc_problem = problems.problem(problem_name) # Get the encoders from the problem encoders = jddc_problem.feature_encoders(data_dir) hparams = trainer_lib.create_hparams(hparams_set, data_dir=data_dir, problem_name=problem_name) jddc_model = registry.model(model_name)(hparams, Modes.EVAL) # Setup helper functions for encoding and decoding def encode(input_str): """Input str to features dict, ready for inference""" inputs = encoders["inputs"].encode(input_str) + [1] # add EOS id batch_inputs = tf.reshape(inputs, [1, -1, 1]) # Make it 3D. return {"inputs": batch_inputs}