def show_random_examples(problem_object, data_dir, num=1): dataset = problem_object.dataset(Modes.TRAIN, data_dir) input_dim = 8 output_dim = 32 for j in range(0, num): offset = 1234 for i, example in enumerate(tfe.Iterator(dataset)): if i > offset: break fig = plt.figure(figsize=(6, 6)) example["inputs"] = tf.reshape(example["inputs"], [1, input_dim, input_dim, 3]) fig.add_subplot(1, 2, 1) plt.imshow(example["inputs"].numpy()[0]) fig.add_subplot(1, 2, 2) example["targets"] = tf.reshape(example["targets"], [1, output_dim, output_dim, 3]) plt.imshow(example["targets"].numpy()[0])
def test(self, data_raw, target_raw, progress): model_path = os.path.join(output_path, f"tcn_weight_{self.mode}.ckpt") try: self.model.load_weights(os.path.join(output_path, f"tcn_weight_{self.mode}.ckpt")) except: return None, None, None data, target = process_data(self.timesteps, data_raw, target_raw) num_input = len(data[0][0]) predict, gt = list(), list() testset = tf.data.Dataset.from_tensor_slices((data, target)) progress.emit(0) if self.use_gpu: todevice = f"/gpu:{self.device}" else: todevice = f"/cpu:0" with tf.device(todevice): if self.mode == 1: test = testset.batch(self.batch_size, drop_remainder=True) for batch_x, batch_y in tfe.Iterator(test): batch_x = tf.reshape(batch_x, (self.batch_size, self.timesteps, num_input)) batch_x = tf.dtypes.cast(batch_x, tf.float32) batch_y = tf.dtypes.cast(batch_y, tf.float32) logits = self.model(batch_x, training=False) predict.extend(logits.numpy().flatten().tolist()) gt.extend(batch_y.numpy().flatten().tolist()) elif self.mode == 2: for i in range(0, len(data), 8): logits = None if i + 10 > len(data): break for j in range(11): x, y = data[i + j], target[i + j] if logits != None: x[-1][-1] = float(logits.numpy()[0][0]) x = tf.convert_to_tensor(x, dtype=tf.float32) x = tf.reshape(x, (1, self.timesteps, num_input)) y = tf.convert_to_tensor(y, dtype=tf.float32) logits = self.model(x, training=False) if j > 2: predict.append(logits.numpy()[0][0]) gt.append(y.numpy()) p = round(i / (len(data) - 1) * 100., 2) progress.emit(p) progress.emit(100) test_predict = np.array(predict) test_target = np.array(gt) test_loss = round(rmse(test_predict, test_target).numpy() * 100., 2) return test_predict, test_target, test_loss
def test(model, eval_data): """Computes the average loss on eval_data, which should be a Dataset.""" avg_loss = tfe.metrics.Mean("loss") for (labels, chars, sequence_length) in tfe.Iterator(eval_data): predictions = model(chars, sequence_length, training=False) avg_loss(loss(labels, predictions)) print("eval/loss: %.6f\n" % avg_loss.result()) with tf.contrib.summary.always_record_summaries(): tf.contrib.summary.scalar("loss", avg_loss.result())
def example_apply_model(ckpt_path, hparams_set="img2img_transformer2d_tiny", problem_name="img2img_allen_brain_dim8to32", model_name="img2img_transformer", data_dir="/mnt/nfs-east1-d/data", input_dim=8, output_dim=32): # HACK: Avoid re-instantiating the model which causes problems... # TODO: Better way to handle this, e.g. delete from globals. if 'model' not in globals(): hp = trainer_lib.create_hparams(hparams_set, data_dir=data_dir, problem_name=problem_name) model = registry.model(model_name)(hp, Modes.TRAIN) problem_object = problems.problem(problem_name) dataset = problem_object.dataset(Modes.TRAIN, data_dir) with tfe.restore_variables_on_create(ckpt_path): for count, example in enumerate(tfe.Iterator(dataset)): if count > 1234: break # Example input fig = plt.figure(figsize=(8, 8)) example["inputs"] = tf.reshape(example["inputs"], [1, input_dim, input_dim, 3]) fig.add_subplot(1, 3, 1) plt.imshow(example["inputs"].numpy()[0]) # Example target fig.add_subplot(1, 3, 2) example["targets"] = tf.reshape(example["targets"], [1, output_dim, output_dim, 3]) plt.imshow(example["targets"].numpy()[0]) # Dummy target (expected by model) example["targets"] = tf.reshape( tf.zeros((1, output_dim, output_dim, 3), dtype=np.uint8), [1, output_dim, output_dim, 3]) # Produce and display prediction predictions, _ = model(example) fig.add_subplot(1, 3, 3) inferred = demo.infer(predictions) plt.imshow(inferred) plt.show() return example, predictions, inferred
def test(model, eval_data): """Computes the average loss on eval_data, which should be a Dataset.""" total_samples = 0 total_loss = 0. for (labels, chars, sequence_length) in tfe.Iterator(eval_data): predictions = model(chars, sequence_length, training=False) batch_size = int(labels.shape[0]) total_samples += batch_size total_loss += loss(labels, predictions) * batch_size mean_loss = total_loss / total_samples print("eval/loss: {:.6f}\n".format(float(mean_loss))) if model.summary_writer is not None: model.summary_writer.scalar("eval/loss", mean_loss)
def test_generates(self): skip = False if not skip: problem_object = registry.problem("github_function_docstring") data_dir = "/mnt/nfs-east1-d/data" tmp_dir = "/mnt/nfs-east1-d/tmp" problem_object = registry.problem("github_function_docstring") problem_object.generate_data(data_dir, tmp_dir) example = tfe.Iterator( problem_object.dataset(Modes.TRAIN, data_dir)).next()
def main(): # Data dimensions & Hyperparameters. n_classes = len(SPECIES) learning_rate, epochs = 1e-2, 5 batch_size, buffer_size = 32, 1000 # Download and load the dataset. train, test = load_data() # Training dataset object. train_data = make_dataset(data=train, batch_size=batch_size, buffer_size=buffer_size) # Testing dataset object. # noinspection PyUnusedLocal test_data = make_dataset(data=test, batch_size=batch_size, buffer_size=buffer_size) # Create model object, optimizer & global step. model = Model(n_classes=n_classes) optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate) global_step = tf.train.get_or_create_global_step() # Loop through training epochs. for epoch in range(1, epochs + 1): try: # tfe.Iterator to access the data in the dataset. for batch, (features, labels) in enumerate(tfe.Iterator(train_data)): # Calculate gradients of loss w.r.t model variables. grads = grad_func(model=model, inputs=features, targets=labels) # noinspection PyTypeChecker optimizer.apply_gradients(zip(grads, model.variables), global_step=global_step) loss = loss_func(model, features, labels) print('\rEpoch: {}\tBatch: {:,}\tStep: {:,}\tLoss: {:.3f}' .format(epoch, batch + 1, global_step.numpy(), loss), end='') except KeyboardInterrupt: print('\nTraining interrupted by user.') break
def train_one_epoch(model, optimizer, train_data, log_interval=10): """Trains model on train_data using optimizer.""" tf.train.get_or_create_global_step() def model_loss(labels, chars, sequence_length): predictions = model((chars, sequence_length), training=True) loss_value = loss(labels, predictions) tf.contrib.summary.scalar("loss", loss_value) return loss_value for (batch, (labels, chars, sequence_length)) in enumerate( tfe.Iterator(train_data)): with tf.contrib.summary.record_summaries_every_n_global_steps(log_interval): batch_model_loss = functools.partial(model_loss, labels, chars, sequence_length) optimizer.minimize( batch_model_loss, global_step=tf.train.get_global_step()) if log_interval and batch % log_interval == 0: print("train/batch #%d\tloss: %.6f" % (batch, batch_model_loss()))
def train_one_epoch(model, optimizer, train_data, log_interval=None): """Trains model on train_data using optimizer.""" def model_loss(labels, chars, sequence_length): predictions = model(chars, sequence_length, training=True) return loss(labels, predictions) # `grad` is a function that returns the gradients of model_loss with respect # to all the variables that contribute to the computation of its output. grad = tfe.implicit_gradients(model_loss) batch = 0 for (labels, chars, sequence_length) in tfe.Iterator(train_data): optimizer.apply_gradients(grad(labels, chars, sequence_length)) if log_interval and batch % log_interval == 0: batch_loss = model_loss(labels, chars, sequence_length) print("train/batch #{}\tloss: {:.6f}".format( batch, float(batch_loss))) if model.summary_writer is not None: model.summary_writer.scalar("train/loss", batch_loss) model.summary_writer.step() batch += 1
def predict_ith(ckpt_path, dataset, model, offset=1234, input_dim=8, output_dim=32): """Apply model to input obtained from dataset modulo offset. Args: offset (int): The offset within `dataset` where we will extract an example. ckpt_path (str): The path from which checkpoints will be loaded. dataset (`obj`): A tf.data dataset object. model (`obj`): A tensor2tensor model object loaded from the registry. """ with tfe.restore_variables_on_create(ckpt_path): for count, example in enumerate(tfe.Iterator(dataset)): if count > offset: break fig = plt.figure(figsize=(8, 8)) example["inputs"] = tf.reshape(example["inputs"], [1, input_dim, input_dim, 3]) fig.add_subplot(1, 3, 1) plt.imshow(example["inputs"].numpy()[0]) fig.add_subplot(1, 3, 2) example["targets"] = tf.reshape(example["targets"], [1, output_dim, output_dim, 3]) plt.imshow(example["targets"].numpy()[0]) example["targets"] = tf.reshape( tf.zeros((1, output_dim, output_dim, 3), dtype=np.uint8), [1, output_dim, output_dim, 3]) predictions, _ = model(example) fig.add_subplot(1, 3, 3) inferred = infer(predictions) plt.imshow(inferred) plt.show() return example, predictions, inferred
epochs = 1000 logdir = '../logs/iris-eager' acc = 0.0 _loss = 0. # File writer for tensorboard visualization. writer = summary.create_file_writer(logdir=logdir) with writer.as_default(): with summary.always_record_summaries(): print('\n{}'.format(75 * '-')) print('Training started: {:%c}\n'.format(datetime.now())) start = time.time() for epoch in range(epochs): # Loop through each data batches. for X_batch, y_batch in tfe.Iterator(train_data): # Run the training step. train_step(model, optimizer, loss, X_batch, y_batch) # Estimate loss _loss = loss(model, X_batch, y_batch) tf.summary.scalar('loss', _loss) # Estimate accuracy. y_pred = tf.argmax(model(X_batch), axis=1, output_type=tf.int32) train_accuracy(y_pred, y_batch) acc = train_accuracy.result() tf.summary.scalar('accuracy', acc)
denorm_x = denorm(logits, _min, _max) denorm_y = denorm(batch_y, _min, _max) lossL2 = tf.add_n(model.losses) return rmse(denorm_x, denorm_y) + lossL2 with tf.device(f"/gpu:{GPU}"): for epoch in range(1, epochs + 1): train = trainset.batch(batch_size, drop_remainder=True) for batch_x, batch_y in tfe.Iterator(train): batch_x = tf.reshape(batch_x, (batch_size, timesteps, num_input)) batch_x = tf.dtypes.cast(batch_x, tf.float32) batch_y = tf.dtypes.cast(batch_y, tf.float32) optimizer.minimize(lambda: loss_function(batch_x, batch_y), global_step=global_step) logits = model(batch_x, training=False) denorm_x = denorm(logits, _min, _max) denorm_y = denorm(batch_y, _min, _max) train_loss = rmse(denorm_x, denorm_y) train_losses.append(train_loss.numpy()) predict, target = list(), list() for i in range(0, len(dataset.test_data), 8): logits = None
# Copy the vocab file locally so we can encode inputs and decode model outputs # All vocabs are stored on GCS vocab_file = os.path.join(gs_data_dir, "vocab.lmptb.10000") #!/home/rlmcavoy/.local/bin/gsutil cp {vocab_file} {data_dir} # The generate_data method of a problem will download data and process it into # a standard format ready for training and evaluation. ptb_problem.generate_data(data_dir, tmp_dir) # Get the encoders from the problem encoders = ptb_problem.feature_encoders(data_dir) # Now let's see the training MNIST data as Tensors. ptb_example = tfe.Iterator(ptb_problem.dataset(Modes.TRAIN, data_dir)).next() print(ptb_example) # Create your own model class MySimpleModel(t2t_model.T2TModel): def body(self, features): #print(2.1) if self._hparams.initializer == "orthogonal": raise ValueError("LSTM models fail with orthogonal initializer.") train = self._hparams.mode == tf.estimator.ModeKeys.TRAIN inputs = features["targets"] encoder_outputs = common_layers.flatten4d3d(inputs)
def train(self, data_raw, target_raw): # progress.emit(0) data, target = process_data(self.timesteps, data_raw, target_raw) num_input = len(data[0][0]) trainset = tf.data.Dataset.from_tensor_slices((data, target)) if self.use_gpu: todevice = f"/gpu:{self.device}" else: todevice = f"/cpu:0" train_losses = list() with tf.device(todevice): for epoch in range(1, self.epochs + 1): train = trainset.batch(self.batch_size, drop_remainder=True) for batch_x, batch_y in tfe.Iterator(train): batch_x = tf.reshape(batch_x, (self.batch_size, self.timesteps, num_input)) batch_x = tf.dtypes.cast(batch_x, tf.float32) batch_y = tf.dtypes.cast(batch_y, tf.float32) self.optimizer.minimize(lambda: self.loss_function(batch_x, batch_y), global_step=self.global_step) predict, gt = list(), list() if self.mode == 1: for batch_x, batch_y in tfe.Iterator(train): batch_x = tf.reshape(batch_x, (self.batch_size, self.timesteps, num_input)) batch_x = tf.dtypes.cast(batch_x, tf.float32) batch_y = tf.dtypes.cast(batch_y, tf.float32) logits = self.model(batch_x, training=False) predict.extend(logits.numpy().flatten().tolist()) gt.extend(batch_y.numpy().flatten().tolist()) elif self.mode == 2: for i in range(0, len(data), 8): logits = None if i + 10 > len(data): break for j in range(11): x, y = data[i + j], target[i + j] if logits != None: x[-1][-1] = float(logits.numpy()[0][0]) x = tf.convert_to_tensor(x, dtype=tf.float32) x = tf.reshape(x, (1, self.timesteps, num_input)) y = tf.convert_to_tensor(y, dtype=tf.float32) logits = self.model(x, training=False) if j > 2: predict.append(logits.numpy()[0][0]) gt.append(y.numpy()) predict = np.array(predict) gt = np.array(gt) train_losses.append(round(rmse(predict, gt).numpy() * 100., 2)) # p = round(epoch / (self.epochs + 1) * 100.) # progress.emit(p) yield np.array(train_losses) # progress.emit(100) self.model.save_weights(os.path.join(output_path, f"tcn_weight_{self.mode}.ckpt")) return np.array(train_losses)
Arguments: model {tf.keras.Model} -- Instance of tf.keras.Model optimizer {tf.train.Optimizer} -- Optimizer to be used. loss {loss} -- Loss function. x {tf.Tensor} -- Input features. y {tf.Tensor} -- Output labels. """ optimizer.minimize(loss=lambda: loss(model, x, y), global_step=tf.train.get_or_create_global_step()) model = Network() optimizer = tf.train.AdamOptimizer(learning_rate=1e-2) # Loop through each batch in the dataset. for b, (_, y_batch) in enumerate(tfe.Iterator(train_data)): y_batch = y_batch.numpy() # Convert to numpy array. print('\nBatch {}'.format(b)) # Loop through each class. for i, class_names in TARGET_NAMES.items(): # Get the index of class occurrences. indices = np.where(y_batch == i)[0] per = len(y_batch[indices]) / len(y_batch) print('Percentage of class {!r}: {:>.2%}'.format( class_names.title(), per)) logdir = '../logs/iris-classification' epochs = 200 accuracies = np.zeros(epochs) writer = summary.create_file_writer(logdir=logdir)
def test_runs(self): problem_object = allen_brain.Img2imgAllenBrainDim8to32() with TemporaryDirectory() as tmp_dir: mock_raw_data(tmp_dir, raw_dim=256, num_images=730) with TemporaryDirectory() as data_dir: problem_object.generate_data(data_dir, tmp_dir) input_xy_dim = problem_object.input_dim target_xy_dim = problem_object.output_dim num_channels = problem_object.num_channels hparams = models.img2img_transformer_2d_adversarial() hparams.data_dir = data_dir hparams.discriminator_tanh_output = True hparams.label_smoothing_factor = 0.1 hparams.discriminator_num_filters = 64 hparams.hidden_size = 64 hparams.loss_variant = "hybrid_vanilla_gan" hparams.filter_size = 64 hparams.batch_size = 1 p_hparams = problem_object.get_hparams(hparams) model = models.Img2imgTransformerAdversarial( hparams, tf.estimator.ModeKeys.TRAIN, p_hparams) @tfe.implicit_value_and_gradients def loss_fn(features): _, losses = model(features) if hparams.loss_variant == "non_gan": return losses["training"] elif hparams.loss_variant == "vanilla_gan": return losses["discriminator_loss"] + losses[ "generator_loss"] elif hparams.loss_variant == "hybrid_vanilla_gan": return losses["training"] + losses[ "discriminator_loss"] + losses["generator_loss"] batch_size = 1 train_dataset = problem_object.dataset(Modes.TRAIN, data_dir) train_dataset = train_dataset.repeat(None).batch(batch_size) optimizer = tf.train.AdamOptimizer() example = tfe.Iterator(train_dataset).next() example["targets"] = tf.reshape( example["targets"], [batch_size, target_xy_dim, target_xy_dim, num_channels]) a, gv = loss_fn(example) if hparams.loss_variant == "hybrid_vanilla_gan": # Check that the loss is near 7.66 self.assertTrue(abs(a.numpy() - 7.66) < 1) elif hparams.loss_variant == "non_gan": # Check that the loss is near 5.56 self.assertTrue(abs(a.numpy() - 5.56) < 1) optimizer.apply_gradients(gv)
import tensorflow as tf import numpy as np from tensorflow.contrib.eager.python import tfe tf.enable_eager_execution() # 创建dataset dataset = tf.data.Dataset.from_tensor_slices( np.array([1.0, 2.0, 3.0, 4.0, 5.0])) # # 实例化了一个Iterator # iterator = dataset.make_one_shot_iterator() # # 从iterator里取出一个元素 # one_element = iterator.get_next() for one_element in tfe.Iterator(dataset): print(one_element)
def accuracy(target, prediction): return tf.reduce_mean( tf.cast( tf.equal( tf.argmax( target, axis=1 # <- axis, nor dimension ), tf.argmax(prediction, axis=1) # <- axis, not dimension ), dtype=tf.float32)) optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) num_batches_per_epoch = x_train.shape[0] // batch_size for epoch in range(epochs): for batch, (x_batch, y_batch) in enumerate(tfe.Iterator(dataset)): optimizer.minimize(lambda: loss(model, x_batch, y_batch), tf.train.get_or_create_global_step()) if batch >= num_batches_per_epoch: break print("Training accuracy after epoch {:04d}: {:.3f}".format( epoch + 1, accuracy(y_train, predict(model, x_train)))) # eager model