def show_random_examples(problem_object, data_dir, num=1):
dataset = problem_object.dataset(Modes.TRAIN, data_dir)
input_dim = 8
output_dim = 32
for j in range(0, num):
offset = 1234
for i, example in enumerate(tfe.Iterator(dataset)):
if i > offset:
break
fig = plt.figure(figsize=(6, 6))
example["inputs"] = tf.reshape(example["inputs"],
[1, input_dim, input_dim, 3])
fig.add_subplot(1, 2, 1)
plt.imshow(example["inputs"].numpy()[0])
fig.add_subplot(1, 2, 2)
example["targets"] = tf.reshape(example["targets"],
[1, output_dim, output_dim, 3])
plt.imshow(example["targets"].numpy()[0])
def test(self, data_raw, target_raw, progress):
model_path = os.path.join(output_path, f"tcn_weight_{self.mode}.ckpt")
try:
self.model.load_weights(os.path.join(output_path, f"tcn_weight_{self.mode}.ckpt"))
except:
return None, None, None
data, target = process_data(self.timesteps, data_raw, target_raw)
num_input = len(data[0][0])
predict, gt = list(), list()
testset = tf.data.Dataset.from_tensor_slices((data, target))
progress.emit(0)
if self.use_gpu:
todevice = f"/gpu:{self.device}"
else:
todevice = f"/cpu:0"
with tf.device(todevice):
if self.mode == 1:
test = testset.batch(self.batch_size, drop_remainder=True)
for batch_x, batch_y in tfe.Iterator(test):
batch_x = tf.reshape(batch_x, (self.batch_size, self.timesteps, num_input))
batch_x = tf.dtypes.cast(batch_x, tf.float32)
batch_y = tf.dtypes.cast(batch_y, tf.float32)
logits = self.model(batch_x, training=False)
predict.extend(logits.numpy().flatten().tolist())
gt.extend(batch_y.numpy().flatten().tolist())
elif self.mode == 2:
for i in range(0, len(data), 8):
logits = None
if i + 10 > len(data):
break
for j in range(11):
x, y = data[i + j], target[i + j]
if logits != None:
x[-1][-1] = float(logits.numpy()[0][0])
x = tf.convert_to_tensor(x, dtype=tf.float32)
x = tf.reshape(x, (1, self.timesteps, num_input))
y = tf.convert_to_tensor(y, dtype=tf.float32)
logits = self.model(x, training=False)
if j > 2:
predict.append(logits.numpy()[0][0])
gt.append(y.numpy())
p = round(i / (len(data) - 1) * 100., 2)
progress.emit(p)
progress.emit(100)
test_predict = np.array(predict)
test_target = np.array(gt)
test_loss = round(rmse(test_predict, test_target).numpy() * 100., 2)
return test_predict, test_target, test_loss
def test(model, eval_data):
"""Computes the average loss on eval_data, which should be a Dataset."""
avg_loss = tfe.metrics.Mean("loss")
for (labels, chars, sequence_length) in tfe.Iterator(eval_data):
predictions = model(chars, sequence_length, training=False)
avg_loss(loss(labels, predictions))
print("eval/loss: %.6f\n" % avg_loss.result())
with tf.contrib.summary.always_record_summaries():
tf.contrib.summary.scalar("loss", avg_loss.result())
def example_apply_model(ckpt_path,
hparams_set="img2img_transformer2d_tiny",
problem_name="img2img_allen_brain_dim8to32",
model_name="img2img_transformer",
data_dir="/mnt/nfs-east1-d/data",
input_dim=8,
output_dim=32):
# HACK: Avoid re-instantiating the model which causes problems...
# TODO: Better way to handle this, e.g. delete from globals.
if 'model' not in globals():
hp = trainer_lib.create_hparams(hparams_set,
data_dir=data_dir,
problem_name=problem_name)
model = registry.model(model_name)(hp, Modes.TRAIN)
problem_object = problems.problem(problem_name)
dataset = problem_object.dataset(Modes.TRAIN, data_dir)
with tfe.restore_variables_on_create(ckpt_path):
for count, example in enumerate(tfe.Iterator(dataset)):
if count > 1234:
break
# Example input
fig = plt.figure(figsize=(8, 8))
example["inputs"] = tf.reshape(example["inputs"],
[1, input_dim, input_dim, 3])
fig.add_subplot(1, 3, 1)
plt.imshow(example["inputs"].numpy()[0])
# Example target
fig.add_subplot(1, 3, 2)
example["targets"] = tf.reshape(example["targets"],
[1, output_dim, output_dim, 3])
plt.imshow(example["targets"].numpy()[0])
# Dummy target (expected by model)
example["targets"] = tf.reshape(
tf.zeros((1, output_dim, output_dim, 3), dtype=np.uint8),
[1, output_dim, output_dim, 3])
# Produce and display prediction
predictions, _ = model(example)
fig.add_subplot(1, 3, 3)
inferred = demo.infer(predictions)
plt.imshow(inferred)
plt.show()
return example, predictions, inferred
def test(model, eval_data):
"""Computes the average loss on eval_data, which should be a Dataset."""
total_samples = 0
total_loss = 0.
for (labels, chars, sequence_length) in tfe.Iterator(eval_data):
predictions = model(chars, sequence_length, training=False)
batch_size = int(labels.shape[0])
total_samples += batch_size
total_loss += loss(labels, predictions) * batch_size
mean_loss = total_loss / total_samples
print("eval/loss: {:.6f}\n".format(float(mean_loss)))
if model.summary_writer is not None:
model.summary_writer.scalar("eval/loss", mean_loss)
def test_generates(self):
skip = False
if not skip:
problem_object = registry.problem("github_function_docstring")
data_dir = "/mnt/nfs-east1-d/data"
tmp_dir = "/mnt/nfs-east1-d/tmp"
problem_object = registry.problem("github_function_docstring")
problem_object.generate_data(data_dir, tmp_dir)
example = tfe.Iterator(
problem_object.dataset(Modes.TRAIN, data_dir)).next()
def main():
# Data dimensions & Hyperparameters.
n_classes = len(SPECIES)
learning_rate, epochs = 1e-2, 5
batch_size, buffer_size = 32, 1000
# Download and load the dataset.
train, test = load_data()
# Training dataset object.
train_data = make_dataset(data=train,
batch_size=batch_size,
buffer_size=buffer_size)
# Testing dataset object.
# noinspection PyUnusedLocal
test_data = make_dataset(data=test,
batch_size=batch_size,
buffer_size=buffer_size)
# Create model object, optimizer & global step.
model = Model(n_classes=n_classes)
optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate)
global_step = tf.train.get_or_create_global_step()
# Loop through training epochs.
for epoch in range(1, epochs + 1):
try:
# tfe.Iterator to access the data in the dataset.
for batch, (features, labels) in enumerate(tfe.Iterator(train_data)):
# Calculate gradients of loss w.r.t model variables.
grads = grad_func(model=model, inputs=features, targets=labels)
# noinspection PyTypeChecker
optimizer.apply_gradients(zip(grads, model.variables),
global_step=global_step)
loss = loss_func(model, features, labels)
print('\rEpoch: {}\tBatch: {:,}\tStep: {:,}\tLoss: {:.3f}'
.format(epoch, batch + 1, global_step.numpy(), loss),
end='')
except KeyboardInterrupt:
print('\nTraining interrupted by user.')
break
def train_one_epoch(model, optimizer, train_data, log_interval=10):
"""Trains model on train_data using optimizer."""
tf.train.get_or_create_global_step()
def model_loss(labels, chars, sequence_length):
predictions = model((chars, sequence_length), training=True)
loss_value = loss(labels, predictions)
tf.contrib.summary.scalar("loss", loss_value)
return loss_value
for (batch, (labels, chars, sequence_length)) in enumerate(
tfe.Iterator(train_data)):
with tf.contrib.summary.record_summaries_every_n_global_steps(log_interval):
batch_model_loss = functools.partial(model_loss, labels, chars,
sequence_length)
optimizer.minimize(
batch_model_loss, global_step=tf.train.get_global_step())
if log_interval and batch % log_interval == 0:
print("train/batch #%d\tloss: %.6f" % (batch, batch_model_loss()))
def train_one_epoch(model, optimizer, train_data, log_interval=None):
"""Trains model on train_data using optimizer."""
def model_loss(labels, chars, sequence_length):
predictions = model(chars, sequence_length, training=True)
return loss(labels, predictions)
# `grad` is a function that returns the gradients of model_loss with respect
# to all the variables that contribute to the computation of its output.
grad = tfe.implicit_gradients(model_loss)
batch = 0
for (labels, chars, sequence_length) in tfe.Iterator(train_data):
optimizer.apply_gradients(grad(labels, chars, sequence_length))
if log_interval and batch % log_interval == 0:
batch_loss = model_loss(labels, chars, sequence_length)
print("train/batch #{}\tloss: {:.6f}".format(
batch, float(batch_loss)))
if model.summary_writer is not None:
model.summary_writer.scalar("train/loss", batch_loss)
model.summary_writer.step()
batch += 1
def predict_ith(ckpt_path,
dataset,
model,
offset=1234,
input_dim=8,
output_dim=32):
"""Apply model to input obtained from dataset modulo offset.
Args:
offset (int): The offset within `dataset` where we will extract an example.
ckpt_path (str): The path from which checkpoints will be loaded.
dataset (`obj`): A tf.data dataset object.
model (`obj`): A tensor2tensor model object loaded from the registry.
"""
with tfe.restore_variables_on_create(ckpt_path):
for count, example in enumerate(tfe.Iterator(dataset)):
if count > offset:
break
fig = plt.figure(figsize=(8, 8))
example["inputs"] = tf.reshape(example["inputs"],
[1, input_dim, input_dim, 3])
fig.add_subplot(1, 3, 1)
plt.imshow(example["inputs"].numpy()[0])
fig.add_subplot(1, 3, 2)
example["targets"] = tf.reshape(example["targets"],
[1, output_dim, output_dim, 3])
plt.imshow(example["targets"].numpy()[0])
example["targets"] = tf.reshape(
tf.zeros((1, output_dim, output_dim, 3), dtype=np.uint8),
[1, output_dim, output_dim, 3])
predictions, _ = model(example)
fig.add_subplot(1, 3, 3)
inferred = infer(predictions)
plt.imshow(inferred)
plt.show()
return example, predictions, inferred
epochs = 1000
logdir = '../logs/iris-eager'
acc = 0.0
_loss = 0.
# File writer for tensorboard visualization.
writer = summary.create_file_writer(logdir=logdir)
with writer.as_default():
with summary.always_record_summaries():
print('\n{}'.format(75 * '-'))
print('Training started: {:%c}\n'.format(datetime.now()))
start = time.time()
for epoch in range(epochs):
# Loop through each data batches.
for X_batch, y_batch in tfe.Iterator(train_data):
# Run the training step.
train_step(model, optimizer, loss, X_batch, y_batch)
# Estimate loss
_loss = loss(model, X_batch, y_batch)
tf.summary.scalar('loss', _loss)
# Estimate accuracy.
y_pred = tf.argmax(model(X_batch), axis=1,
output_type=tf.int32)
train_accuracy(y_pred, y_batch)
acc = train_accuracy.result()
tf.summary.scalar('accuracy', acc)
denorm_x = denorm(logits, _min, _max)
denorm_y = denorm(batch_y, _min, _max)
lossL2 = tf.add_n(model.losses)
return rmse(denorm_x, denorm_y) + lossL2
with tf.device(f"/gpu:{GPU}"):
for epoch in range(1, epochs + 1):
train = trainset.batch(batch_size, drop_remainder=True)
for batch_x, batch_y in tfe.Iterator(train):
batch_x = tf.reshape(batch_x, (batch_size, timesteps, num_input))
batch_x = tf.dtypes.cast(batch_x, tf.float32)
batch_y = tf.dtypes.cast(batch_y, tf.float32)
optimizer.minimize(lambda: loss_function(batch_x, batch_y),
global_step=global_step)
logits = model(batch_x, training=False)
denorm_x = denorm(logits, _min, _max)
denorm_y = denorm(batch_y, _min, _max)
train_loss = rmse(denorm_x, denorm_y)
train_losses.append(train_loss.numpy())
predict, target = list(), list()
for i in range(0, len(dataset.test_data), 8):
logits = None
# Copy the vocab file locally so we can encode inputs and decode model outputs
# All vocabs are stored on GCS
vocab_file = os.path.join(gs_data_dir, "vocab.lmptb.10000")
#!/home/rlmcavoy/.local/bin/gsutil cp {vocab_file} {data_dir}
# The generate_data method of a problem will download data and process it into
# a standard format ready for training and evaluation.
ptb_problem.generate_data(data_dir, tmp_dir)
# Get the encoders from the problem
encoders = ptb_problem.feature_encoders(data_dir)
# Now let's see the training MNIST data as Tensors.
ptb_example = tfe.Iterator(ptb_problem.dataset(Modes.TRAIN, data_dir)).next()
print(ptb_example)
# Create your own model
class MySimpleModel(t2t_model.T2TModel):
def body(self, features):
#print(2.1)
if self._hparams.initializer == "orthogonal":
raise ValueError("LSTM models fail with orthogonal initializer.")
train = self._hparams.mode == tf.estimator.ModeKeys.TRAIN
inputs = features["targets"]
encoder_outputs = common_layers.flatten4d3d(inputs)
def train(self, data_raw, target_raw):
# progress.emit(0)
data, target = process_data(self.timesteps, data_raw, target_raw)
num_input = len(data[0][0])
trainset = tf.data.Dataset.from_tensor_slices((data, target))
if self.use_gpu:
todevice = f"/gpu:{self.device}"
else:
todevice = f"/cpu:0"
train_losses = list()
with tf.device(todevice):
for epoch in range(1, self.epochs + 1):
train = trainset.batch(self.batch_size, drop_remainder=True)
for batch_x, batch_y in tfe.Iterator(train):
batch_x = tf.reshape(batch_x, (self.batch_size, self.timesteps, num_input))
batch_x = tf.dtypes.cast(batch_x, tf.float32)
batch_y = tf.dtypes.cast(batch_y, tf.float32)
self.optimizer.minimize(lambda: self.loss_function(batch_x, batch_y), global_step=self.global_step)
predict, gt = list(), list()
if self.mode == 1:
for batch_x, batch_y in tfe.Iterator(train):
batch_x = tf.reshape(batch_x, (self.batch_size, self.timesteps, num_input))
batch_x = tf.dtypes.cast(batch_x, tf.float32)
batch_y = tf.dtypes.cast(batch_y, tf.float32)
logits = self.model(batch_x, training=False)
predict.extend(logits.numpy().flatten().tolist())
gt.extend(batch_y.numpy().flatten().tolist())
elif self.mode == 2:
for i in range(0, len(data), 8):
logits = None
if i + 10 > len(data):
break
for j in range(11):
x, y = data[i + j], target[i + j]
if logits != None:
x[-1][-1] = float(logits.numpy()[0][0])
x = tf.convert_to_tensor(x, dtype=tf.float32)
x = tf.reshape(x, (1, self.timesteps, num_input))
y = tf.convert_to_tensor(y, dtype=tf.float32)
logits = self.model(x, training=False)
if j > 2:
predict.append(logits.numpy()[0][0])
gt.append(y.numpy())
predict = np.array(predict)
gt = np.array(gt)
train_losses.append(round(rmse(predict, gt).numpy() * 100., 2))
# p = round(epoch / (self.epochs + 1) * 100.)
# progress.emit(p)
yield np.array(train_losses)
# progress.emit(100)
self.model.save_weights(os.path.join(output_path, f"tcn_weight_{self.mode}.ckpt"))
return np.array(train_losses)
Arguments:
model {tf.keras.Model} -- Instance of tf.keras.Model
optimizer {tf.train.Optimizer} -- Optimizer to be used.
loss {loss} -- Loss function.
x {tf.Tensor} -- Input features.
y {tf.Tensor} -- Output labels.
"""
optimizer.minimize(loss=lambda: loss(model, x, y),
global_step=tf.train.get_or_create_global_step())
model = Network()
optimizer = tf.train.AdamOptimizer(learning_rate=1e-2)
# Loop through each batch in the dataset.
for b, (_, y_batch) in enumerate(tfe.Iterator(train_data)):
y_batch = y_batch.numpy() # Convert to numpy array.
print('\nBatch {}'.format(b))
# Loop through each class.
for i, class_names in TARGET_NAMES.items():
# Get the index of class occurrences.
indices = np.where(y_batch == i)[0]
per = len(y_batch[indices]) / len(y_batch)
print('Percentage of class {!r}: {:>.2%}'.format(
class_names.title(), per))
logdir = '../logs/iris-classification'
epochs = 200
accuracies = np.zeros(epochs)
writer = summary.create_file_writer(logdir=logdir)
def test_runs(self):
problem_object = allen_brain.Img2imgAllenBrainDim8to32()
with TemporaryDirectory() as tmp_dir:
mock_raw_data(tmp_dir, raw_dim=256, num_images=730)
with TemporaryDirectory() as data_dir:
problem_object.generate_data(data_dir, tmp_dir)
input_xy_dim = problem_object.input_dim
target_xy_dim = problem_object.output_dim
num_channels = problem_object.num_channels
hparams = models.img2img_transformer_2d_adversarial()
hparams.data_dir = data_dir
hparams.discriminator_tanh_output = True
hparams.label_smoothing_factor = 0.1
hparams.discriminator_num_filters = 64
hparams.hidden_size = 64
hparams.loss_variant = "hybrid_vanilla_gan"
hparams.filter_size = 64
hparams.batch_size = 1
p_hparams = problem_object.get_hparams(hparams)
model = models.Img2imgTransformerAdversarial(
hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
@tfe.implicit_value_and_gradients
def loss_fn(features):
_, losses = model(features)
if hparams.loss_variant == "non_gan":
return losses["training"]
elif hparams.loss_variant == "vanilla_gan":
return losses["discriminator_loss"] + losses[
"generator_loss"]
elif hparams.loss_variant == "hybrid_vanilla_gan":
return losses["training"] + losses[
"discriminator_loss"] + losses["generator_loss"]
batch_size = 1
train_dataset = problem_object.dataset(Modes.TRAIN, data_dir)
train_dataset = train_dataset.repeat(None).batch(batch_size)
optimizer = tf.train.AdamOptimizer()
example = tfe.Iterator(train_dataset).next()
example["targets"] = tf.reshape(
example["targets"],
[batch_size, target_xy_dim, target_xy_dim, num_channels])
a, gv = loss_fn(example)
if hparams.loss_variant == "hybrid_vanilla_gan":
# Check that the loss is near 7.66
self.assertTrue(abs(a.numpy() - 7.66) < 1)
elif hparams.loss_variant == "non_gan":
# Check that the loss is near 5.56
self.assertTrue(abs(a.numpy() - 5.56) < 1)
optimizer.apply_gradients(gv)
import tensorflow as tf
import numpy as np
from tensorflow.contrib.eager.python import tfe
tf.enable_eager_execution()
# 创建dataset
dataset = tf.data.Dataset.from_tensor_slices(
np.array([1.0, 2.0, 3.0, 4.0, 5.0]))
# # 实例化了一个Iterator
# iterator = dataset.make_one_shot_iterator()
# # 从iterator里取出一个元素
# one_element = iterator.get_next()
for one_element in tfe.Iterator(dataset):
print(one_element)
def accuracy(target, prediction):
return tf.reduce_mean(
tf.cast(
tf.equal(
tf.argmax(
target,
axis=1 # <- axis, nor dimension
),
tf.argmax(prediction, axis=1) # <- axis, not dimension
),
dtype=tf.float32))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
num_batches_per_epoch = x_train.shape[0] // batch_size
for epoch in range(epochs):
for batch, (x_batch, y_batch) in enumerate(tfe.Iterator(dataset)):
optimizer.minimize(lambda: loss(model, x_batch, y_batch),
tf.train.get_or_create_global_step())
if batch >= num_batches_per_epoch:
break
print("Training accuracy after epoch {:04d}: {:.3f}".format(
epoch + 1, accuracy(y_train, predict(model, x_train))))
# eager model
