def __init__(self):
# 初始化训练集、验证集
self.train_set, self.valid_set = ImageDataSet().build()
# 初始化模型
self.model_save_dir = config['model_save_dir'] + '/model_weight'
self.model = CNN(config['max_length'], len(config['char_set']))
try:
tf.print("尝试加载模型文件..")
self.model.load_weights(self.model_save_dir)
tf.print("加载模型成功")
except:
tf.print("未读取到模型文件..")
# 初始化优化器
self.optimizer = tf.keras.optimizers.Adam(lr=0.001)
self.train_loss = metrics.Mean(name="train_loss")
self.train_char_metric = CharAcc(name="train_char_accuracy")
self.train_img_metric = ImgAcc(name="train_img_accuracy")
self.valid_loss = metrics.Mean(name='valid_loss')
self.valid_char_metric = CharAcc(name="valid_char_metric")
self.valid_img_metric = ImgAcc(name="valid_img_metric")
self.val_step = iter(self.valid_set)
self.loss_list = [0]
self.less_loss = 999
self.wait = 0
# 初始化tensorboard
stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
self.summary_writer = tf.summary.create_file_writer(
os.path.join("logs/" + stamp))
def train(X_train, log_dir, writer):
fixed_seed = make_noise(BATCH_SIZE)
for epoch in range(NB_EPOCH):
g_train_loss, d_train_loss = metrics.Mean(), metrics.Mean()
for _ in range(TRAINING_RATIO):
d_loss = train_discriminator(X_train)
d_train_loss(d_loss)
g_loss = train_generator()
g_train_loss(g_loss)
da_loss = -np.mean(np.array(d_train_loss.result()))
ga_loss = -np.mean(np.array(g_train_loss.result()))
g_train_loss.reset_states()
d_train_loss.reset_states()
with writer.as_default():
scalar("da_loss", da_loss, step=epoch)
scalar("ga_loss", ga_loss, step=epoch)
save_images(generator, fixed_seed, writer, epoch)
writer.flush()
print(
f"Epoch: {epoch:05d}/{NB_EPOCH} = da_loss {da_loss:.5f}, ga_loss {ga_loss:.5f}"
)
def __init__(self, args):
super(WGAN_GP, self).__init__()
self.model_name = args.gan_type
self.batch_size = args.batch_size
self.z_dim = args.z_dim
self.lam = 10.
self.checkpoint_dir = check_folder(
os.path.join(args.checkpoint_dir, self.model_name))
self.result_dir = args.result_dir
self.datasets_name = args.datasets
self.log_dir = args.log_dir
self.learnning_rate = args.lr
self.epoches = args.epoch
self.datasets = load_mnist_data(dataset_name=self.datasets_name,
model_name="WGAN_GP")
self.g = self.make_generator_model(self.z_dim, is_training=True)
self.d = self.make_discriminator_model(is_training=True)
self.g_optimizer = keras.optimizers.RMSprop(lr=5 * self.learnning_rate)
self.d_optimizer = keras.optimizers.RMSprop(lr=self.learnning_rate)
self.g_loss_metric = metrics.Mean("g_loss", dtype=tf.float32)
self.critic_loss_metric = metrics.Mean("critic_loss", dtype=tf.float32)
self.checkpoint = tf.train.Checkpoint(
step=tf.Variable(0),
generator_optimizer=self.g_optimizer,
discriminator_optimizer=self.d_optimizer,
generator=self.g,
discriminator=self.d)
self.manager = tf.train.CheckpointManager(self.checkpoint,
self.checkpoint_dir,
max_to_keep=3)
def __init__(self, args):
super(LSGAN, self).__init__()
self.model_name = args.gan_type
self.batch_size = args.batch_size
self.z_dim = args.z_dim
self.checkpoint_dir = check_folder(
os.path.join(args.checkpoint_dir, self.model_name))
self.result_dir = args.result_dir
self.datasets_name = args.datasets
self.log_dir = args.log_dir
self.learnning_rate = args.lr
self.epoches = args.epoch
self.datasets = load_mnist_data(datasets=self.datasets_name,
batch_size=args.batch_size)
self.g = self.make_generator_model(self.z_dim, is_training=True)
self.d = self.make_discriminator_model(is_training=True)
self.g_optimizer = optimizers.Adam(lr=5 * self.learnning_rate,
beta_1=0.5)
self.d_optimizer = optimizers.Adam(lr=self.learnning_rate, beta_1=0.5)
self.g_loss_metric = metrics.Mean('g_loss', dtype=tf.float32)
self.d_loss_metric = metrics.Mean('d_loss', dtype=tf.float32)
self.checkpoint = tf.train.Checkpoint(
step=tf.Variable(0),
generator_optimizer=self.g_optimizer,
discriminator_optimizer=self.d_optimizer,
generator=self.g,
discriminator=self.d)
self.manager = tf.train.CheckpointManager(self.checkpoint,
self.checkpoint_dir,
max_to_keep=3)
def __init__(self):
super(kerasModel, self).__init__()
self.layersList = []
self.layersList.append(
kl.Dense(9,
activation="relu",
input_shape=(4, ),
use_bias=False,
kernel_initializer=ki.VarianceScaling(),
name="dense_1"))
self.layersList.append(
kl.Dense(1,
activation="sigmoid",
kernel_initializer=ki.VarianceScaling(),
use_bias=False,
name="out"))
self.loss = discountedLoss()
self.optimizer = ko.Adam(lr=1e-2)
self.train_loss = kme.Mean(name='train_loss')
self.validation_loss = kme.Mean(name='val_loss')
self.metric = kme.Accuracy(name="accuracy")
@tf.function()
def predict(x):
"""
This is where we run
through our whole dataset and return it, when training and testing.
"""
for l in self.layersList:
x = l(x)
return x
self.predict = predict
@tf.function()
def train_step(x, labels, adv):
"""
This is a TensorFlow function, run once for each epoch for the
whole input. We move forward first, then calculate gradients with
Gradient Tape to move backwards.
"""
with tf.GradientTape() as tape:
predictions = self.predict(x)
loss = self.loss.call(y_true=labels,
y_pred=predictions,
adv=adv)
gradients = tape.gradient(loss, self.trainable_variables)
self.optimizer.apply_gradients(
zip(gradients, self.trainable_variables))
self.train_loss(loss)
return loss
self.train_step = train_step
def train(self, NB_EPOCH, save_checkpoint, resume_from):
if resume_from != 0:
print(f"RESUMED_FROM: {resume_from}/{NB_EPOCH}")
fixed_seed = np.load(os.path.join(self.log_dir, 'seed.npy'))
fixed_seed = tf.constant(fixed_seed)
resume_from += 1
else:
fixed_seed = make_noise(self.sample_shape[0] * self.sample_shape[1], self.Z_SIZE)
np.save(os.path.join(self.log_dir, 'seed'), fixed_seed.numpy())
print(f"Training for {NB_EPOCH} epochs, NB_BATCHES: {self.NB_BATCHES}")
time_left = "is be determined"
for epoch in range(resume_from, NB_EPOCH):
ga_loss, da_loss = [], []
g_train_loss, d_train_loss = metrics.Mean(), metrics.Mean()
start_time = time.time()
for i, image_batch in enumerate(self.X_train.take(self.NB_BATCHES)):
print(f"Epoch: {epoch:05d}/{NB_EPOCH} in progress {i}/{self.NB_BATCHES} ending {time_left}", end='\r')
if self.tensor_to_img is True:
image_batch = self.extract_images_tensor(self.data_dir, image_batch)
for _ in range(self.TRAINING_RATIO):
d_loss = self.train_discriminator(image_batch)
d_train_loss(d_loss)
g_loss = self.train_generator()
g_train_loss(g_loss)
da_loss.append(d_train_loss.result())
ga_loss.append(g_train_loss.result())
g_train_loss.reset_states()
d_train_loss.reset_states()
da_loss = -np.mean(da_loss)
ga_loss = -np.mean(ga_loss)
with self.writer.as_default():
scalar("da_loss", da_loss, step=epoch)
scalar("ga_loss", ga_loss, step=epoch)
self.save_images(self.generator, fixed_seed, self.writer, epoch)
self.writer.flush()
if epoch % save_checkpoint == 0:
self.checkpoint.save(file_prefix=self.checkpoint_prefix)
date = time.strftime('%d/%m/%Y %H:%M:%S')
time_spent = time.time() - start_time
time_left = "in " + format_time(time_spent * (NB_EPOCH - epoch))
time_spent = format_time(time_spent)
if epoch % save_checkpoint == 0:
print(f"Epoch CHK: {epoch:05d}/{NB_EPOCH} {date} = da_loss {da_loss:.5f}, ga_loss {ga_loss:.5f}, time_spent {time_spent}")
else:
print(f"Epoch : {epoch:05d}/{NB_EPOCH} {date} = da_loss {da_loss:.5f}, ga_loss {ga_loss:.5f}, time_spent {time_spent}")
def train_epochs(train_dataset, val_dataset, model, epochs=50):
#-------------------------------------------------------------
# define parameters and customize configuration
EPOCHS = epochs # num Epochs of training
BS = 128 # batch size, the number of images to train at one train_step
INIT_LR = 1e-3 # initial learning rate
optimizer = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)
loss_object = losses.SparseCategoricalCrossentropy(name='train_loss')
train_loss = metrics.Mean(name='rain_loss')
train_accuracy = metrics.SparseCategoricalAccuracy(name='train_acc')
val_loss = metrics.Mean(name='val_loss')
val_accuracy = metrics.SparseCategoricalAccuracy(name='val_acc')
history = {'loss': [], 'acc': [], 'val_loss': [], 'val_acc': []}
for epoch in range(0, EPOCHS):
# show the current epoch number
sys.stdout.flush()
epochStart = time.time()
train_loss.reset_states()
train_accuracy.reset_states()
val_loss.reset_states()
val_accuracy.reset_states()
# for all training data set : 1237 images
for images, labels in train_dataset:
train_step(images, labels, model, loss_object, optimizer,
train_loss, train_accuracy)
# for all validation data set : 200 images
for val_images, val_labels in val_dataset:
val_step(val_images, val_labels, model, loss_object, val_loss,
val_accuracy)
template = 'Epoch {}/{}, loss: {}, acc: {}, val_loss: {}, val_acc: {}'
print(
template.format(epoch + 1, EPOCHS, train_loss.result(),
train_accuracy.result() * 100, val_loss.result(),
val_accuracy.result() * 100))
history['loss'].append(train_loss.result())
history['acc'].append(train_accuracy.result())
history['val_loss'].append(val_loss.result())
history['val_acc'].append(val_accuracy.result())
# show timing information for the epoch
epochEnd = time.time()
elapsed = (epochEnd - epochStart) / 60.0
print("took {:.4} minutes".format(elapsed))
# save weights
model.save('mnist.h5')
return history
def train(model_file='./mnist/trained_model', num_epochs=5, init=None):
train_ds, test_ds = load_data()
model = MNISTModel()
optimizer = optimizers.Adam()
train_loss = metrics.Mean(name='train_loss')
train_accuracy = metrics.CategoricalAccuracy(name='train_accuracy')
test_loss = metrics.Mean(name='test_loss')
test_accuracy = metrics.CategoricalAccuracy(name='test_accuracy')
if init != None:
model.load_weights(model_file)
@tf.function
def train_step(images, labels):
with tf.GradientTape() as tape:
logits = model(images)
loss_value = loss_object(labels, logits)
grads = tape.gradient(loss_value, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
train_loss(loss_value)
train_accuracy(labels, logits)
@tf.function
def test_step(images, labels):
logits = model(images)
loss_value = loss_object(labels, logits)
test_loss(loss_value)
test_accuracy(labels, logits)
def loss_object(labels, logits):
return tf.nn.softmax_cross_entropy_with_logits(labels=labels,
logits=logits)
for epoch in range(num_epochs):
for images, labels in train_ds:
train_step(images, labels)
for test_images, test_labels in test_ds:
test_step(test_images, test_labels)
template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'
print(
template.format(epoch + 1, train_loss.result(),
train_accuracy.result() * 100, test_loss.result(),
test_accuracy.result() * 100))
model.save_weights(model_file)
def test(model, dataset):
avg_loss = metrics.Mean('loss', dtype=tf.float32)
avg_accuracy = metrics.Mean('accuracy', dtype=tf.float32)
for (images, labels) in dataset:
logits = model(images, training=False)
avg_loss(compute_loss(labels, logits))
avg_accuracy(compute_accuracy(labels, logits))
print('# test_loss :',
avg_loss.result().numpy(), ', test accurary :',
avg_accuracy.result().numpy())
def train_model(model, train_data, train_len, dev_data, args):
optimizer = optimizers.Adam(learning_rate=args.lr)
train_loss = metrics.Mean(name='train_loss')
train_metric = metrics.SparseCategoricalAccuracy(name='train_accuracy')
valid_loss = metrics.Mean(name='valid_loss')
valid_metric = metrics.SparseCategoricalAccuracy(name='valid_accuracy')
loss_fun = losses.SparseCategoricalCrossentropy()
step = 0
best_valid_acc = 0
for epoch in tf.range(args.epochs):
for input_ids, input_mask, labels in train_data:
train_step(model, input_ids, input_mask, labels, optimizer,
train_loss, train_metric, loss_fun)
step += 1
# print('step',step)
if step % 100 == 0 and step % (
(int(train_len / args.batch_size / 2 / 100)) * 100) != 0:
logs = 'Epoch={},step={},Loss:{},Accuracy:{},Valid Loss:{},Valid Accuracy:{},best_valid_acc:{}'
tf.print(
tf.strings.format(
logs, (epoch, step, train_loss.result(),
train_metric.result(), valid_loss.result(),
valid_metric.result(), best_valid_acc)))
tf.print("")
if step % (
(int(train_len / args.batch_size / 2 / 100)) * 100) == 0:
for input_ids, input_mask, labels in dev_data:
valid_step(model, input_ids, input_mask, labels, optimizer,
valid_loss, valid_metric, loss_fun)
if valid_metric.result() >= best_valid_acc:
best_valid_acc = valid_metric.result()
save_path = args.model_save_path
# model.save(save_path,save_format='h5')
# model.save(save_path, save_format='tf')
model.save_weights(save_path, save_format='tf')
logs = 'Epoch={},step={},Loss:{},Accuracy:{},Valid Loss:{},Valid Accuracy:{},best_valid_acc:{}'
printbar()
tf.print(
tf.strings.format(
logs, (epoch, step, train_loss.result(),
train_metric.result(), valid_loss.result(),
valid_metric.result(), best_valid_acc)))
tf.print("")
train_loss.reset_states()
train_metric.reset_states()
valid_loss.reset_states()
valid_metric.reset_states()
def __init__(self, config):
self.mode = config["mode"]
self.data_config = config["dataset"]
self.aug_config = config["augmentation"]
self.get_dataset()
training_config = config["training"]
self.epochs = training_config["epochs"]
self.print_iter = training_config["print_iter"]
self.save_epoch = training_config["save_epoch"]
self.gene_AB = Generator()
self.gene_BA = Generator()
self.disc_A = Discriminator()
self.disc_B = Discriminator()
self.gene_AB_opt = optimizers.Adam(**training_config["g_optimizer"])
self.gene_BA_opt = optimizers.Adam(**training_config["g_optimizer"])
self.disc_A_opt = optimizers.Adam(**training_config["d_optimizer"])
self.disc_B_opt = optimizers.Adam(**training_config["d_optimizer"])
self.gene_AB_losses = metrics.Mean()
self.gene_BA_losses = metrics.Mean()
self.disc_A_losses = metrics.Mean()
self.disc_B_losses = metrics.Mean()
self.test_gene_AB_losses = metrics.Mean()
self.test_gene_BA_losses = metrics.Mean()
self.test_disc_A_losses = metrics.Mean()
self.test_disc_B_losses = metrics.Mean()
self.get_ckpt_manager(training_config["save_path"])
def __init__(self,
timesteps,
original_dim,
hidden_dim,
latent_dim,
RNN=tf.keras.layers.LSTM,
name="RVAE",
beta=1.0,
**kwargs):
super(RecurrentVariationalAutoEncoder, self).__init__(name=name, **kwargs)
self.timesteps = timesteps
self.original_dim = original_dim
self.hidden_dim = hidden_dim
self.latent_dim = latent_dim
# -- Metrics
self.vae_loss = metrics.Mean(name="vae_loss")
self.reconstruction_loss = metrics.Mean(name="reconstruction_loss")
self.kl_loss = metrics.Mean(name="kl_loss")
self.kl_annealed = metrics.Mean(name="kl_annealed")
self.accuracy = metrics.Mean(name="accuracy")
# -- Variables
self.kl_weight = tf.Variable(beta, trainable=False)
# -- Encoder
inputs = layers.Input(shape=(timesteps, original_dim,))
h_encoder = layers.LSTM(hidden_dim, return_sequences=True)(inputs)
h_encoder = layers.LSTM(hidden_dim)(h_encoder)
z_mean = layers.Dense(latent_dim, activation="linear")(h_encoder)
z_logvar = layers.Dense(latent_dim, activation="linear")(h_encoder)
z = layers.Lambda(sampling)([z_mean, z_logvar])
self.encoder = Model(inputs=[inputs], outputs=[z_mean, z_logvar, z], name="Encoder")
# -- Decoder
z_inputs = layers.Input(shape=(latent_dim,))
h_decoder = layers.RepeatVector(timesteps)(z_inputs)
h_decoder = layers.LSTM(hidden_dim, return_sequences=True)(h_decoder)
h_decoder = layers.LSTM(hidden_dim, return_sequences=True)(h_decoder)
outputs = layers.TimeDistributed(layers.Dense(original_dim))(h_decoder)
self.decoder = Model(inputs=[z_inputs], outputs=[outputs], name="Decoder")
def run(self):
train_loss = metrics.Mean(name='train_loss')
train_accuracy = metrics.SparseCategoricalAccuracy(
name='train_accuracy')
self.chooseLoss()
self.chooseOptimizer()
if self.loss_object is None:
return "Error choosing cost function in " + self.frame + ": " + self.loss + " not available in TensorFlow"
if self.optimizer_object is None:
return "Error choosing optimizer in " + self.frame + ": " + self.optimizer + " not available in TensorFlow"
with GradientTape() as tape:
for epoch in range(self.epoch):
predictions = self.model(self.inputTrain)
loss = self.loss_object(self.outputTrain, predictions)
gradients = tape.gradient(loss, self.model.trainable_variables)
self.optimizer_object.apply_gradients(
zip(gradients, self.model.trainable_variables))
train_loss(loss)
train_accuracy(self.outputTrain, predictions)
result = "Train --> Loss: " + str(
train_loss.result()) + ", Accuracy: " + str(
train_accuracy.result() * 100)
if self.test is True:
test_loss = metrics.Mean(name='test_loss')
test_accuracy = metrics.SparseCategoricalAccuracy(
name='test_accuracy')
predictions = self.model(self.inputTest)
t_loss = self.loss_object(self.outputTest, predictions)
test_loss(t_loss)
test_accuracy(self.outputTest, predictions)
result = result + "Test --> Loss: " + str(
test_loss.result()) + ", Accuracy: " + str(
test_accuracy.result() * 100)
self.saveModel()
self.logger("Trained " + self.frame + " model saved correctly!")
return result
def train(model, optimizer, dataset, log_freq=1000):
"""
Trains model on `dataset` using `optimizer`.
"""
start = time.time()
# Metrics are stateful. They accumulate values and return a cumulative
# result when you call .result(). Clear accumulated values with .reset_states()
train_avg_loss = metrics.Mean('loss', dtype=tf.float64)
# Datasets can be iterated over like any other Python iterable.
#cProfile.runctx('next(dataset.__iter__())', {}, {'dataset': dataset})
for (meta_data, images, labels) in dataset:
loss = train_step(model, optimizer, meta_data, images, labels)
train_avg_loss(loss)
if tf.equal(optimizer.iterations % log_freq, 0):
print('step:', int(optimizer.iterations), 'loss:',
train_avg_loss.result().numpy(), 'RMSE:',
np.sqrt(train_avg_loss.result().numpy()))
with train_summary_writer.as_default():
tf.summary.scalar('RMSE',
np.sqrt(train_avg_loss.result().numpy()),
step=optimizer.iterations)
train_avg_loss.reset_states()
end = time.time()
print('Epoch #{} ({} total steps): {}sec'.format(i + 1,
int(optimizer.iterations),
end - start))
def train(model: keras.Model,
optimizer: optimizers.Optimizer,
train_dataset: Dataset,
log_steps=50):
"""
Trainms model on `dataset` using `optimizer`.
:param train_dataset: tf.data.Dataset
:param model: the `keras.Model` being trained
:param optimizer: the `optimizers.Optimizer` being used
:return:
"""
loss_function = tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True)
mean_loss_metric = metrics.Mean('loss', dtype=tf.float32)
accuracy_metric = tf.keras.metrics.SparseCategoricalAccuracy()
for images, labels in train_dataset:
train_step(model, optimizer, images, labels, loss_function,
accuracy_metric, mean_loss_metric)
if optimizer.iterations % log_steps == 0:
print('batch:', int(optimizer.iterations), 'mean loss:',
mean_loss_metric.result().numpy(), 'accuracy:',
accuracy_metric.result().numpy())
mean_loss_metric.reset_states()
accuracy_metric.reset_states()
def train_epoch(epoch, train_dataset, model, optimizer):
summary_writer = tf.summary.create_file_writer(logdir='log')
loss_meter = metrics.Mean()
acc_meter = metrics.Accuracy()
for step, (x, y) in enumerate(train_dataset):
#记录下所有可训练变量的梯度，以便下一步更新
with tf.GradientTape() as tape:
x = tf.reshape(x, (-1, 28 * 28))
out = model(x)
acc_meter.update_state(y, out)
#计算单个方差
loss = tf.square(out - y)
#张量求和，也就是计算所有图片的均方误差和，计算均方差
loss = tf.reduce_mean(loss)
loss_meter.update_state(float(loss))
#获取每个变量对应的梯度值
grads = tape.gradient(loss, model.trainable_variables)
#根据每个变量的梯度值，更新相关变量的模型参数
optimizer.apply_gradients(zip(grads, model.trainable_variables))
with summary_writer.as_default():
tf.summary.scalar('train-loss', loss_meter.result(), step=100)
tf.summary.scalar('test-acc', acc_meter.result(), step=100)
#显示每训练100张图片之后的loss值
if (step % 100 == 0):
print(epoch, step, 'loss:', loss)
return model
def train(model, optimizer, dataset, log_freq=50):
"""Trains model on `dataset` using `optimizer`."""
start = time.time()
# Metrics are stateful. They accumulate values and return a cumulative
# result when you call .result(). Clear accumulated values with .reset_states()
avg_loss = metrics.Mean('loss', dtype=tf.float32)
# Datasets can be iterated over like any other Python iterable.
for images, labels in dataset:
loss = train_step(model, optimizer, images, labels)
avg_loss(loss)
if tf.equal(optimizer.iterations % log_freq, 0):
# summary_ops_v2.scalar('loss', avg_loss.result(), step=optimizer.iterations)
# summary_ops_v2.scalar('accuracy', compute_accuracy.result(), step=optimizer.iterations)
print('loss:',
avg_loss.result().numpy(), 'acc:',
compute_accuracy.result().numpy())
avg_loss.reset_states()
compute_accuracy.reset_states()
rate = log_freq / (time.time() - start)
print('Step #%d\tLoss: %.6f (%d steps/sec)' %
(int(optimizer.iterations), loss, rate))
start = time.time()
def train(model, optimizer, dataset, log_freq=50):
avg_loss = metrics.Mean('loss', dtype=tf.float32)
avg_accuracy = metrics.Mean('accuracy', dtype=tf.float32)
for images, labels in dataset:
loss, accuracy = train_step(model, optimizer, images, labels)
avg_loss(loss)
avg_accuracy(accuracy)
if tf.equal(optimizer.iterations % log_freq, 0):
print('# step:', int(optimizer.iterations), ', train_loss:',
avg_loss.result().numpy(), ', train_accuracy:',
avg_accuracy.result().numpy())
avg_loss.reset_states()
avg_accuracy.reset_states()
def test(model, dataset, step_num):
avg_loss = metrics.Mean('loss', dtype=tf.float32)
for (images, labels) in dataset:
logits = model(images, training=False)
avg_loss(compute_loss(labels, logits))
compute_accuracy(labels, logits)
print("Model test set loss:{:0.4f} accuracy:{.0.2f}".format(
avg_loss.result(),
compute_accuracy.result() * 100))
print('loss:', avg_loss.result(), 'acc:', compute_accuracy.result())
def test_loading():
"""Test to load model using different approahches."""
loss_fn = tf.keras.losses.MeanSquaredError()
loss_metric = metrics.Mean()
optimizer = tf.keras.optimizers.SGD(learning_rate=1e-3)
x_shape = (2, 2, 1)
custom_objects = {}
qkeras_utils._add_supported_quantized_objects(custom_objects)
train_ds = generate_dataset(train_size=1,
batch_size=1,
input_shape=x_shape,
num_class=2)
model_fold = get_qconv2d_batchnorm_model(input_shape=x_shape,
kernel_size=(2, 2),
folding_mode="ema_stats_folding")
model_fold.compile(loss=loss_fn, optimizer=optimizer, metrics="acc")
run_training(model_fold,
10,
loss_fn,
loss_metric,
optimizer,
train_ds,
do_print=False)
# test load model from json to ensure saving/loading model architecture works
json_string = model_fold.to_json()
clear_session()
model_from_json = qkeras_utils.quantized_model_from_json(json_string)
assert json_string == model_from_json.to_json()
# test reload model from hdf5 files to ensure saving/loading works
_, fname = tempfile.mkstemp(".h5")
model_fold.save(fname)
model_loaded = qkeras_utils.load_qmodel(fname)
weight1 = model_fold.layers[1].get_folded_quantized_weight()
weight2 = model_loaded.layers[1].get_folded_quantized_weight()
assert_equal(weight1[0], weight2[0])
assert_equal(weight1[1], weight2[1])
# test convert a folded model to a normal model for zpm
# the kernel/bias weight in the normal model should be the same as the folded
# kernel/bias in the folded model
normal_model = bn_folding_utils.convert_folded_model_to_normal(model_fold)
weight2 = normal_model.layers[1].get_weights()
assert_equal(weight1[0], weight2[0])
assert_equal(weight1[1], weight2[1])
def test(model, dataset, step_num):
"""perform an evaluation of model on the examples from dataset"""
avg_loss = metrics.Mean('loss', dtype=tf.float32)
for (images, labels) in dataset:
logits = model(images, training=False)
avg_loss(compute_loss(labels, logits))
compute_accuracy(labels, logits)
print('Model test set loss: {:0.4f} accuracy: {:0.2f}%'.format(
avg_loss.result(), compute_accuracy.result() * 100))
print('loss:', avg_loss.result(), 'acc:', compute_accuracy.result())
def __init__(self, config, datasets):
self.config = config
self.datasets = datasets
self.mode = config["mode"]
training_config = config["training"]
self.epochs = training_config["epochs"]
self.print_iter = training_config["print_iter"]
self.save_epoch = training_config["save_epoch"]
model_func = get_model(config["model"]["type"])
self.model = model_func(
**config[config["model"]["type"]]
)
lr = scheduler.CosineAnnealingScheduler(
training_config["init_learning_rate"],
training_config["epochs"]
)
self.optimizer = optimizers.Adam(lr, **config["optimizer"])
self.bce_w = training_config["bce_loss_weight"]
self.dice_w = training_config["dice_loss_weight"]
self.bce = losses.BinaryCrossEntropyLoss()
self.dice = losses.DiceLoss()
self.train_bce = metrics.Mean()
self.train_dice = metrics.Mean()
self.train_iou = metrics.MeanIoU(num_classes=2)
if config["mode"] == "train":
self.valid_datasets = DataLoader(
"valid", **config["dataset"]
)
self.test_bce = metrics.Mean()
self.test_dice = metrics.Mean()
self.test_iou = metrics.MeanIoU(num_classes=2)
self.get_ckpt_manager(config["save_path"])
def test(model, dataset):
"""
Perform an evaluation of `model` on the examples from `dataset`.
"""
valid_avg_loss = metrics.Mean('loss', dtype=tf.float64)
for (meta_data, images, labels) in dataset:
images, meta_data = preprocess(images, meta_data)
y_pred = model(meta_data, images, training=False)
valid_avg_loss(compute_loss(labels, y_pred))
print('valid loss: {:0.4f} RMSE: {:0.2f}'.format(
valid_avg_loss.result(), np.sqrt(valid_avg_loss.result().numpy())))
return np.sqrt(valid_avg_loss.result().numpy())
def test(model, dataset, step_num):
"""
Perform an evaluation of `model` on the examples from `dataset`.
"""
avg_loss = metrics.Mean('loss', dtype=tf.float32)
for (images, labels) in dataset:
logits = model(images, training=False)
avg_loss(
tf.keras.metrics.sparse_categorical_crossentropy(labels, logits))
accuracy1 = tf.keras.metrics.sparse_categorical_accuracy(
labels, logits)
print('Model test set loss: {:0.4f} accuracy: {:0.2f}%'.format(
avg_loss.result(),
accuracy1.result() * 100))
print('loss:', avg_loss.result(), 'acc:', accuracy1.result())
def __init__(self,
model,
learning_rate=0.001,
checkpoint_dir="model/checkpoints"):
self.ckpt = tf.train.Checkpoint(
epoch=tf.Variable(1),
val_loss=tf.Variable(np.inf),
optimizer=optimizers.Adam(lr=learning_rate),
model=model)
self.ckpt_manager = tf.train.CheckpointManager(
checkpoint=self.ckpt, directory=checkpoint_dir, max_to_keep=3)
self.restore_checkpoint()
self.loss_fn = losses.SparseCategoricalCrossentropy(from_logits=True)
self.train_metrics = [
metrics.Mean(),
metrics.SparseCategoricalAccuracy()
]
self.val_metrics = [
metrics.Mean(),
metrics.SparseCategoricalAccuracy()
]
def train(model, optimizer, dataset, log_freq=50):
# Metrics are stateful. They accumulate values and return a cumulative
# resutl when you call .result().Clear accumulated values with .reset_states()
avg_loss = metrics.Mean('loss', dtype=tf.float32)
for images, labels in dataset:
loss = train_step(model, optimizer, images, labels)
avg_loss(loss)
if tf.equal(optimizer.iterations % log_freq, 0):
print('step', int(optimizer.iterations), 'loss',
avg_loss.result().numpy(), 'acc',
compute_accuracy.result().numpy())
avg_loss.reset_states()
compute_accuracy.reset_states()
def test(model: keras.Model, test_dataset: Dataset):
"""
Evaluate `model` on examples from `dataset`
:param model: the model being tested
:param test_dataset: the test dataset (images,labels)
:param step_no:
:return:
"""
loss_function = tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True)
mean_loss_metric = metrics.Mean('loss', dtype=tf.float32)
accuracy_metric = tf.keras.metrics.SparseCategoricalAccuracy()
for (images, labels) in test_dataset:
logits = model(images, training=False)
mean_loss_metric(loss_function(labels, logits))
accuracy_metric(labels, logits)
print('Test set loss: {:0.4f} accuracy {:0.2f}%'.format(
mean_loss_metric.result(),
accuracy_metric.result() * 100))
def compute_labelled():
# cast y labelled to int and then one-hot
y_lab = tf.boolean_mask(y, labelled_idx, axis=0)
y_lab = tf.cast(K.flatten(y_lab), dtype=tf.int32)
y_lab_oh = tf.one_hot(y_lab, depth=self.n_classes)
# predict y_lab from X_lab
y_lab_pred = self.y_classifier(X_lab, training=training)
y_lab_pred = self.y_logits(y_lab_pred, training=training)
y_lab_pred = self.y_softmax(y_lab_pred)
y_lab_loss = self.y_bce(y_lab_oh, y_lab_pred)
# alpha is the relative weight between generative and discriminative learning
# paper uses 0.1 * N
alpha = 0.1 * X.shape[0]
agg_loss = alpha * y_lab_loss
# handle labelled data
X_y_lab = self.merger([X_lab, y_lab_oh])
Z_mu_lab, Z_log_var_lab, Z_lab = self.encode(X_y_lab,
training=training)
Z_y_lab = self.merger([Z_lab, y_lab_oh])
X_hat_lab_mu, X_hat_lab_log_var = self.decode(Z_y_lab,
training=training)
# log likelihood for labelled
like_lab = self.likelihood([
X_lab, Z_mu_lab, Z_log_var_lab, y_lab_oh, X_hat_lab_mu,
X_hat_lab_log_var
])
# can't apply to empty tensor - gives NaN
like_lab = K.mean(like_lab, axis=0) # take mean over batch
# flip sign and add to loss
like_lab *= -1
agg_loss += like_lab
# add related metrics
for key, metric in zip(
['likelihood_labelled', 'labelled_pred_bce'],
[like_lab, y_lab_loss]):
if key not in self.summary_metrics:
self.summary_metrics[key] = metrics.Mean(name=key,
dtype='float32')
self.summary_metrics[key](metric)
return agg_loss
def compute_unlabelled():
inner_likes = []
for idx in range(self.n_classes):
y_ula = idx * tf.ones(shape=tf.shape(X_ula)[0], dtype=tf.int32)
y_ula_oh = tf.one_hot(y_ula, depth=self.n_classes)
# encode and decode for each class
X_y_ula = self.merger([X_ula, y_ula_oh])
Z_mu_ula, Z_log_var_ula, Z_ula = self.encode(X_y_ula,
training=training)
Z_y_ula = self.merger([Z_ula, y_ula_oh])
X_hat_ula_mu, X_hat_ula_log_var = self.decode(
Z_y_ula, training=training)
# weld together likelihoods from each class
inner_like = self.likelihood([
X_ula, Z_mu_ula, Z_log_var_ula, y_ula_oh, X_hat_ula_mu,
X_hat_ula_log_var
])
inner_likes.append(K.expand_dims(inner_like, axis=-1))
like_ula = self.merger(inner_likes)
# score unlabelled
y_ula_pred = self.y_classifier(X_ula, training=training)
y_ula_pred = self.y_logits(y_ula_pred, training=training)
y_ula_pred = self.y_softmax(y_ula_pred)
# H(q(y|x))
like_ula = y_ula_pred * (like_ula -
K.log(y_ula_pred + K.epsilon()))
like_ula = K.sum(like_ula, axis=1) # sum over classes
like_ula = K.mean(like_ula) # take mean over batch
# flip sign
like_ula *= -1
agg_loss = like_ula
# add related metrics
for key, metric in zip(['likelihood_unlabelled'], [like_ula]):
if key not in self.summary_metrics:
self.summary_metrics[key] = metrics.Mean(name=key,
dtype='float32')
self.summary_metrics[key](metric)
return agg_loss
depth_t = len(ja_vocab.i2w)
input_dim = depth_x
hidden_dim = 128
output_dim = depth_t
model = EncoderDecoder(input_dim, hidden_dim, output_dim)
'''
3. モデルの学習・評価
'''
criterion = tf.losses.CategoricalCrossentropy()
optimizer = optimizers.Adam(learning_rate=0.001,
beta_1=0.9,
beta_2=0.999,
amsgrad=True)
train_loss = metrics.Mean()
val_loss = metrics.Mean()
def compute_loss(t, y):
return criterion(t, y)
def train_step(x, t, depth_t, teacher_forcing_rate=0.5):
use_teacher_forcing = (random.random() < teacher_forcing_rate)
with tf.GradientTape() as tape:
preds = model(x, t, use_teacher_forcing=use_teacher_forcing)
mask_t = tf.cast(tf.not_equal(t, 0), tf.float32)
t = tf.one_hot(t, depth=depth_t, dtype=tf.float32)
t = t * mask_t[:, :, tf.newaxis]
loss = compute_loss(t, preds)
grads = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
