def __init__(self): # 初始化训练集、验证集 self.train_set, self.valid_set = ImageDataSet().build() # 初始化模型 self.model_save_dir = config['model_save_dir'] + '/model_weight' self.model = CNN(config['max_length'], len(config['char_set'])) try: tf.print("尝试加载模型文件..") self.model.load_weights(self.model_save_dir) tf.print("加载模型成功") except: tf.print("未读取到模型文件..") # 初始化优化器 self.optimizer = tf.keras.optimizers.Adam(lr=0.001) self.train_loss = metrics.Mean(name="train_loss") self.train_char_metric = CharAcc(name="train_char_accuracy") self.train_img_metric = ImgAcc(name="train_img_accuracy") self.valid_loss = metrics.Mean(name='valid_loss') self.valid_char_metric = CharAcc(name="valid_char_metric") self.valid_img_metric = ImgAcc(name="valid_img_metric") self.val_step = iter(self.valid_set) self.loss_list = [0] self.less_loss = 999 self.wait = 0 # 初始化tensorboard stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") self.summary_writer = tf.summary.create_file_writer( os.path.join("logs/" + stamp))
def train(X_train, log_dir, writer): fixed_seed = make_noise(BATCH_SIZE) for epoch in range(NB_EPOCH): g_train_loss, d_train_loss = metrics.Mean(), metrics.Mean() for _ in range(TRAINING_RATIO): d_loss = train_discriminator(X_train) d_train_loss(d_loss) g_loss = train_generator() g_train_loss(g_loss) da_loss = -np.mean(np.array(d_train_loss.result())) ga_loss = -np.mean(np.array(g_train_loss.result())) g_train_loss.reset_states() d_train_loss.reset_states() with writer.as_default(): scalar("da_loss", da_loss, step=epoch) scalar("ga_loss", ga_loss, step=epoch) save_images(generator, fixed_seed, writer, epoch) writer.flush() print( f"Epoch: {epoch:05d}/{NB_EPOCH} = da_loss {da_loss:.5f}, ga_loss {ga_loss:.5f}" )
def __init__(self, args): super(WGAN_GP, self).__init__() self.model_name = args.gan_type self.batch_size = args.batch_size self.z_dim = args.z_dim self.lam = 10. self.checkpoint_dir = check_folder( os.path.join(args.checkpoint_dir, self.model_name)) self.result_dir = args.result_dir self.datasets_name = args.datasets self.log_dir = args.log_dir self.learnning_rate = args.lr self.epoches = args.epoch self.datasets = load_mnist_data(dataset_name=self.datasets_name, model_name="WGAN_GP") self.g = self.make_generator_model(self.z_dim, is_training=True) self.d = self.make_discriminator_model(is_training=True) self.g_optimizer = keras.optimizers.RMSprop(lr=5 * self.learnning_rate) self.d_optimizer = keras.optimizers.RMSprop(lr=self.learnning_rate) self.g_loss_metric = metrics.Mean("g_loss", dtype=tf.float32) self.critic_loss_metric = metrics.Mean("critic_loss", dtype=tf.float32) self.checkpoint = tf.train.Checkpoint( step=tf.Variable(0), generator_optimizer=self.g_optimizer, discriminator_optimizer=self.d_optimizer, generator=self.g, discriminator=self.d) self.manager = tf.train.CheckpointManager(self.checkpoint, self.checkpoint_dir, max_to_keep=3)
def __init__(self, args): super(LSGAN, self).__init__() self.model_name = args.gan_type self.batch_size = args.batch_size self.z_dim = args.z_dim self.checkpoint_dir = check_folder( os.path.join(args.checkpoint_dir, self.model_name)) self.result_dir = args.result_dir self.datasets_name = args.datasets self.log_dir = args.log_dir self.learnning_rate = args.lr self.epoches = args.epoch self.datasets = load_mnist_data(datasets=self.datasets_name, batch_size=args.batch_size) self.g = self.make_generator_model(self.z_dim, is_training=True) self.d = self.make_discriminator_model(is_training=True) self.g_optimizer = optimizers.Adam(lr=5 * self.learnning_rate, beta_1=0.5) self.d_optimizer = optimizers.Adam(lr=self.learnning_rate, beta_1=0.5) self.g_loss_metric = metrics.Mean('g_loss', dtype=tf.float32) self.d_loss_metric = metrics.Mean('d_loss', dtype=tf.float32) self.checkpoint = tf.train.Checkpoint( step=tf.Variable(0), generator_optimizer=self.g_optimizer, discriminator_optimizer=self.d_optimizer, generator=self.g, discriminator=self.d) self.manager = tf.train.CheckpointManager(self.checkpoint, self.checkpoint_dir, max_to_keep=3)
def __init__(self): super(kerasModel, self).__init__() self.layersList = [] self.layersList.append( kl.Dense(9, activation="relu", input_shape=(4, ), use_bias=False, kernel_initializer=ki.VarianceScaling(), name="dense_1")) self.layersList.append( kl.Dense(1, activation="sigmoid", kernel_initializer=ki.VarianceScaling(), use_bias=False, name="out")) self.loss = discountedLoss() self.optimizer = ko.Adam(lr=1e-2) self.train_loss = kme.Mean(name='train_loss') self.validation_loss = kme.Mean(name='val_loss') self.metric = kme.Accuracy(name="accuracy") @tf.function() def predict(x): """ This is where we run through our whole dataset and return it, when training and testing. """ for l in self.layersList: x = l(x) return x self.predict = predict @tf.function() def train_step(x, labels, adv): """ This is a TensorFlow function, run once for each epoch for the whole input. We move forward first, then calculate gradients with Gradient Tape to move backwards. """ with tf.GradientTape() as tape: predictions = self.predict(x) loss = self.loss.call(y_true=labels, y_pred=predictions, adv=adv) gradients = tape.gradient(loss, self.trainable_variables) self.optimizer.apply_gradients( zip(gradients, self.trainable_variables)) self.train_loss(loss) return loss self.train_step = train_step
def train(self, NB_EPOCH, save_checkpoint, resume_from): if resume_from != 0: print(f"RESUMED_FROM: {resume_from}/{NB_EPOCH}") fixed_seed = np.load(os.path.join(self.log_dir, 'seed.npy')) fixed_seed = tf.constant(fixed_seed) resume_from += 1 else: fixed_seed = make_noise(self.sample_shape[0] * self.sample_shape[1], self.Z_SIZE) np.save(os.path.join(self.log_dir, 'seed'), fixed_seed.numpy()) print(f"Training for {NB_EPOCH} epochs, NB_BATCHES: {self.NB_BATCHES}") time_left = "is be determined" for epoch in range(resume_from, NB_EPOCH): ga_loss, da_loss = [], [] g_train_loss, d_train_loss = metrics.Mean(), metrics.Mean() start_time = time.time() for i, image_batch in enumerate(self.X_train.take(self.NB_BATCHES)): print(f"Epoch: {epoch:05d}/{NB_EPOCH} in progress {i}/{self.NB_BATCHES} ending {time_left}", end='\r') if self.tensor_to_img is True: image_batch = self.extract_images_tensor(self.data_dir, image_batch) for _ in range(self.TRAINING_RATIO): d_loss = self.train_discriminator(image_batch) d_train_loss(d_loss) g_loss = self.train_generator() g_train_loss(g_loss) da_loss.append(d_train_loss.result()) ga_loss.append(g_train_loss.result()) g_train_loss.reset_states() d_train_loss.reset_states() da_loss = -np.mean(da_loss) ga_loss = -np.mean(ga_loss) with self.writer.as_default(): scalar("da_loss", da_loss, step=epoch) scalar("ga_loss", ga_loss, step=epoch) self.save_images(self.generator, fixed_seed, self.writer, epoch) self.writer.flush() if epoch % save_checkpoint == 0: self.checkpoint.save(file_prefix=self.checkpoint_prefix) date = time.strftime('%d/%m/%Y %H:%M:%S') time_spent = time.time() - start_time time_left = "in " + format_time(time_spent * (NB_EPOCH - epoch)) time_spent = format_time(time_spent) if epoch % save_checkpoint == 0: print(f"Epoch CHK: {epoch:05d}/{NB_EPOCH} {date} = da_loss {da_loss:.5f}, ga_loss {ga_loss:.5f}, time_spent {time_spent}") else: print(f"Epoch : {epoch:05d}/{NB_EPOCH} {date} = da_loss {da_loss:.5f}, ga_loss {ga_loss:.5f}, time_spent {time_spent}")
def train_epochs(train_dataset, val_dataset, model, epochs=50): #------------------------------------------------------------- # define parameters and customize configuration EPOCHS = epochs # num Epochs of training BS = 128 # batch size, the number of images to train at one train_step INIT_LR = 1e-3 # initial learning rate optimizer = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS) loss_object = losses.SparseCategoricalCrossentropy(name='train_loss') train_loss = metrics.Mean(name='rain_loss') train_accuracy = metrics.SparseCategoricalAccuracy(name='train_acc') val_loss = metrics.Mean(name='val_loss') val_accuracy = metrics.SparseCategoricalAccuracy(name='val_acc') history = {'loss': [], 'acc': [], 'val_loss': [], 'val_acc': []} for epoch in range(0, EPOCHS): # show the current epoch number sys.stdout.flush() epochStart = time.time() train_loss.reset_states() train_accuracy.reset_states() val_loss.reset_states() val_accuracy.reset_states() # for all training data set : 1237 images for images, labels in train_dataset: train_step(images, labels, model, loss_object, optimizer, train_loss, train_accuracy) # for all validation data set : 200 images for val_images, val_labels in val_dataset: val_step(val_images, val_labels, model, loss_object, val_loss, val_accuracy) template = 'Epoch {}/{}, loss: {}, acc: {}, val_loss: {}, val_acc: {}' print( template.format(epoch + 1, EPOCHS, train_loss.result(), train_accuracy.result() * 100, val_loss.result(), val_accuracy.result() * 100)) history['loss'].append(train_loss.result()) history['acc'].append(train_accuracy.result()) history['val_loss'].append(val_loss.result()) history['val_acc'].append(val_accuracy.result()) # show timing information for the epoch epochEnd = time.time() elapsed = (epochEnd - epochStart) / 60.0 print("took {:.4} minutes".format(elapsed)) # save weights model.save('mnist.h5') return history
def train(model_file='./mnist/trained_model', num_epochs=5, init=None): train_ds, test_ds = load_data() model = MNISTModel() optimizer = optimizers.Adam() train_loss = metrics.Mean(name='train_loss') train_accuracy = metrics.CategoricalAccuracy(name='train_accuracy') test_loss = metrics.Mean(name='test_loss') test_accuracy = metrics.CategoricalAccuracy(name='test_accuracy') if init != None: model.load_weights(model_file) @tf.function def train_step(images, labels): with tf.GradientTape() as tape: logits = model(images) loss_value = loss_object(labels, logits) grads = tape.gradient(loss_value, model.trainable_variables) optimizer.apply_gradients(zip(grads, model.trainable_variables)) train_loss(loss_value) train_accuracy(labels, logits) @tf.function def test_step(images, labels): logits = model(images) loss_value = loss_object(labels, logits) test_loss(loss_value) test_accuracy(labels, logits) def loss_object(labels, logits): return tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits) for epoch in range(num_epochs): for images, labels in train_ds: train_step(images, labels) for test_images, test_labels in test_ds: test_step(test_images, test_labels) template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}' print( template.format(epoch + 1, train_loss.result(), train_accuracy.result() * 100, test_loss.result(), test_accuracy.result() * 100)) model.save_weights(model_file)
def test(model, dataset): avg_loss = metrics.Mean('loss', dtype=tf.float32) avg_accuracy = metrics.Mean('accuracy', dtype=tf.float32) for (images, labels) in dataset: logits = model(images, training=False) avg_loss(compute_loss(labels, logits)) avg_accuracy(compute_accuracy(labels, logits)) print('# test_loss :', avg_loss.result().numpy(), ', test accurary :', avg_accuracy.result().numpy())
def train_model(model, train_data, train_len, dev_data, args): optimizer = optimizers.Adam(learning_rate=args.lr) train_loss = metrics.Mean(name='train_loss') train_metric = metrics.SparseCategoricalAccuracy(name='train_accuracy') valid_loss = metrics.Mean(name='valid_loss') valid_metric = metrics.SparseCategoricalAccuracy(name='valid_accuracy') loss_fun = losses.SparseCategoricalCrossentropy() step = 0 best_valid_acc = 0 for epoch in tf.range(args.epochs): for input_ids, input_mask, labels in train_data: train_step(model, input_ids, input_mask, labels, optimizer, train_loss, train_metric, loss_fun) step += 1 # print('step',step) if step % 100 == 0 and step % ( (int(train_len / args.batch_size / 2 / 100)) * 100) != 0: logs = 'Epoch={},step={},Loss:{},Accuracy:{},Valid Loss:{},Valid Accuracy:{},best_valid_acc:{}' tf.print( tf.strings.format( logs, (epoch, step, train_loss.result(), train_metric.result(), valid_loss.result(), valid_metric.result(), best_valid_acc))) tf.print("") if step % ( (int(train_len / args.batch_size / 2 / 100)) * 100) == 0: for input_ids, input_mask, labels in dev_data: valid_step(model, input_ids, input_mask, labels, optimizer, valid_loss, valid_metric, loss_fun) if valid_metric.result() >= best_valid_acc: best_valid_acc = valid_metric.result() save_path = args.model_save_path # model.save(save_path,save_format='h5') # model.save(save_path, save_format='tf') model.save_weights(save_path, save_format='tf') logs = 'Epoch={},step={},Loss:{},Accuracy:{},Valid Loss:{},Valid Accuracy:{},best_valid_acc:{}' printbar() tf.print( tf.strings.format( logs, (epoch, step, train_loss.result(), train_metric.result(), valid_loss.result(), valid_metric.result(), best_valid_acc))) tf.print("") train_loss.reset_states() train_metric.reset_states() valid_loss.reset_states() valid_metric.reset_states()
def __init__(self, config): self.mode = config["mode"] self.data_config = config["dataset"] self.aug_config = config["augmentation"] self.get_dataset() training_config = config["training"] self.epochs = training_config["epochs"] self.print_iter = training_config["print_iter"] self.save_epoch = training_config["save_epoch"] self.gene_AB = Generator() self.gene_BA = Generator() self.disc_A = Discriminator() self.disc_B = Discriminator() self.gene_AB_opt = optimizers.Adam(**training_config["g_optimizer"]) self.gene_BA_opt = optimizers.Adam(**training_config["g_optimizer"]) self.disc_A_opt = optimizers.Adam(**training_config["d_optimizer"]) self.disc_B_opt = optimizers.Adam(**training_config["d_optimizer"]) self.gene_AB_losses = metrics.Mean() self.gene_BA_losses = metrics.Mean() self.disc_A_losses = metrics.Mean() self.disc_B_losses = metrics.Mean() self.test_gene_AB_losses = metrics.Mean() self.test_gene_BA_losses = metrics.Mean() self.test_disc_A_losses = metrics.Mean() self.test_disc_B_losses = metrics.Mean() self.get_ckpt_manager(training_config["save_path"])
def __init__(self, timesteps, original_dim, hidden_dim, latent_dim, RNN=tf.keras.layers.LSTM, name="RVAE", beta=1.0, **kwargs): super(RecurrentVariationalAutoEncoder, self).__init__(name=name, **kwargs) self.timesteps = timesteps self.original_dim = original_dim self.hidden_dim = hidden_dim self.latent_dim = latent_dim # -- Metrics self.vae_loss = metrics.Mean(name="vae_loss") self.reconstruction_loss = metrics.Mean(name="reconstruction_loss") self.kl_loss = metrics.Mean(name="kl_loss") self.kl_annealed = metrics.Mean(name="kl_annealed") self.accuracy = metrics.Mean(name="accuracy") # -- Variables self.kl_weight = tf.Variable(beta, trainable=False) # -- Encoder inputs = layers.Input(shape=(timesteps, original_dim,)) h_encoder = layers.LSTM(hidden_dim, return_sequences=True)(inputs) h_encoder = layers.LSTM(hidden_dim)(h_encoder) z_mean = layers.Dense(latent_dim, activation="linear")(h_encoder) z_logvar = layers.Dense(latent_dim, activation="linear")(h_encoder) z = layers.Lambda(sampling)([z_mean, z_logvar]) self.encoder = Model(inputs=[inputs], outputs=[z_mean, z_logvar, z], name="Encoder") # -- Decoder z_inputs = layers.Input(shape=(latent_dim,)) h_decoder = layers.RepeatVector(timesteps)(z_inputs) h_decoder = layers.LSTM(hidden_dim, return_sequences=True)(h_decoder) h_decoder = layers.LSTM(hidden_dim, return_sequences=True)(h_decoder) outputs = layers.TimeDistributed(layers.Dense(original_dim))(h_decoder) self.decoder = Model(inputs=[z_inputs], outputs=[outputs], name="Decoder")
def run(self): train_loss = metrics.Mean(name='train_loss') train_accuracy = metrics.SparseCategoricalAccuracy( name='train_accuracy') self.chooseLoss() self.chooseOptimizer() if self.loss_object is None: return "Error choosing cost function in " + self.frame + ": " + self.loss + " not available in TensorFlow" if self.optimizer_object is None: return "Error choosing optimizer in " + self.frame + ": " + self.optimizer + " not available in TensorFlow" with GradientTape() as tape: for epoch in range(self.epoch): predictions = self.model(self.inputTrain) loss = self.loss_object(self.outputTrain, predictions) gradients = tape.gradient(loss, self.model.trainable_variables) self.optimizer_object.apply_gradients( zip(gradients, self.model.trainable_variables)) train_loss(loss) train_accuracy(self.outputTrain, predictions) result = "Train --> Loss: " + str( train_loss.result()) + ", Accuracy: " + str( train_accuracy.result() * 100) if self.test is True: test_loss = metrics.Mean(name='test_loss') test_accuracy = metrics.SparseCategoricalAccuracy( name='test_accuracy') predictions = self.model(self.inputTest) t_loss = self.loss_object(self.outputTest, predictions) test_loss(t_loss) test_accuracy(self.outputTest, predictions) result = result + "Test --> Loss: " + str( test_loss.result()) + ", Accuracy: " + str( test_accuracy.result() * 100) self.saveModel() self.logger("Trained " + self.frame + " model saved correctly!") return result
def train(model, optimizer, dataset, log_freq=1000): """ Trains model on `dataset` using `optimizer`. """ start = time.time() # Metrics are stateful. They accumulate values and return a cumulative # result when you call .result(). Clear accumulated values with .reset_states() train_avg_loss = metrics.Mean('loss', dtype=tf.float64) # Datasets can be iterated over like any other Python iterable. #cProfile.runctx('next(dataset.__iter__())', {}, {'dataset': dataset}) for (meta_data, images, labels) in dataset: loss = train_step(model, optimizer, meta_data, images, labels) train_avg_loss(loss) if tf.equal(optimizer.iterations % log_freq, 0): print('step:', int(optimizer.iterations), 'loss:', train_avg_loss.result().numpy(), 'RMSE:', np.sqrt(train_avg_loss.result().numpy())) with train_summary_writer.as_default(): tf.summary.scalar('RMSE', np.sqrt(train_avg_loss.result().numpy()), step=optimizer.iterations) train_avg_loss.reset_states() end = time.time() print('Epoch #{} ({} total steps): {}sec'.format(i + 1, int(optimizer.iterations), end - start))
def train(model: keras.Model, optimizer: optimizers.Optimizer, train_dataset: Dataset, log_steps=50): """ Trainms model on `dataset` using `optimizer`. :param train_dataset: tf.data.Dataset :param model: the `keras.Model` being trained :param optimizer: the `optimizers.Optimizer` being used :return: """ loss_function = tf.keras.losses.SparseCategoricalCrossentropy( from_logits=True) mean_loss_metric = metrics.Mean('loss', dtype=tf.float32) accuracy_metric = tf.keras.metrics.SparseCategoricalAccuracy() for images, labels in train_dataset: train_step(model, optimizer, images, labels, loss_function, accuracy_metric, mean_loss_metric) if optimizer.iterations % log_steps == 0: print('batch:', int(optimizer.iterations), 'mean loss:', mean_loss_metric.result().numpy(), 'accuracy:', accuracy_metric.result().numpy()) mean_loss_metric.reset_states() accuracy_metric.reset_states()
def train_epoch(epoch, train_dataset, model, optimizer): summary_writer = tf.summary.create_file_writer(logdir='log') loss_meter = metrics.Mean() acc_meter = metrics.Accuracy() for step, (x, y) in enumerate(train_dataset): #记录下所有可训练变量的梯度,以便下一步更新 with tf.GradientTape() as tape: x = tf.reshape(x, (-1, 28 * 28)) out = model(x) acc_meter.update_state(y, out) #计算单个方差 loss = tf.square(out - y) #张量求和,也就是计算所有图片的均方误差和,计算均方差 loss = tf.reduce_mean(loss) loss_meter.update_state(float(loss)) #获取每个变量对应的梯度值 grads = tape.gradient(loss, model.trainable_variables) #根据每个变量的梯度值,更新相关变量的模型参数 optimizer.apply_gradients(zip(grads, model.trainable_variables)) with summary_writer.as_default(): tf.summary.scalar('train-loss', loss_meter.result(), step=100) tf.summary.scalar('test-acc', acc_meter.result(), step=100) #显示每训练100张图片之后的loss值 if (step % 100 == 0): print(epoch, step, 'loss:', loss) return model
def train(model, optimizer, dataset, log_freq=50): """Trains model on `dataset` using `optimizer`.""" start = time.time() # Metrics are stateful. They accumulate values and return a cumulative # result when you call .result(). Clear accumulated values with .reset_states() avg_loss = metrics.Mean('loss', dtype=tf.float32) # Datasets can be iterated over like any other Python iterable. for images, labels in dataset: loss = train_step(model, optimizer, images, labels) avg_loss(loss) if tf.equal(optimizer.iterations % log_freq, 0): # summary_ops_v2.scalar('loss', avg_loss.result(), step=optimizer.iterations) # summary_ops_v2.scalar('accuracy', compute_accuracy.result(), step=optimizer.iterations) print('loss:', avg_loss.result().numpy(), 'acc:', compute_accuracy.result().numpy()) avg_loss.reset_states() compute_accuracy.reset_states() rate = log_freq / (time.time() - start) print('Step #%d\tLoss: %.6f (%d steps/sec)' % (int(optimizer.iterations), loss, rate)) start = time.time()
def train(model, optimizer, dataset, log_freq=50): avg_loss = metrics.Mean('loss', dtype=tf.float32) avg_accuracy = metrics.Mean('accuracy', dtype=tf.float32) for images, labels in dataset: loss, accuracy = train_step(model, optimizer, images, labels) avg_loss(loss) avg_accuracy(accuracy) if tf.equal(optimizer.iterations % log_freq, 0): print('# step:', int(optimizer.iterations), ', train_loss:', avg_loss.result().numpy(), ', train_accuracy:', avg_accuracy.result().numpy()) avg_loss.reset_states() avg_accuracy.reset_states()
def test(model, dataset, step_num): avg_loss = metrics.Mean('loss', dtype=tf.float32) for (images, labels) in dataset: logits = model(images, training=False) avg_loss(compute_loss(labels, logits)) compute_accuracy(labels, logits) print("Model test set loss:{:0.4f} accuracy:{.0.2f}".format( avg_loss.result(), compute_accuracy.result() * 100)) print('loss:', avg_loss.result(), 'acc:', compute_accuracy.result())
def test_loading(): """Test to load model using different approahches.""" loss_fn = tf.keras.losses.MeanSquaredError() loss_metric = metrics.Mean() optimizer = tf.keras.optimizers.SGD(learning_rate=1e-3) x_shape = (2, 2, 1) custom_objects = {} qkeras_utils._add_supported_quantized_objects(custom_objects) train_ds = generate_dataset(train_size=1, batch_size=1, input_shape=x_shape, num_class=2) model_fold = get_qconv2d_batchnorm_model(input_shape=x_shape, kernel_size=(2, 2), folding_mode="ema_stats_folding") model_fold.compile(loss=loss_fn, optimizer=optimizer, metrics="acc") run_training(model_fold, 10, loss_fn, loss_metric, optimizer, train_ds, do_print=False) # test load model from json to ensure saving/loading model architecture works json_string = model_fold.to_json() clear_session() model_from_json = qkeras_utils.quantized_model_from_json(json_string) assert json_string == model_from_json.to_json() # test reload model from hdf5 files to ensure saving/loading works _, fname = tempfile.mkstemp(".h5") model_fold.save(fname) model_loaded = qkeras_utils.load_qmodel(fname) weight1 = model_fold.layers[1].get_folded_quantized_weight() weight2 = model_loaded.layers[1].get_folded_quantized_weight() assert_equal(weight1[0], weight2[0]) assert_equal(weight1[1], weight2[1]) # test convert a folded model to a normal model for zpm # the kernel/bias weight in the normal model should be the same as the folded # kernel/bias in the folded model normal_model = bn_folding_utils.convert_folded_model_to_normal(model_fold) weight2 = normal_model.layers[1].get_weights() assert_equal(weight1[0], weight2[0]) assert_equal(weight1[1], weight2[1])
def test(model, dataset, step_num): """perform an evaluation of model on the examples from dataset""" avg_loss = metrics.Mean('loss', dtype=tf.float32) for (images, labels) in dataset: logits = model(images, training=False) avg_loss(compute_loss(labels, logits)) compute_accuracy(labels, logits) print('Model test set loss: {:0.4f} accuracy: {:0.2f}%'.format( avg_loss.result(), compute_accuracy.result() * 100)) print('loss:', avg_loss.result(), 'acc:', compute_accuracy.result())
def __init__(self, config, datasets): self.config = config self.datasets = datasets self.mode = config["mode"] training_config = config["training"] self.epochs = training_config["epochs"] self.print_iter = training_config["print_iter"] self.save_epoch = training_config["save_epoch"] model_func = get_model(config["model"]["type"]) self.model = model_func( **config[config["model"]["type"]] ) lr = scheduler.CosineAnnealingScheduler( training_config["init_learning_rate"], training_config["epochs"] ) self.optimizer = optimizers.Adam(lr, **config["optimizer"]) self.bce_w = training_config["bce_loss_weight"] self.dice_w = training_config["dice_loss_weight"] self.bce = losses.BinaryCrossEntropyLoss() self.dice = losses.DiceLoss() self.train_bce = metrics.Mean() self.train_dice = metrics.Mean() self.train_iou = metrics.MeanIoU(num_classes=2) if config["mode"] == "train": self.valid_datasets = DataLoader( "valid", **config["dataset"] ) self.test_bce = metrics.Mean() self.test_dice = metrics.Mean() self.test_iou = metrics.MeanIoU(num_classes=2) self.get_ckpt_manager(config["save_path"])
def test(model, dataset): """ Perform an evaluation of `model` on the examples from `dataset`. """ valid_avg_loss = metrics.Mean('loss', dtype=tf.float64) for (meta_data, images, labels) in dataset: images, meta_data = preprocess(images, meta_data) y_pred = model(meta_data, images, training=False) valid_avg_loss(compute_loss(labels, y_pred)) print('valid loss: {:0.4f} RMSE: {:0.2f}'.format( valid_avg_loss.result(), np.sqrt(valid_avg_loss.result().numpy()))) return np.sqrt(valid_avg_loss.result().numpy())
def test(model, dataset, step_num): """ Perform an evaluation of `model` on the examples from `dataset`. """ avg_loss = metrics.Mean('loss', dtype=tf.float32) for (images, labels) in dataset: logits = model(images, training=False) avg_loss( tf.keras.metrics.sparse_categorical_crossentropy(labels, logits)) accuracy1 = tf.keras.metrics.sparse_categorical_accuracy( labels, logits) print('Model test set loss: {:0.4f} accuracy: {:0.2f}%'.format( avg_loss.result(), accuracy1.result() * 100)) print('loss:', avg_loss.result(), 'acc:', accuracy1.result())
def __init__(self, model, learning_rate=0.001, checkpoint_dir="model/checkpoints"): self.ckpt = tf.train.Checkpoint( epoch=tf.Variable(1), val_loss=tf.Variable(np.inf), optimizer=optimizers.Adam(lr=learning_rate), model=model) self.ckpt_manager = tf.train.CheckpointManager( checkpoint=self.ckpt, directory=checkpoint_dir, max_to_keep=3) self.restore_checkpoint() self.loss_fn = losses.SparseCategoricalCrossentropy(from_logits=True) self.train_metrics = [ metrics.Mean(), metrics.SparseCategoricalAccuracy() ] self.val_metrics = [ metrics.Mean(), metrics.SparseCategoricalAccuracy() ]
def train(model, optimizer, dataset, log_freq=50): # Metrics are stateful. They accumulate values and return a cumulative # resutl when you call .result().Clear accumulated values with .reset_states() avg_loss = metrics.Mean('loss', dtype=tf.float32) for images, labels in dataset: loss = train_step(model, optimizer, images, labels) avg_loss(loss) if tf.equal(optimizer.iterations % log_freq, 0): print('step', int(optimizer.iterations), 'loss', avg_loss.result().numpy(), 'acc', compute_accuracy.result().numpy()) avg_loss.reset_states() compute_accuracy.reset_states()
def test(model: keras.Model, test_dataset: Dataset): """ Evaluate `model` on examples from `dataset` :param model: the model being tested :param test_dataset: the test dataset (images,labels) :param step_no: :return: """ loss_function = tf.keras.losses.SparseCategoricalCrossentropy( from_logits=True) mean_loss_metric = metrics.Mean('loss', dtype=tf.float32) accuracy_metric = tf.keras.metrics.SparseCategoricalAccuracy() for (images, labels) in test_dataset: logits = model(images, training=False) mean_loss_metric(loss_function(labels, logits)) accuracy_metric(labels, logits) print('Test set loss: {:0.4f} accuracy {:0.2f}%'.format( mean_loss_metric.result(), accuracy_metric.result() * 100))
def compute_labelled(): # cast y labelled to int and then one-hot y_lab = tf.boolean_mask(y, labelled_idx, axis=0) y_lab = tf.cast(K.flatten(y_lab), dtype=tf.int32) y_lab_oh = tf.one_hot(y_lab, depth=self.n_classes) # predict y_lab from X_lab y_lab_pred = self.y_classifier(X_lab, training=training) y_lab_pred = self.y_logits(y_lab_pred, training=training) y_lab_pred = self.y_softmax(y_lab_pred) y_lab_loss = self.y_bce(y_lab_oh, y_lab_pred) # alpha is the relative weight between generative and discriminative learning # paper uses 0.1 * N alpha = 0.1 * X.shape[0] agg_loss = alpha * y_lab_loss # handle labelled data X_y_lab = self.merger([X_lab, y_lab_oh]) Z_mu_lab, Z_log_var_lab, Z_lab = self.encode(X_y_lab, training=training) Z_y_lab = self.merger([Z_lab, y_lab_oh]) X_hat_lab_mu, X_hat_lab_log_var = self.decode(Z_y_lab, training=training) # log likelihood for labelled like_lab = self.likelihood([ X_lab, Z_mu_lab, Z_log_var_lab, y_lab_oh, X_hat_lab_mu, X_hat_lab_log_var ]) # can't apply to empty tensor - gives NaN like_lab = K.mean(like_lab, axis=0) # take mean over batch # flip sign and add to loss like_lab *= -1 agg_loss += like_lab # add related metrics for key, metric in zip( ['likelihood_labelled', 'labelled_pred_bce'], [like_lab, y_lab_loss]): if key not in self.summary_metrics: self.summary_metrics[key] = metrics.Mean(name=key, dtype='float32') self.summary_metrics[key](metric) return agg_loss
def compute_unlabelled(): inner_likes = [] for idx in range(self.n_classes): y_ula = idx * tf.ones(shape=tf.shape(X_ula)[0], dtype=tf.int32) y_ula_oh = tf.one_hot(y_ula, depth=self.n_classes) # encode and decode for each class X_y_ula = self.merger([X_ula, y_ula_oh]) Z_mu_ula, Z_log_var_ula, Z_ula = self.encode(X_y_ula, training=training) Z_y_ula = self.merger([Z_ula, y_ula_oh]) X_hat_ula_mu, X_hat_ula_log_var = self.decode( Z_y_ula, training=training) # weld together likelihoods from each class inner_like = self.likelihood([ X_ula, Z_mu_ula, Z_log_var_ula, y_ula_oh, X_hat_ula_mu, X_hat_ula_log_var ]) inner_likes.append(K.expand_dims(inner_like, axis=-1)) like_ula = self.merger(inner_likes) # score unlabelled y_ula_pred = self.y_classifier(X_ula, training=training) y_ula_pred = self.y_logits(y_ula_pred, training=training) y_ula_pred = self.y_softmax(y_ula_pred) # H(q(y|x)) like_ula = y_ula_pred * (like_ula - K.log(y_ula_pred + K.epsilon())) like_ula = K.sum(like_ula, axis=1) # sum over classes like_ula = K.mean(like_ula) # take mean over batch # flip sign like_ula *= -1 agg_loss = like_ula # add related metrics for key, metric in zip(['likelihood_unlabelled'], [like_ula]): if key not in self.summary_metrics: self.summary_metrics[key] = metrics.Mean(name=key, dtype='float32') self.summary_metrics[key](metric) return agg_loss
depth_t = len(ja_vocab.i2w) input_dim = depth_x hidden_dim = 128 output_dim = depth_t model = EncoderDecoder(input_dim, hidden_dim, output_dim) ''' 3. モデルの学習・評価 ''' criterion = tf.losses.CategoricalCrossentropy() optimizer = optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=True) train_loss = metrics.Mean() val_loss = metrics.Mean() def compute_loss(t, y): return criterion(t, y) def train_step(x, t, depth_t, teacher_forcing_rate=0.5): use_teacher_forcing = (random.random() < teacher_forcing_rate) with tf.GradientTape() as tape: preds = model(x, t, use_teacher_forcing=use_teacher_forcing) mask_t = tf.cast(tf.not_equal(t, 0), tf.float32) t = tf.one_hot(t, depth=depth_t, dtype=tf.float32) t = t * mask_t[:, :, tf.newaxis] loss = compute_loss(t, preds) grads = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(grads, model.trainable_variables))