def test(): def imshow(ax, normalize_image, title): img = ((normalize_image + 1) * 127.5).astype(np.uint8)[0] ax.imshow(img) ax.axes.get_xaxis().set_visible(False) ax.axes.get_yaxis().set_visible(False) ax.set_title(title) epochs = [50, 100, 150, 200] models = ["facade_%d" % epoch for epoch in epochs] gans = [] for model in models: p2p = Pix2Pix() p2p.load_model(model) gans.append(p2p) for real, cond in load_batch("facade", "test", 256, 1): ax = plt.subplot(231) imshow(ax, cond, "conditional") ax = plt.subplot(232) imshow(ax, real, "real") for gi, (epoch, gan) in enumerate(zip(epochs, gans)): fake = gan.generator.predict(cond) ax = plt.subplot(2, 3, gi + 3) imshow(ax, fake, "epoch %d" % epoch) plt.show()
def train(self, dataset, batch_size=1, epochs=100, save=True): """ Train the gan. :param dataset: str, name of dataset, such as "facade" :param batch_size: int, batch size :param epochs: int, training epochs :param save: bool, whether save model after training """ for ei in range(epochs): if ei % 10 == 0 and ei > 0: h5_file = "%s/%s_%d.h5" % (MODEL_DIR, dataset, ei) self.gan.save_weights(h5_file) is_valid = np.ones((batch_size, 32, 32, 1)) is_invalid = np.zeros((batch_size, 32, 32, 1)) for bi, (real, cond) in enumerate(load_batch(dataset, "train", 256, batch_size)): fake = self.generator.predict(cond) # train discriminator d_bc_real, d_acc_real = self.discriminator.train_on_batch([real, cond], is_valid) d_bc_fake, d_acc_fake = self.discriminator.train_on_batch([fake, cond], is_invalid) d_bc = 0.5 * (d_bc_real + d_bc_fake) d_acc = 0.5 * (d_acc_real + d_acc_fake) # train generator g_metrics = self.gan.train_on_batch([real, cond], [is_valid, real]) g_loss, g_bc, g_mae = g_metrics print("epoch-%d batch-%d | d_bc:%.4f d_acc:%.4f | g_loss: %.4f g_bc:%.4f g_mae:%.4f" % (ei + 1, bi + 1, d_bc, d_acc, g_loss, g_bc, g_mae)) h5_file = "%s/%s_%d.h5" % (MODEL_DIR, dataset, epochs) self.gan.save_weights(h5_file) if save: h5_file = "%s/%s.h5" % (MODEL_DIR, dataset) self.gan.save_weights(h5_file)
def get_cost(f_cost, samples, batch, mat, costs, pgb, f_update=None, lrate=0.): ''' Compute the cost of the given batches and update the parameters if it is necessary ''' x, mask_x, y, mask_y = load_batch(samples, batch, mat) costs.extend(f_cost(x, mask_x, y, mask_y)) if f_update: f_update(lrate) if numpy.isnan(costs[-1]) or numpy.isinf(costs[-1]): pgb.pause() raise FloatingPointError, 'Bad cost detected!' pgb.disp(costs, 'COST')
def get_cost(f_cost, samples, batch, mat, costs, pgb, f_update=None, lrate=0.): ''' Compute the cost of the given batches and update the parameters if it is necessary ''' x, mask_x, y, mask_y = load_batch(samples, batch, mat) costs.extend(f_cost(x, mask_x, y, mask_y)) if f_update: f_update(lrate) if numpy.isnan(costs[-1]) or numpy.isinf(costs[-1]): pgb.pause() raise FloatingPointError, 'Bad cost detected!' pgb.disp(costs, 'COST')
def predict(f_enc, f_dec, samples, batches, mat, max_len, header): ''' Sample words and compute the prediction error ''' preds = [] errs = [] progress = ProgressBar(numpy.sum([len(batch) for batch in batches]), 20, header) for batch in batches: x, mask_x, y, mask_y = load_batch(samples, batch, mat) [prev_h] = f_enc(x, mask_x) n_steps = mask_x.sum(0) n_samples = x.shape[1] sents = numpy.zeros((n_samples, max_len), 'int32') # First step - No embedded word is fed into the decoder sents[:, 0], prev_h = f_dec(numpy.asarray([-1] * n_samples, 'int32'), prev_h) n_ends = n_steps - (sents[:, 0] == 0) for i in range(1, max_len - 1): prev_words = sents[:, i - 1] if not n_ends.any(): break next_words, prev_h = f_dec(prev_words, prev_h) sents[:, i] = next_words * (n_ends > 0) n_ends -= (next_words == 0) * (n_ends > 0) for i in range(n_samples): idx = 0 while idx < max_len and n_steps[i] > 0: if sents[i, idx] == 0: n_steps[i] -= 1 idx += 1 preds.append(sents[i, :idx].tolist()) y = numpy.concatenate( [y, numpy.zeros((max_len - len(y), n_samples), 'int32')]).T mask_y = numpy.concatenate( [mask_y, numpy.zeros((max_len - len(mask_y), n_samples), 'int32')]).T errs.extend(((sents != y) * mask_y * 1.).sum(1) / mask_y.sum(1)) progress.disp(errs, ' ERR') return preds, numpy.mean(errs)
def predict(f_enc, f_dec, samples, batches, mat, max_len, header): ''' Sample words and compute the prediction error ''' preds = [] errs = [] progress = ProgressBar(numpy.sum([len(batch) for batch in batches]), 20, header) for batch in batches: x, mask_x, y, mask_y = load_batch(samples, batch, mat) [prev_h] = f_enc(x, mask_x) n_steps = mask_x.sum(0) n_samples = x.shape[1] sents = numpy.zeros((n_samples, max_len), 'int32') # First step - No embedded word is fed into the decoder sents[:, 0], prev_h = f_dec(numpy.asarray([-1] * n_samples, 'int32'), prev_h) n_ends = n_steps - (sents[:, 0] == 0) for i in range(1, max_len - 1): prev_words = sents[:, i - 1] if not n_ends.any(): break next_words, prev_h = f_dec(prev_words, prev_h) sents[:, i] = next_words * (n_ends > 0) n_ends -= (next_words == 0) * (n_ends > 0) for i in range(n_samples): idx = 0 while idx < max_len and n_steps[i] > 0: if sents[i, idx] == 0: n_steps[i] -= 1 idx += 1 preds.append(sents[i, : idx].tolist()) y = numpy.concatenate([y, numpy.zeros((max_len - len(y), n_samples), 'int32')]).T mask_y = numpy.concatenate([mask_y, numpy.zeros((max_len - len(mask_y), n_samples), 'int32')]).T errs.extend(((sents != y) * mask_y * 1.).sum(1) / mask_y.sum(1)) progress.disp(errs, ' ERR') return preds, numpy.mean(errs)
def get_input(inFile, inType, modelType, offset=0., scale=1., winLen=None, winShift=10): if winLen is None: winLen = model.input_shape[3] typeInfo = modelType.split('_') timeDistributed = (len(typeInfo) > 1) and any( [x in typeInfo[1] for x in ['dist', 'stateful']]) if inType == 'tinyfeats': feaStream = audioproc.load_bin(inFile) if not timeDistributed: feaStream, starts = fbank_stream(feaStream, winLen) elif inType == 'audio': if os.path.isfile(inFile): feaStream = audioproc.wav2fbank(inFile) nFiles = 1 if not timeDistributed: feaStream, starts = fbank_stream(feaStream, winLen) elif os.path.isdir(inFile): # assumes each clip is winLen feaStream, files = audioproc.wav2fbank_batch(inFile) nFiles = len(files) if timeDistributed: feaStream = np.reshape( feaStream, (nFiles, feaStream.shape[0], feaStream.shape[1])) elif inType == 'serverfeats': feaStream = audioproc.load_serverfeats(inFile) elif inType == 'features': feaStream = data.load_batch(inFile, var='features') feaStream = data.apply_norm(feaStream, offset, scale) feaStream = data.reshape_for_model(feaStream, modelType) return feaStream
def predict(f_enc, f_dec, samples, batches, mat, beam_size, max_len, header): ''' Sample words and compute the prediction error ''' preds = [] errs = [] progress = ProgressBar(numpy.sum([len(batch) for batch in batches]), 20, header) for batch in batches: x, mask_x, y, mask_y = load_batch(samples, batch, mat) [init_h] = f_enc(x, mask_x) n_steps = mask_x.sum(0) n_samples = x.shape[1] prev_sents = numpy.zeros((beam_size, n_samples, max_len), 'int32') # First step - No embedded word is fed into the decoder prev_words = numpy.asarray([-1] * n_samples, 'int32') prev_sents[:, :, 0], prev_log_prob, prev_h = f_dec(prev_words, init_h) prev_h = numpy.tile(prev_h, (beam_size, 1, 1)) prev_n_ends = n_steps - (prev_sents[:, :, 0] == 0) for i in range(1, max_len - 1): hypo_sents = [[]] * n_samples hypo_log_prob = [[]] * n_samples hypo_h = [[]] * n_samples hypo_n_ends = [[]] * n_samples has_hypos = numpy.asarray([False] * n_samples) for j in range(beam_size): if not prev_n_ends[j].any(): continue next_words, next_log_prob, next_h = f_dec( prev_sents[j, :, i - 1], prev_h[j]) for k in range(n_samples): if prev_n_ends[j, k] > 0: has_hypos[k] = True next_sents = numpy.tile(prev_sents[j, k], (beam_size, 1)) next_sents[:, i] = next_words[:, k] hypo_sents[k].extend(next_sents) hypo_log_prob[k].extend(next_log_prob[:, k] + prev_log_prob[j, k]) hypo_h[k].extend([next_h[k]] * beam_size) hypo_n_ends[k].extend(prev_n_ends[j, k] - (next_words[:, k] == 0)) else: hypo_sents[k].append(prev_sents[j, k].copy()) hypo_log_prob[k].append(prev_log_prob[j, k]) hypo_h[k].append(prev_h[j, k].copy()) hypo_n_ends[k].append(0) if not has_hypos.any(): break for j in range(n_samples): if not has_hypos[j]: continue indices = numpy.argsort(hypo_log_prob[j])[:-beam_size - 1:-1] for k in range(beam_size): prev_sents[k, j] = hypo_sents[j][indices[k]] prev_log_prob[k, j] = hypo_log_prob[j][indices[k]] prev_h[k, j] = hypo_h[j][indices[k]] prev_n_ends[k, j] = hypo_n_ends[j][indices[k]] sents = prev_sents[prev_log_prob.argmax(0), numpy.arange(n_samples)] for i in range(n_samples): idx = 0 while idx < max_len and n_steps[i] > 0: if sents[i, idx] == 0: n_steps[i] -= 1 idx += 1 preds.append(sents[i, :idx].tolist()) y = numpy.concatenate( [y, numpy.zeros((max_len - len(y), n_samples), 'int32')]).T mask_y = numpy.concatenate( [mask_y, numpy.zeros((max_len - len(mask_y), n_samples), 'int32')]).T errs.extend(((sents != y) * mask_y * 1.).sum(1) / mask_y.sum(1)) progress.disp(errs, ' ERR') return preds, numpy.mean(errs)
return np.sum(np.square(u - v)) / u.shape[0] if __name__ == '__main__': # config config = Config() # model with config model = Model(config) error = [] # to keep it simple, we define the number of beatches we wanna do for batch_indx in range(config.batch_number): # load random snippets from one file x, t = load_batch(config) # # for each of them generate an initial state initial_state = np.zeros((config.batch_number, 6)) with tf.GradientTape() as tape: output = model(x) train_loss = config.loss_function(t, output) train_error = mse(t, output) gradients = tape.gradient(train_loss, model.trainable_variables) if batch_indx % PRINT_STEP == 0: print("* Batch: [{:4d}/{}] === Loss: {:04.5f} === Error: {:04.5f}". format(batch_indx, config.batch_number, train_loss, train_error)) config.optimizer.apply_gradients( zip(gradients, model.trainable_variables))
def train(): tf.global_variables_initializer().run() could_load, checkpoint_counter = load() if could_load: start_epoch = (int)(checkpoint_counter / num_batches) start_batch_id = checkpoint_counter - start_epoch * num_batches counter = checkpoint_counter print("Checkpoint Load Successed") else: start_epoch = 0 start_batch_id = 0 counter = 1 print("train from scratch...") train_iter = [] train_loss = [] IOU = 0.5 F1 = 0.7 # utils.count_params() print("Total train image:{}".format(len(train_img))) print("Total validate image:{}".format(len(valid_img))) print("Total epoch:{}".format(args.num_epochs)) print("Batch size:{}".format(args.batch_size)) print("Learning rate:{}".format(args.learning_rate)) print("Checkpoint step:{}".format(args.checkpoint_step)) print("Data Argument:") print("h_flip: {}".format(args.h_flip)) print("v_flip: {}".format(args.v_flip)) print("rotate: {}".format(args.rotation)) print("clip size: {}".format(args.clip_size)) loss_tmp = [] for i in range(start_epoch, args.num_epochs): epoch_time = time.time() id_list = np.random.permutation(len(train_img)) # if (i > args.start_valid): # if (i - args.start_valid) % args.valid_step == 0: # val_iou = validation() # print("last iou valu:{}".format(IOU)) # print("new_iou value:{}".format(val_iou)) # if val_iou > IOU: # print("Save the checkpoint...") # saver.save(sess, './mapmodfy/checkpoint/model.ckpt', global_step=counter, write_meta_graph=True) # IOU = val_iou for j in range(start_batch_id, num_batches): img_d = [] lab_d = [] for ind in range(args.batch_size): id = id_list[j * args.batch_size + ind] img_d.append(train_img[id]) lab_d.append(train_label[id]) x_batch, y_batch = load_batch(img_d, lab_d) # print(x_batch) feed_dict = {img: x_batch, label: y_batch, is_training: True} _, loss, pred1 = sess.run( [train_step, sigmoid_cross_entropy_loss, pred], feed_dict=feed_dict) loss_tmp.append(loss) # print(loss) if (counter % 200 == 0): tmp = np.median(loss_tmp) train_iter.append(counter) train_loss.append(tmp) print('Epoch', i, '|Iter', counter, '|Loss', tmp) loss_tmp.clear() counter += 1 start_batch_id = 0 print('Time:', time.time() - epoch_time) # if(i > args.start_valid and (i+1)%5==0): # saver.save(sess, './checkpoint/model.ckpt', global_step=counter) if (i > args.start_valid): if (i - args.start_valid) % args.valid_step == 0: val_iou = validation() print("last iou valu:{}".format(IOU)) print("new_iou value:{}".format(val_iou)) if val_iou > IOU: print("Save the checkpoint...") saver.save(sess, './checkpoint/model.ckpt', global_step=counter, write_meta_graph=True) IOU = val_iou saver.save(sess, './checkpoint/model.ckpt', global_step=counter)
def fit(self, train, val, max_epochs=None, batch_size=512, class_weights=None, early_stopping=0, save_model=False): outpath = os.path.join(self.outpath, 'cnn_model/') if os.path.isdir(outpath): shutil.rmtree(outpath) os.makedirs(outpath, exist_ok=True) os.mkdir(os.path.join(outpath, 'model_weights/')) if max_epochs is None: max_epochs = 300 self.epochs = 0 train_n = len(train) val_n = len(val) decreasing = 0 tloss = [] tacc = [] vloss = [] vacc = [] stop = False # iterate through every epoch while not stop: sys.stderr.flush() print('Epoch: %i' % self.epochs) print('Training:') shuffle(train) for i in tqdm(range(0, train_n, batch_size)): batch_paths = train[i:i + batch_size] if i + batch_size >= train_n: batch_paths = train[i:] x, y = data.load_batch(batch_paths, len(batch_paths)) metrics = self.train_on_batch(x, y, class_weight=class_weights, return_dict=True) tloss.append(metrics['loss']) tacc.append(metrics['acc']) del x, y # print metrics avg_loss = sum(tloss) / len(tloss) avg_acc = sum(tacc) / len(tacc) self.train_loss.append(avg_loss) self.train_acc.append(avg_acc) print('\nTrain Loss: %f Train Accuracy: %f' % (avg_loss, avg_acc)) print('Validation:') # test on val for each epoch for i in tqdm(range(0, val_n, batch_size)): batch_paths = val[i:i + batch_size] if i + batch_size >= val_n: batch_paths = val[i:] x, y = data.load_batch(batch_paths, len(batch_paths)) metrics = self.model.test_on_batch(x, y, return_dict=True) vloss.append(metrics['loss']) vacc.append(metrics['acc']) del x, y # print metrics avg_loss = sum(vloss) / len(vloss) avg_acc = sum(vacc) / len(vacc) self.val_loss.append(avg_loss) self.val_acc.append(avg_acc) print('Validation Loss: %f Validation Accuracy: %f' % (avg_loss, avg_acc)) # check stopping criteria if self.epochs >= max_epochs - 1: stop = True if early_stopping > 0: if len(self.val_loss) >= early_stopping: if self.val_loss[-1] > self.val_loss[-2]: if decreasing >= early_stopping: stop = True else: decreasing += 1 else: decreasing = 0 # save if model improves if self.epochs >= 1 and self.val_loss[-1] < self.val_loss[-2]: self.model.save_weights( os.path.join(outpath, 'model_weights/', 'weights_' + str(self.epochs))) if save_model: self.model.save(os.path.join(outpath, 'model')) self.epochs += 1
def train(): tf.global_variables_initializer().run() could_load, checkpoint_counter = load() if could_load: start_epoch = (int)(checkpoint_counter / num_batches) start_batch_id = checkpoint_counter - start_epoch * num_batches counter = checkpoint_counter print("Checkpoint Load Successed") else: start_epoch = 0 start_batch_id = 0 counter = 1 print("train from scratch...") train_iter=[] train_loss=[] utils.count_params() print("Total image:{}".format(len(train_img))) print("Total epoch:{}".format(args.num_epochs)) print("Batch size:{}".format(args.batch_size)) print("Learning rate:{}".format(args.learning_rate)) print("Checkpoint step:{}".format(args.checkpoint_step)) print("Data Argument:") print("h_flip: {}".format(args.h_flip)) print("v_flip: {}".format(args.v_flip)) print("rotate: {}".format(args.rotation)) print("clip size: {}".format(args.clip_size)) for i in range(start_epoch,args.num_epochs): id_list = np.random.permutation(len(train_img)) epoch_time=time.time() for j in range(start_batch_id,num_batches): img_d=[] lab_d=[] for ind in range(args.batch_size): id = id_list[j * args.batch_size + ind] img_d.append(train_img[id]) lab_d.append(train_label[id]) x_batch, y_batch = load_batch(img_d,lab_d,args) feed_dict = {img: x_batch, label: y_batch } loss_tmp = [] _, loss, pred1 = sess.run([train_step, sigmoid_cross_entropy_loss, pred], feed_dict=feed_dict) loss_tmp.append(loss) if (counter % 100 == 0): tmp = np.mean(loss_tmp) train_iter.append(counter) train_loss.append(tmp) print('Epoch', i, '|Iter', counter, '|Loss', tmp) counter += 1 start_batch_id=0 print('Time:', time.time() - epoch_time) #if((i+1)%10==0 ):#lr dst from 10 every 10 epoches by 0.1 #learning_rate = 0.1 * learning_rate #last_checkpoint_name = "checkpoint/latest_model_epoch_" + "_pspet.ckpt" # print("Saving latest checkpoint") # saver.save(sess, last_checkpoint_name) if((i+1)%args.checkpoint_step==0):#save 20,30,40,50 checkpoint args.learning_rate=0.1*args.learning_rate print(args.learning_rate) saver.save(sess,'./checkpoint/model.ckpt',global_step=counter,write_meta_graph=True) """ host = host_subplot(111) plt.subplots_adjust(right=0.8) p1, = host.plot(train_iter, train_loss, label="training loss") host.legend(loc=5) host.axis["left"].label.set_color(p1.get_color()) host.set_xlim([0, counter]) plt.draw() plt.show() """ fig1, ax1 = plt.subplots(figsize=(11, 8)) ax1.plot(train_iter, train_loss) ax1.set_title("Training loss vs Iter") ax1.set_xlabel("Iter") ax1.set_ylabel("Training loss") plt.savefig('Training loss_vs_Iter.png') plt.clf() remain_time=(args.num_epochs - 1 - i) * (time.time() - epoch_time) m, s = divmod(remain_time, 60) h, m = divmod(m, 60) print("Remaining training time = %d hours %d minutes %d seconds\n" % (h, m, s))
def train(verbose): X_train, Y_train, X_dev, Y_dev = data.load_batch('mfcc') #load training and validation batches net = network.RNN_network() total_loss = net.loss() training_losses = [] validation_losses = [] #adaptive learning rate global_step = tf.Variable(0, trainable=False) # adaptive_learning_rate = tf.train.exponential_decay(learning_rate_init, global_step, 1400, learning_decay, staircase=True) optimizer_a = tf.train.AdamOptimizer(learning_rate) # optimizer = optimizer_a.minimize(total_loss, global_step=global_step) #add gradient clipping gradients, variables = zip(*optimizer_a.compute_gradients(total_loss)) gradients, _ = tf.clip_by_global_norm(gradients, clip_norm=1.0) optimizer = optimizer_a.apply_gradients(zip(gradients, variables), global_step=global_step) run_options = tf.RunOptions(report_tensor_allocations_upon_oom = True) with tf.Session() as sess: t0 = time.time() sess.run(tf.global_variables_initializer(), options=run_options) net.load_state(sess, CKPT_PATH) n_train_batch = len(X_train) print("Number of training batch:", n_train_batch) idx_train = list(range(n_train_batch)) n_dev_batch = len(X_dev) print("Number of validation batch:", n_dev_batch) idx_dev = list(range(n_dev_batch)) for epoch_idx in range(num_epochs): np.random.shuffle(idx_train) loss_epoch = 0 #training mode for i in range(n_train_batch): _total_loss, _train_step, net_output = sess.run( [total_loss, optimizer, net()], feed_dict={ net.batchX_placeholder:X_train[idx_train[i]], net.batchY_placeholder:Y_train[idx_train[i]] }) loss_epoch += _total_loss #check for NaNs in network output if (np.any(np.isnan(net_output))): print("\nepoch: " + repr(epoch_idx) + " Nan output!") if verbose == 1: print("batch_loss:", _total_loss) t1 = time.time() print("\nepoch: " + repr(epoch_idx) + " || loss_epoch: " + repr(loss_epoch) + " || ", end=' ') timer(t0, t1) training_losses.append(loss_epoch) if epoch_idx % 10 == 0: tf.train.Saver().save(sess, CKPT_PATH, global_step=epoch_idx) if epoch_idx % 1 == 0: #validation mode dev_loss_epoch = 0 for j in range(n_dev_batch): _total_dev_loss = sess.run( [total_loss], feed_dict={ net.batchX_placeholder: X_dev[idx_dev[j]], net.batchY_placeholder: Y_dev[idx_dev[j]] }) dev_loss_epoch += _total_dev_loss[0] #dev loss across all validation batches print('********epoch: '+ repr(epoch_idx) + " || validation loss: " + repr(dev_loss_epoch) + " || ", end=' ') validation_losses.append(dev_loss_epoch) tf.train.Saver().save(sess, SAVE_PATH + "/" + repr(time.time()) + "/" + "save.ckpt") print("finished.") training_losses = np.array(training_losses) np.save(SAVE_PATH + "training_losses.npy", training_losses) validation__losses = np.array(validation_losses) np.save(SAVE_PATH + "validation_losses.npy", validation__losses)
def main(_): #Create the log directory here. Must be done here otherwise import will activate this unneededly. if not os.path.exists(FLAGS.log_dir): os.mkdir(FLAGS.log_dir) labels_dict = load_labels_into_dict(FLAGS.labels_file) #======================= TRAINING PROCESS ========================= #Now we start to construct the graph and build our model with tf.Graph().as_default() as graph: tf.logging.set_verbosity(tf.logging.INFO) #Set the verbosity to INFO level ##################### # Data Loading # dataset = get_split('train', FLAGS.dataset_dir, FLAGS.num_classes, FLAGS.labels_file, file_pattern=FLAGS.file_pattern, file_pattern_for_counting=FLAGS.file_pattern_for_counting) images, _, labels = load_batch(dataset, FLAGS.preprocessing, FLAGS.batch_size, FLAGS.image_size) num_batches_per_epoch = int(dataset.num_samples / FLAGS.batch_size) num_steps_per_epoch = num_batches_per_epoch decay_steps = int(FLAGS.epochs_before_decay * num_steps_per_epoch) ###################### # Select the network # ###################### network_fn = nets_factory.get_network_fn(FLAGS.model_name, num_classes=(dataset.num_classes), weight_decay=FLAGS.weight_decay, is_training=True) logits, end_points = network_fn(images) final_tensor = tf.identity(end_points['Predictions'], name="final_result") tf.summary.histogram('activations', final_tensor) #Perform one-hot-encoding of the labels (Try one-hot-encoding within the load_batch function!) ? one_hot_labels = slim.one_hot_encoding(labels, dataset.num_classes) with tf.name_scope('cross_entropy_loss'): #Performs the equivalent to tf.nn.sparse__entropy_with_logits but enhanced with checks loss = tf.losses.softmax_cross_entropy(onehot_labels=one_hot_labels, logits = logits) #Optionally calculate a weighted loss if FLAGS.class_weight: loss = tf.losses.compute_weighted_loss(loss, weights=FLAGS.class_weight) total_loss = tf.losses.get_total_loss() #obtain the regularization losses as well #Create the global step for monitoring the learning_rate and training. global_step = tf.train.get_or_create_global_step() #Define decaying learning rate lr = tf.train.exponential_decay(learning_rate = FLAGS.learning_rate, global_step = global_step, decay_steps = decay_steps, decay_rate = FLAGS.learning_rate_decay, staircase = True) #TODO: add options to decide optimizer optimizer = tf.train.AdamOptimizer(learning_rate = lr) variables_to_train = _get_variables_to_train() train_op = slim.learning.create_train_op(total_loss, optimizer, variables_to_train=variables_to_train) accuracy, prediction = _add_evaluation_step(final_tensor, one_hot_labels) tf.summary.scalar('losses/Total_Loss', total_loss) tf.summary.scalar('learning_rate', lr) my_summary_op = tf.summary.merge_all() sv = tf.train.Supervisor(logdir = FLAGS.log_dir, summary_op = None, init_fn=_get_init_fn()) #Run the managed session with sv.managed_session() as sess: for step in range(num_steps_per_epoch * FLAGS.num_epochs): if step % num_batches_per_epoch == 0: logging.info('Epoch %s/%s', step/num_batches_per_epoch + 1, FLAGS.num_epochs) learning_rate_value, accuracy_value = sess.run([lr, accuracy]) logging.info('Current Learning Rate: %s', learning_rate_value) logging.info('Current Batch Accuracy: %s', accuracy_value) if FLAGS.save_every_epoch: sv.saver.save(sess, sv.save_path, global_step = sv.global_step) if step % FLAGS.log_every_n_steps == 0: loss, _ = _train_step(sess, train_op, sv.global_step, log=True) summaries = sess.run(my_summary_op) sv.summary_computed(sess, summaries) else: loss, _ = _train_step(sess, train_op, sv.global_step) #Log the final training loss and train accuracy logging.info('Final Loss: %s', loss) logging.info('Final Train Accuracy: %s', sess.run(accuracy)) logging.info('Finished training! Saving model to disk now {}'.format(sv.save_path)) sv.saver.save(sess, sv.save_path, global_step = sv.global_step)
img_name_vector, cap_vector, test_size=0.2, random_state=0) # feel free to change these parameters according to your system's configuration BATCH_SIZE = 64 BUFFER_SIZE = 1000 embedding_dim = 512 units = 512 vocab_size = len(tokenizer.word_index) + 1 num_steps = len(img_name_train) // BATCH_SIZE # shape of the vector extracted from InceptionV3 is (64, 2048) # these two variables represent that features_shape = 2048 attention_features_shape = 65 batch = load_batch(img_name_train, cap_train) encoder = CNN_Encoder_Adaptive(embedding_dim) decoder = RNN_Decoder_Adaptive(embedding_dim, units, vocab_size) checkpoint_path = "./checkpoints/train_adaptive" ckpt = tf.train.Checkpoint(encoder=encoder, decoder=decoder) status = ckpt.restore(tf.train.latest_checkpoint(checkpoint_path)) # captions on the validation set for i in range(len(val_img_name_vector)): rid = np.random.randint(0, len(val_img_name_vector)) image = val_img_name_vector[rid] real_caption = ' '.join( [tokenizer.index_word[i] for i in val_cap_vector[rid] if i not in [0]]) result, attention_plot = evaluate(
cap_vector, tokenizer, max_length = preprocess_caption(train_captions, 5000) # Create training and validation sets using 80-20 split img_name_train, img_name_val, cap_train, cap_val = train_test_split( img_name_vector, cap_vector, test_size=0.1, random_state=0) vocab_size = len(tokenizer.word_index) + 1 print('vocab_size:' + str(vocab_size)) num_steps = len(img_name_train) // BATCH_SIZE val_num_steps = len(img_name_val) // BATCH_SIZE # shape of the vector extracted from InceptionV3 is (64, 2048) # these two variables represent that features_shape = 2048 attention_features_shape = 64 dataset = load_batch(img_name_train, cap_train, BATCH_SIZE, BUFFER_SIZE) val_dataset = load_batch(img_name_val, cap_val, BATCH_SIZE, BUFFER_SIZE) encoder = CNN_Encoder(embedding_dim) decoder = RNN_Decoder(embedding_dim, units, vocab_size) optimizer = tf.train.AdamOptimizer() checkpoint_path = "./checkpoints/train" ckpt = tf.train.Checkpoint(encoder=encoder, decoder=decoder) ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=50) start_epoch = 0 if ckpt_manager.latest_checkpoint:
def predict(f_enc, f_dec, samples, batches, mat, beam_size, max_len, header): ''' Sample words and compute the prediction error ''' preds = [] errs = [] progress = ProgressBar(numpy.sum([len(batch) for batch in batches]), 20, header) for batch in batches: x, mask_x, y, mask_y = load_batch(samples, batch, mat) [init_h] = f_enc(x, mask_x) n_steps = mask_x.sum(0) n_samples = x.shape[1] prev_sents = numpy.zeros((beam_size, n_samples, max_len), 'int32') # First step - No embedded word is fed into the decoder prev_words = numpy.asarray([-1] * n_samples, 'int32') prev_sents[:, :, 0], prev_log_prob, prev_h = f_dec(prev_words, init_h) prev_h = numpy.tile(prev_h, (beam_size, 1, 1)) prev_n_ends = n_steps - (prev_sents[:, :, 0] == 0) for i in range(1, max_len - 1): hypo_sents = [[]] * n_samples hypo_log_prob = [[]] * n_samples hypo_h = [[]] * n_samples hypo_n_ends = [[]] * n_samples has_hypos = numpy.asarray([False] * n_samples) for j in range(beam_size): if not prev_n_ends[j].any(): continue next_words, next_log_prob, next_h = f_dec(prev_sents[j, :, i - 1], prev_h[j]) for k in range(n_samples): if prev_n_ends[j, k] > 0: has_hypos[k] = True next_sents = numpy.tile(prev_sents[j, k], (beam_size, 1)) next_sents[:, i] = next_words[:, k] hypo_sents[k].extend(next_sents) hypo_log_prob[k].extend(next_log_prob[:, k] + prev_log_prob[j, k]) hypo_h[k].extend([next_h[k]] * beam_size) hypo_n_ends[k].extend(prev_n_ends[j, k] - (next_words[:, k] == 0)) else: hypo_sents[k].append(prev_sents[j, k].copy()) hypo_log_prob[k].append(prev_log_prob[j, k]) hypo_h[k].append(prev_h[j, k].copy()) hypo_n_ends[k].append(0) if not has_hypos.any(): break for j in range(n_samples): if not has_hypos[j]: continue indices = numpy.argsort(hypo_log_prob[j])[: -beam_size - 1: -1] for k in range(beam_size): prev_sents[k, j] = hypo_sents[j][indices[k]] prev_log_prob[k, j] = hypo_log_prob[j][indices[k]] prev_h[k, j] = hypo_h[j][indices[k]] prev_n_ends[k, j] = hypo_n_ends[j][indices[k]] sents = prev_sents[prev_log_prob.argmax(0), numpy.arange(n_samples)] for i in range(n_samples): idx = 0 while idx < max_len and n_steps[i] > 0: if sents[i, idx] == 0: n_steps[i] -= 1 idx += 1 preds.append(sents[i, : idx].tolist()) y = numpy.concatenate([y, numpy.zeros((max_len - len(y), n_samples), 'int32')]).T mask_y = numpy.concatenate([mask_y, numpy.zeros((max_len - len(mask_y), n_samples), 'int32')]).T errs.extend(((sents != y) * mask_y * 1.).sum(1) / mask_y.sum(1)) progress.disp(errs, ' ERR') return preds, numpy.mean(errs)