def __init__(self, args, data, tuning): self.FLAGS = args self.data = data self.tuning = tuning self.embedding_init = embed(self.data) self.model = LSTMClassifier(self.FLAGS, self.embedding_init) logits = self.model.inference() self.train_loss = self.model.loss(logits) self.train_op = self.model.training(self.train_loss[0]) pred = self.model.inference(forward_only=True) self.test_loss = self.model.loss(pred, forward_only=True) # Visualizing loss function and accuracy during training over epochs self.plotter = LossAccPlotter(title="Training plots", save_to_filepath="../img/lstm_plot.png", show_regressions=False, show_averages=False, show_loss_plot=True, show_acc_plot=True, show_plot_window=True, x_label="Epoch") self.init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) print("Network initialized..")
def plot_losses(data, suffix, fi_len, batch_per_epoch, **kwargs): # plot_data = {'X': [], 'Y': [], 'legend': []} other_loss = OrderedDict() # plot settings save_to_filepath = os.path.join("{}_log".format(data), "{}_plot_losses.png".format(suffix)) plotter = LossAccPlotter(title="{} loss over time".format(suffix), save_to_filepath=save_to_filepath, show_regressions=False, show_averages=False, show_other_loss=True, show_log_loss=True, show_loss_plot=True, show_err_plot=True, show_plot_window=False, epo_max=1000, x_label="Epoch") ## load loss data log_path = kwargs["log_path"] log_file2 = open(log_path, "r") st_time = time.time() i = 0 for li in log_file2: li_or = li.split(" | ") if len(li_or) == 1: continue iter = li_or[0].split("\t")[0][1:] loss_train = str2flo(li_or[0].split(":")[1].split(",")[0]) err_train = str2flo(li_or[0].split(",")[1]) loss_val = str2flo(li_or[1].split(":")[1].split(",")[0]) err_val = str2flo(li_or[1].split(",")[1]) for li2 in li_or: if "loss" not in li2: continue # pdb.set_trace() key = li2.split(": ")[0] value = str2flo(li2.split(": ")[1]) if key == 'vi loss': value *= 1e-2 other_loss[key] = value float_epoch = str2flo(iter) / batch_per_epoch plotter.add_values(float_epoch, loss_train=loss_train, loss_val=loss_val, err_train=err_train, err_val=err_val, redraw=False, other_loss=other_loss) i += 1 time_str = "{}\r".format( calculate_remaining(st_time, time.time(), i, fi_len)) # print(time_string, end = '\r') sys.stdout.write(time_str) sys.stdout.flush() sys.stdout.write("\n") sys.stdout.flush() log_file2.close() plotter.redraw() # save as image
def plot_loss_acc_history(fit_history, optimizer_name): plotter = LossAccPlotter( title=optimizer + ': Loss and Accuracy Performance', save_to_filepath='loss_acc_plots/' + optimizer + '.png', show_regressions=True, show_averages=False, show_loss_plot=True, show_acc_plot=True, show_plot_window=True, x_label="Epoch") num_epochs = len(fit_history['acc']) for epoch in range(num_epochs): acc_train = fit_history['acc'][epoch] loss_train = fit_history['loss'][epoch] acc_val = fit_history['val_acc'][epoch] loss_val = fit_history['val_loss'][epoch] plotter.add_values(epoch, loss_train=loss_train, acc_train=acc_train, loss_val=loss_val, acc_val=acc_val, redraw=False) plotter.redraw() plotter.block()
def plot_loss_err(data, suffix, fi_len, ema, batch_per_epoch, **kwargs): ## load loss data log_path = kwargs["log_path"] log_file2 = open(log_path, "r") st_time = time.time() # plot settings save_to_filepath = os.path.join("{}_log".format(data), "{}_plot_loss_err.png".format(suffix)) plotter = LossAccPlotter(title="{} loss over time".format(suffix), save_to_filepath=save_to_filepath, show_regressions=True, show_averages=True, show_loss_plot=True, show_err_plot=True, show_ema_plot=ema, show_plot_window=False, x_label="Epoch") i = 0 for li in log_file2: li_or = li.split(" | ") if len(li_or) == 1: continue iter = li_or[0].split("\t")[0][1:] loss_train = str2flo(li_or[0].split(":")[1].split(",")[0]) err_train = str2flo(li_or[0].split(",")[1]) loss_val = str2flo(li_or[1].split(":")[1].split(",")[0]) err_val = str2flo(li_or[1].split(",")[1]) ema_err_train = ema_err_val = None if ema: ema_err_train = li_or[3].split(":")[1].split(",")[0] ema_err_val = li_or[3].split(",")[1] if "None" not in ema_err_train: ema_err_train = str2flo(ema_err_train) ema_err_val = str2flo(ema_err_val) else: ema_err_train = ema_err_val = None float_epoch = str2flo(iter) / batch_per_epoch plotter.add_values(float_epoch, loss_train=loss_train, loss_val=loss_val, err_train=err_train, err_val=err_val, ema_err_train=ema_err_train, ema_err_val=ema_err_val, redraw=False) i += 1 time_str = "{}\r".format( calculate_remaining(st_time, time.time(), i, fi_len)) sys.stdout.write(time_str) sys.stdout.flush() sys.stdout.write("\n") log_file2.close() plotter.redraw() # save as image
def plot_image(title, filename, training, validation): plotter = LossAccPlotter(title, save_to_filepath=os.path.join( input_dir, filename), show_regressions=False, show_averages=False, show_loss_plot=True, show_acc_plot=True, show_plot_window=False, x_label="Iteration") # Store counter for next validation row to plot next_validation_row_index_to_plot = 0 for row in range(len(training)): training_record = training[row] training_iter = training_record[1] loss_train = training_record[2] # Plot Both Training and Validation Record if len(validation) > next_validation_row_index_to_plot and \ validation[next_validation_row_index_to_plot][1] == training_iter: val_record = validation[next_validation_row_index_to_plot] next_validation_row_index_to_plot = next_validation_row_index_to_plot + 1 loss_val = val_record[2] if len(val_record) == 3: plotter.add_values(training_iter, loss_train=loss_train, loss_val=loss_val, redraw=False) elif len(val_record) > 3: inst_acc_val = val_record[4] inst_acc_train = training_record[4] plotter.add_values(training_iter, loss_train=loss_train, loss_val=loss_val, acc_train=inst_acc_train, acc_val=inst_acc_val, redraw=False) else: # Plot Training Record only if len(training_record) == 3: plotter.add_values(training_iter, loss_train=loss_train, redraw=False) elif len(training_record) > 3: # Valid value inst_acc_train = training_record[4] plotter.add_values(training_iter, loss_train=loss_train, acc_train=inst_acc_train, redraw=False) plotter.redraw() plotter.block()
def plotResults(title, a_train, c_train, a_val, c_val): plotter = LossAccPlotter(title=title, show_averages=False, save_to_filepath=plot_loc + "lossAcc_{}.png".format(title), show_plot_window=show_not_save) for e in range(len(a_train)): plotter.add_values(e, loss_train=c_train[e], acc_train=a_train[e], loss_val=c_val[e], acc_val=a_val[e], redraw=False) plotter.redraw() plotter.block()
def show_chart(loss_train, loss_val, acc_train, acc_val, lap=None, title=None): """Shows a plot using the LossAccPlotter and all provided values. Args: loss_train: y-values of the loss function of the training dataset. loss_val: y-values of the loss function of the validation dataset. acc_train: y-values of the accuracy of the training dataset. acc_val: y-values of the accuracy of the validation dataset. lap: A LossAccPlotter-Instance or None. If None then a new LossAccPlotter will be instantiated. (Default is None.) title: The title to use for the plot, i.e. LossAccPlotter.title . """ lap = LossAccPlotter() if lap is None else lap # set the plot title, which will be shown at the very top of the plot if title is not None: lap.title = title # add loss train line/values for idx in range(loss_train.shape[0]): lt_val = loss_train[idx] if loss_train[idx] != -1.0 else None lap.add_values(idx, loss_train=lt_val, redraw=False) # add loss validation line/values for idx in range(loss_val.shape[0]): lv_val = loss_val[idx] if loss_val[idx] != -1.0 else None lap.add_values(idx, loss_val=lv_val, redraw=False) # add accuracy training line/values for idx in range(acc_train.shape[0]): at_val = acc_train[idx] if acc_train[idx] != -1.0 else None lap.add_values(idx, acc_train=at_val, redraw=False) # add accuracy validation line/values for idx in range(acc_val.shape[0]): av_val = acc_val[idx] if acc_val[idx] != -1.0 else None lap.add_values(idx, acc_val=av_val, redraw=False) # redraw once after adding all values, because that's significantly # faster than redrawing many times lap.redraw() # block at the end so that the plot does not close immediatly. print("Close the chart to continue.") lap.block()
def train(instrument, batch_size, latent_dim, epochs, mode=None): data_loader = load_data(instrument, batch_size) generator = Generator(latent_dim) generator.apply(weights_init) discriminator = Discriminator() discriminator.apply(weights_init) # print("Generator's state_dict:") # for param_tensor in generator.state_dict(): # print(param_tensor, "\t", generator.state_dict()[param_tensor].size()) # print("Discriminator's state_dict:") # for param_tensor in discriminator.state_dict(): # print(param_tensor, "\t", discriminator.state_dict()[param_tensor].size()) g_optimizer = optim.Adam(generator.parameters(), lr=0.002, betas=(0.5, 0.999)) d_optimizer = optim.Adam(discriminator.parameters(), lr=0.002, betas=(0.5, 0.999)) loss = nn.BCELoss() if not os.path.exists("./plots/" + instrument): os.makedirs("./plots/" + instrument) plotter = LossAccPlotter(save_to_filepath="./loss/" + instrument + "/loss.png", show_regressions=False, show_acc_plot=False, show_averages=False, show_plot_window=True, x_label="Epoch") if not os.path.exists("./model/" + instrument): os.makedirs("./model/" + instrument) if not os.path.exists("./loss/" + instrument): os.makedirs("./loss/" + instrument) epoch = 0 d_loss_list = [] g_loss_list = [] while True: for num_batch, real_data in enumerate(data_loader): ############################ # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) ########################### for i in range(1): real_data_d = next(iter(data_loader)) size = real_data_d.size(0) real_data_t = torch.ones(size, 239, 4) for i in range(size): for j in range(239): real_data_t[i][j] = torch.log( real_data_d[i, j + 1, :] / real_data_d[i, j, 3]) y_real = Variable(torch.ones(size, 1, 1)) y_fake = Variable(torch.zeros(size, 1, 1)) real_data = Variable(real_data_t.float()) fake_data = Variable( torch.from_numpy( np.random.normal(0, 0.2, (size, latent_dim, 1))).float()) fake_gen = generator(fake_data).detach() prediction_real = discriminator(real_data) loss_real = loss(prediction_real, y_real) prediction_fake = discriminator(fake_gen) loss_fake = loss(prediction_fake, y_fake) d_loss = loss_real + loss_fake d_optimizer.zero_grad() d_loss.backward() d_optimizer.step() ############################ # (2) Update G network: maximize log(D(G(z))) ########################### size = real_data.size(0) y_real = Variable(torch.ones(size, 1, 1)) fake_data = Variable( torch.from_numpy( np.random.normal(0, 0.2, (size, latent_dim, 1))).float()) fake_gen = generator(fake_data) prediction = discriminator(fake_gen) g_loss = loss(prediction, y_real) g_optimizer.zero_grad() g_loss.backward() g_optimizer.step() if num_batch % 20 == 0: tmp = torch.ones(size, 239, 4) data = real_data if num_batch == 0 else fake_gen for batch in range(size): for t in range(238, 0, -1): data[batch, t, 0] = torch.sum( data[batch, 0:t, 3]) + data[batch, t, 0] data[batch, t, 1] = torch.sum( data[batch, 0:t, 3]) + data[batch, t, 1] data[batch, t, 2] = torch.sum( data[batch, 0:t, 3]) + data[batch, t, 2] data[batch, t, 3] = torch.sum(data[batch, 0:t + 1, 3]) data = torch.exp(data) tmp = tmp * data print("epoch: %d, num_batch: %d, d-loss: %.4f, g-loss: %.4f" % (epoch, num_batch, d_loss.data.numpy(), g_loss.data.numpy())) visualize(instrument, tmp, epoch, num_batch) plotter.add_values(epoch, loss_train=g_loss.item(), loss_val=d_loss.item()) d_loss_list.append(d_loss.item()) g_loss_list.append(g_loss.item()) if epoch % 10 == 0: torch.save( generator, "./model/" + instrument + "/generator_epoch_" + str(epoch) + ".model") torch.save( discriminator, "./model/" + instrument + "/discriminator_epoch_" + str(epoch) + ".model") d_loss_np = np.array(d_loss_list) np.save( "./loss/" + instrument + "/d_loss_epoch_" + str(epoch) + ".npy", d_loss_np) g_loss_np = np.array(g_loss_list) np.save( "./loss/" + instrument + "/g_loss_epoch_" + str(epoch) + ".npy", g_loss_np) epoch += 1 if mode == "test" and epoch == epochs: break
"500" : 1e-4, "10000": 1e-5, "20000": 1e-6 } lr_decay_power = 4 momentum = 0.9 weight_decay = 0.005 n_classes = 3 n_boxes = 5 start = time.time() plotter = LossAccPlotter(title="YOLOv2 loss", save_to_filepath="plot.png", show_regressions=True, show_averages=True, show_loss_plot=True, show_acc_plot=False, show_plot_window=True, x_label="Batch") # load image generator #print("loading image generator...") #generator = ImageGenerator(item_path, background_path) print("loading ImageNet generator") imageNet_data = ImageNet_data("./XmlToTxt/water_bottle_img", "./XmlToTxt/water_bottle_bbox", "./XmlToTxt/images_list.txt", n_classes) # load model print("loading initial model...") yolov2 = YOLOv2(n_classes=n_classes, n_boxes=n_boxes) model = YOLOv2Predictor(yolov2) serializers.load_hdf5(initial_weight_file, model)
def main(): rand.seed() #env = gym.make('Asteroids-v0') env = gym.make('Breakout-v0') num_actions = env.action_space.n #print(num_actions) plotter = LossAccPlotter(title="mem256_perGame",show_acc_plot=True,save_to_filepath="/mem256_perGame.png",show_loss_plot=False) plotter2 = LossAccPlotter(title="mem256_100",show_acc_plot=True,save_to_filepath="/mem256_100.png",show_loss_plot=False) #plotter.save_plot("./mem256.png") observation = env.reset() #observation = downsample(observation) #reward = 0 action = 0 total_reward = 0 total_reward2 = 0 env.render() prev_obs = [] curr_obs = [] D = [] step = 0 rate = 1 sess, output_net, x, cost, trainer, mask, reward, nextQ = initialize() #load(sess) startPrinting = False for i in range(5): observation, rw, done, info = env.step(action) # pass in 0 for action observation = convert_to_small_and_grayscale(observation) prev_obs = deepcopy(curr_obs) curr_obs = obsUpdate(curr_obs,observation) #e = [rw, action, deepcopy(prev_obs), deepcopy(curr_obs)] #D.append(e) action = 1 #print(i) print("Entering mini-loop") for _ in range(10): step +=1 #print(step) if done: observation = env.reset() if (len(D) > 256): D.pop() if step % 1000 == 0: rate = rate / 2 if (rate < 0.05): rate=0.05 #if step % 1000 == 0: #save(sess) action = magic(curr_obs, sess, output_net, x,step,rate, False) #change this to just take in curr_obs, sess, and False #action = env.action_space.sample() env.render() observation, rw, done, info = env.step(action) # take a random action #print(action, rw, step) observation = convert_to_small_and_grayscale(observation) e = [rw, action, deepcopy(prev_obs), deepcopy(curr_obs)] D.append(e) prev_obs = deepcopy(curr_obs) curr_obs = obsUpdate(curr_obs,observation) print("Entering full loop") while step < 10001: step +=1 #print(step) if done: print("saving to plot....") plot_reward = total_reward plotter.add_values(step, acc_train=plot_reward) total_reward = 0 observation = env.reset() if (len(D) > 256): D.pop() if step % 100 == 0: print(step,"steps have passed") save(sess, step) rate = rate / 2 startPrinting = True if (rate < 0.05): rate=0.05 print("saving to plot2....") plot_reward = total_reward2/100 plotter2.add_values(step, acc_train=plot_reward) total_reward2 = 0 #print(step,"steps have passed") if step % 500 == 0: plotter.save_plot("./mem256_perGame.png") plotter2.save_plot("./mem256_100.png") action = magic(curr_obs, sess, output_net, x,step,rate, startPrinting) #change this to just take in curr_obs, sess, and False #action = env.action_space.sample() env.render() observation, rw, done, info = env.step(action) # take a random action #print(action, rw, step) observation = convert_to_small_and_grayscale(observation) e = [rw, action, deepcopy(prev_obs), deepcopy(curr_obs)] D.insert(0,e) prev_obs = deepcopy(curr_obs) curr_obs = obsUpdate(curr_obs,observation) update_q_function(D, sess, output_net, x, cost, trainer, mask, reward, nextQ) total_reward = total_reward + rw total_reward2 = total_reward2 + rw plotter.block() plotter2.block()
def main(): """Run various checks on the LossAccPlotter. They all follow the same pattern: Generate some random data (lines) to display. Then display them (using various settings). """ print("") print("------------------") print("1 datapoint") print("------------------") # generate example values for: loss train, loss validation, accuracy train # and accuracy validation (loss_train, loss_val, acc_train, acc_val) = create_values(1) # generate a plot showing the example values show_chart(loss_train, loss_val, acc_train, acc_val, title="A single datapoint") print("") print("------------------") print("150 datapoints") print("Saved to file 'plot.png'") print("------------------") (loss_train, loss_val, acc_train, acc_val) = create_values(150) show_chart(loss_train, loss_val, acc_train, acc_val, lap=LossAccPlotter(save_to_filepath="plot.png"), title="150 datapoints, saved to file 'plot.png'") print("") print("------------------") print("150 datapoints") print("No accuracy chart") print("------------------") (loss_train, loss_val, _, _) = create_values(150) show_chart(loss_train, loss_val, np.array([]), np.array([]), lap=LossAccPlotter(show_acc_plot=False), title="150 datapoints, no accuracy chart") print("") print("------------------") print("150 datapoints") print("No loss chart") print("------------------") (_, _, acc_train, acc_val) = create_values(150) show_chart(np.array([]), np.array([]), acc_train, acc_val, lap=LossAccPlotter(show_loss_plot=False), title="150 datapoints, no loss chart") print("") print("------------------") print("150 datapoints") print("No accuracy chart") print("------------------") (loss_train, loss_val, _, _) = create_values(150) show_chart(loss_train, loss_val, np.array([]), np.array([]), lap=LossAccPlotter(show_acc_plot=False), title="150 datapoints, no accuracy chart") print("") print("------------------") print("150 datapoints") print("Only validation values (no training lines)") print("------------------") (_, loss_val, _, acc_val) = create_values(150) show_chart(np.array([]), loss_val, np.array([]), acc_val, title="150 datapoints, only validation (no training)") print("") print("------------------") print("150 datapoints") print("No regressions") print("------------------") (loss_train, loss_val, acc_train, acc_val) = create_values(150) show_chart(loss_train, loss_val, acc_train, acc_val, lap=LossAccPlotter(show_regressions=False), title="150 datapoints, regressions deactivated") print("") print("------------------") print("150 datapoints") print("No averages") print("------------------") (loss_train, loss_val, acc_train, acc_val) = create_values(150) show_chart(loss_train, loss_val, acc_train, acc_val, lap=LossAccPlotter(show_averages=False), title="150 datapoints, averages deactivated") print("") print("------------------") print("150 datapoints") print("x-index 5 of loss_train should create a warning as its set to NaN") print("------------------") (loss_train, loss_val, acc_train, acc_val) = create_values(150) # this should create a warning when LossAccPlotter.add_values() gets called. loss_train[5] = float("nan") show_chart( loss_train, loss_val, acc_train, acc_val, title="150 datapoints, one having value NaN (loss train at x=5)") print("") print("------------------") print("1000 datapoints training") print("100 datapoints validation") print("------------------") nb_points_train = 1000 nb_points_val = 100 (loss_train, loss_val, acc_train, acc_val) = create_values(nb_points_train) # set 9 out of 10 values of the validation arrays to -1.0 (Which will be # interpreted as None in show_chart(). Numpy doesnt support None directly, # only NaN, which is already used before to check whether the Plotter # correctly creates a warning if any data point is NaN.) all_indices = np.arange(0, nb_points_train - 1, 1) keep_indices = np.arange(0, nb_points_train - 1, int(nb_points_train / nb_points_val)) set_to_none_indices = np.delete(all_indices, keep_indices) loss_val[set_to_none_indices] = -1.0 acc_val[set_to_none_indices] = -1.0 show_chart( loss_train, loss_val, acc_train, acc_val, title="1000 training datapoints, but only 100 validation datapoints") print("") print("------------------") print("5 datapoints") print("slowly added, one by one") print("------------------") (loss_train, loss_val, acc_train, acc_val) = create_values(5) lap = LossAccPlotter(title="5 datapoints, slowly added one by one") for idx in range(loss_train.shape[0]): lap.add_values(idx, loss_train=loss_train[idx], loss_val=loss_val[idx], acc_train=acc_train[idx], acc_val=acc_val[idx], redraw=True) sleep(1.0) print("Close the chart to continue.") lap.block()
if option.is_gd(): print("\tOptimization: Gradient Descent") elif option.is_sgd(): print("\tOptimization: Stochastic Gradient Descent") if option.is_linear(): print("\tActivation Function: Linear") elif option.is_sigmoid(): print("\tActivation Function: Sigmoid") if option.is_l2norm(): print("\tRegularization: L2Norm") elif option.is_dropout(): print("\tRegularization: Drop out") net = Network(training, test, option) if read_weights_from_file == "y": loaded = np.load('weights.npz') net.set_hid_weights(loaded['hid_weights']).set_out_weights( loaded['out_weights']).set_hid_bias( loaded['hid_bias']).set_out_bias(loaded['out_bias']) plotter = LossAccPlotter(show_acc_plot=False) for i in range(NUM_OF_ITER): training_loss = 0 if read_weights_from_file != "y": training_loss = net.train(i == NUM_OF_ITER - 1) test_loss = net.test(i == NUM_OF_ITER - 1) plotter.add_values(i, loss_train=training_loss, loss_val=test_loss) plotter.block()
train_acc.append(None) val_acc_top1.append(None) val_acc_top5.append(None) count_acc += 1 print("Start plotting") # print(len(loss)) # print(train_acc) # print(val_acc_top1) # print(val_acc_top5) plot_title = 'Loss and Accuracy Plotted Using ' + num_classes + ' Keywords (' + num_train_per_class + ' training videos each) and minibatch size of ' + batch_size plotter = LossAccPlotter(title=plot_title, save_to_filepath=num_classes + '_' + num_train_per_class + '_' + batch_size + '.png', show_regressions=False, show_averages=False, show_loss_plot=True, show_acc_plot=True, show_plot_window=True, x_label="Epoch") for epoch in range(len(loss)): plotter.add_values(epoch, loss_train=loss[epoch], acc_train=train_acc[epoch], acc_val=val_acc_top1[epoch]) plotter.block() print("Finished plotting")
def main(): """Create the example plots in the following way: 1. Generate example data (all plots use more or less the same data) 2. Generate plot 1: "standard" example with loss and accuracy 3. Generate plot 2: Same as 1, but only loss / no accuracy 4. Generate plot 3: Same as 1, but no validation lines (only training dataset) 5. Generate plot 4: Same as 1, but use only every 10th validation datapoint (i.e. resembles real world scenario where you rarely validate your machine learning method) """ nb_points = 500 loss_train = add_noise(np.linspace(0.9, 0.1, num=nb_points), 0.025) loss_val = add_noise(np.linspace(0.7, 0.3, num=nb_points), 0.045) acc_train = add_noise(np.linspace(0.52, 0.95, num=nb_points), 0.025) acc_val = add_noise(np.linspace(0.65, 0.75, num=nb_points), 0.045) # Normal example plot lap = LossAccPlotter(save_to_filepath="example_plot.png") show_chart(loss_train, loss_val, acc_train, acc_val, lap=lap, title="Example Plot with Loss and Accuracy") # Plot showing only the results of the loss function (accuracy off) lap = LossAccPlotter(show_acc_plot=False, save_to_filepath="example_plot_loss.png") show_chart(loss_train, loss_val, acc_train, acc_val, lap=lap, title="Example Plot, only Loss Function") # Plot showing only training dataset values (but for both loss and accuracy) lap = LossAccPlotter(save_to_filepath="example_plot_only_training.png") show_chart( loss_train, np.array([]), acc_train, np.array([]), lap=lap, title="Example Plot, only Training Dataset / no Validation Dataset") # Plot with a different update interval for training and validation dataset # (i.e. only one validation value for every 10 training values) # # Set 9 out of 10 validation values to -1, which will be transformed into # None in show_chart(). (same technique as in check_laplotter.py) nb_points_train = nb_points nb_points_val = int(nb_points * 0.1) all_indices = np.arange(0, nb_points_train - 1, 1) keep_indices = np.arange(0, nb_points_train - 1, int(nb_points_train / nb_points_val)) set_to_none_indices = np.delete(all_indices, keep_indices) loss_val[set_to_none_indices] = -1.0 acc_val[set_to_none_indices] = -1.0 lap = LossAccPlotter(show_acc_plot=False, save_to_filepath="example_plot_update_intervals.png") show_chart(loss_train, loss_val, acc_train, acc_val, lap=lap, title="Example Plot with different Update Intervals for Training " \ "and Validation Datasets")
def train(ngram, name, bar, drop_out, dataset, is_cuda=False, edges=False): plotter = LossAccPlotter(title="This is an example plot", save_to_filepath="/tmp/my_plot.png", show_regressions=True, show_averages=True, show_loss_plot=True, show_acc_plot=True, show_plot_window=False, x_label="Epoch") print('load data helper.') data_helper = DataHelper(mode='train') b_size = len(data_helper.label) print('*' * 100) print('train set total:', b_size) if os.path.exists(os.path.join('.', name + '.pkl')) and name != 'temp_model': print('load model from file') model = torch.load(os.path.join('.', name + '.pkl')) else: print('new model') if name == 'temp_model': name == 'temp_model' edges_weights, edges_mappings, count = cal_PMI() ## -----------------************************** import the datahelper class to get the vocab-5 doc*****************------------------------ ## class_num = len(data_helper.labels_str) is changed, consider just a score model = Model(class_num=data_helper.labels_str, hidden_size_node=200, vocab=data_helper.vocab, n_gram=ngram, drop_out=drop_out, edges_matrix=edges_mappings, edges_num=count, trainable_edges=edges, pmi=edges_weights, cuda=is_cuda) ### --------------------------------------- ### print(model) if is_cuda: print('cuda') model.cuda() loss_func = torch.nn.MSELoss() loss_mae = torch.nn.L1Loss(reduction='sum') optim = torch.optim.Adam(model.parameters(), weight_decay=1e-3) iter = 0 if bar: pbar = tqdm.tqdm(total=NUM_ITER_EVAL) best_acc = 0.0 last_best_epoch = 0 start_time = time.time() total_loss = 0.0 total_correct = 0 total = 0 accuracy = 0.0 num_epoch = 500 weight_decays = 1e-4 for content, label, epoch in data_helper.batch_iter( batch_size=32, num_epoch=num_epoch): improved = '' model.train() pred = model(content) pred_sq = torch.squeeze(pred, 1) l2_reg = 0.5 * weight_decays * ( model.seq_edge_w.weight.to('cpu').detach().numpy()**2).sum() loss = loss_func(pred_sq, label.float()) + l2_reg #-------------------------------------------# error = loss_mae(pred_sq.cpu().data, label.cpu()) accuracy += error total += len(pred) ##batch size = len(label) total_loss += (loss.item() * len(pred)) total_correct += loss.item() optim.zero_grad() loss.backward() optim.step() iter += 1 if bar: pbar.update() if iter % NUM_ITER_EVAL == 0: if bar: pbar.close() val_acc, val_loss = dev(model) if val_acc < best_acc: best_acc = val_acc last_best_epoch = epoch improved = '* ' torch.save(model, name + '.pkl') msg = 'Epoch: {0:>6} Iter: {1:>6}, Train Loss: {5:>7.2}, Train Error: {6:>7.2}' \ + 'Val Acc: {2:>7.2}, Time: {3}{4}, val error:{7:>7.2}' \ # + ' Time: {5} {6}' print( msg.format(epoch, iter, val_acc, get_time_dif(start_time), improved, total_correct / (NUM_ITER_EVAL), float(accuracy) / float(total), val_loss)) plotter.add_values(epoch, loss_train=total_correct / (NUM_ITER_EVAL), acc_train=float(accuracy) / float(total), loss_val=val_loss, acc_val=best_acc) total_loss = 0.0 total_correct = 0 accuracy = 0.0 total = 0 if bar: pbar = tqdm.tqdm(total=NUM_ITER_EVAL) plotter.block() return name
def train_model(num_tasks, models, dataloaders, dataset_sizes, criterion, optimizers, schedulers, epochs=15): since = time.time() num_tasks = num_tasks use_gpu = torch.cuda.is_available() final_outputs = [ ] #this is the variable to accumulate the outputs of all columns for each task middle_outputs = [ ] #this is the variable for keeping outputs of current task in each column #we iterate for each task for task_id in range(num_tasks): #everytime we do a new task, we empty the final outputs final_outputs[:] = [] # we now iterate for each previous column until the one of our task for i in range(0, task_id + 1): #we save the weights with best results best_model_wts = copy.deepcopy(models[task_id][i].state_dict()) model = models[task_id][i] optimizer = optimizers[i] scheduler = schedulers[i] # if it's not the column corresponding to the task, do not train if task_id != i: #this is the case for "previous" columns so we only need to pass data, not train num_epochs = 1 middle_outputs[:] = [] else: num_epochs = epochs dataloader = dataloaders[i] dataset_size = dataset_sizes[i] best_acc = 0.0 # let's add a plotter for loss and accuracy plotter = LossAccPlotter() for epoch in range(num_epochs): print('Epoch {}/{}'.format(epoch, num_epochs - 1)) print('-' * 10) # Each epoch has a training and validation phase for phase in ['train', 'val']: if phase == 'train': scheduler.step() model.train(True) # Set model to training mode else: model.train(False) # Set model to evaluate mode running_loss = 0.0 running_corrects = 0 # Iterate over data. #tqdm shows a progression bar for data in tqdm(dataloader[phase]): # get the inputs inputs, labels = data if inputs.type() != int and labels.type() != int: # wrap them in Variable if use_gpu: inputs = Variable(inputs.cuda()) labels = Variable(labels.cuda()) else: inputs, labels = Variable(inputs), Variable( labels) # zero the parameter gradients optimizer.zero_grad() # forward outputs, middle_outputs = model( inputs, final_outputs) #we save the outputs of this column in middle outputs and we have previous columns in final _, preds = torch.max(outputs.data, 1) loss = criterion(outputs, labels) # backward + optimize only if in training phase if phase == 'train': loss.backward() optimizer.step() # statistics running_loss += loss.data[0] * inputs.size(0) running_corrects += torch.sum(preds == labels.data) epoch_loss = running_loss / dataset_size[phase] epoch_acc = running_corrects / dataset_size[phase] print('{} Loss: {:.4f} Acc: {:.4f}'.format( phase, epoch_loss, epoch_acc)) if phase == 'train': plotter.add_values(epoch, loss_train=epoch_loss, acc_train=epoch_acc, redraw=False) else: plotter.add_values(epoch, loss_val=epoch_loss, acc_val=epoch_acc, redraw=False) # deep copy the model if phase == 'val' and epoch_acc > best_acc: best_acc = epoch_acc best_model_wts = copy.deepcopy(model.state_dict()) print() plotter.redraw() if task_id != i: final_outputs.append(middle_outputs) #we add the ouput of this column to final outputs plotter.save_plot('plots%d.%d.png' % (task_id, i)) print('Best val Acc: {:4f}'.format(best_acc)) # load best model weights models[task_id][i].load_state_dict(best_model_wts) time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) return models