def __init__(self, args, data, tuning):
self.FLAGS = args
self.data = data
self.tuning = tuning
self.embedding_init = embed(self.data)
self.model = LSTMClassifier(self.FLAGS, self.embedding_init)
logits = self.model.inference()
self.train_loss = self.model.loss(logits)
self.train_op = self.model.training(self.train_loss[0])
pred = self.model.inference(forward_only=True)
self.test_loss = self.model.loss(pred, forward_only=True)
# Visualizing loss function and accuracy during training over epochs
self.plotter = LossAccPlotter(title="Training plots",
save_to_filepath="../img/lstm_plot.png",
show_regressions=False,
show_averages=False,
show_loss_plot=True,
show_acc_plot=True,
show_plot_window=True,
x_label="Epoch")
self.init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
print("Network initialized..")
def plot_losses(data, suffix, fi_len, batch_per_epoch, **kwargs):
# plot_data = {'X': [], 'Y': [], 'legend': []}
other_loss = OrderedDict()
# plot settings
save_to_filepath = os.path.join("{}_log".format(data),
"{}_plot_losses.png".format(suffix))
plotter = LossAccPlotter(title="{} loss over time".format(suffix),
save_to_filepath=save_to_filepath,
show_regressions=False,
show_averages=False,
show_other_loss=True,
show_log_loss=True,
show_loss_plot=True,
show_err_plot=True,
show_plot_window=False,
epo_max=1000,
x_label="Epoch")
## load loss data
log_path = kwargs["log_path"]
log_file2 = open(log_path, "r")
st_time = time.time()
i = 0
for li in log_file2:
li_or = li.split(" | ")
if len(li_or) == 1:
continue
iter = li_or[0].split("\t")[0][1:]
loss_train = str2flo(li_or[0].split(":")[1].split(",")[0])
err_train = str2flo(li_or[0].split(",")[1])
loss_val = str2flo(li_or[1].split(":")[1].split(",")[0])
err_val = str2flo(li_or[1].split(",")[1])
for li2 in li_or:
if "loss" not in li2:
continue
# pdb.set_trace()
key = li2.split(": ")[0]
value = str2flo(li2.split(": ")[1])
if key == 'vi loss':
value *= 1e-2
other_loss[key] = value
float_epoch = str2flo(iter) / batch_per_epoch
plotter.add_values(float_epoch,
loss_train=loss_train,
loss_val=loss_val,
err_train=err_train,
err_val=err_val,
redraw=False,
other_loss=other_loss)
i += 1
time_str = "{}\r".format(
calculate_remaining(st_time, time.time(), i, fi_len))
# print(time_string, end = '\r')
sys.stdout.write(time_str)
sys.stdout.flush()
sys.stdout.write("\n")
sys.stdout.flush()
log_file2.close()
plotter.redraw() # save as image
def plot_loss_acc_history(fit_history, optimizer_name):
plotter = LossAccPlotter(
title=optimizer + ': Loss and Accuracy Performance',
save_to_filepath='loss_acc_plots/' + optimizer + '.png',
show_regressions=True,
show_averages=False,
show_loss_plot=True,
show_acc_plot=True,
show_plot_window=True,
x_label="Epoch")
num_epochs = len(fit_history['acc'])
for epoch in range(num_epochs):
acc_train = fit_history['acc'][epoch]
loss_train = fit_history['loss'][epoch]
acc_val = fit_history['val_acc'][epoch]
loss_val = fit_history['val_loss'][epoch]
plotter.add_values(epoch,
loss_train=loss_train,
acc_train=acc_train,
loss_val=loss_val,
acc_val=acc_val,
redraw=False)
plotter.redraw()
plotter.block()
def plot_loss_err(data, suffix, fi_len, ema, batch_per_epoch, **kwargs):
## load loss data
log_path = kwargs["log_path"]
log_file2 = open(log_path, "r")
st_time = time.time()
# plot settings
save_to_filepath = os.path.join("{}_log".format(data),
"{}_plot_loss_err.png".format(suffix))
plotter = LossAccPlotter(title="{} loss over time".format(suffix),
save_to_filepath=save_to_filepath,
show_regressions=True,
show_averages=True,
show_loss_plot=True,
show_err_plot=True,
show_ema_plot=ema,
show_plot_window=False,
x_label="Epoch")
i = 0
for li in log_file2:
li_or = li.split(" | ")
if len(li_or) == 1:
continue
iter = li_or[0].split("\t")[0][1:]
loss_train = str2flo(li_or[0].split(":")[1].split(",")[0])
err_train = str2flo(li_or[0].split(",")[1])
loss_val = str2flo(li_or[1].split(":")[1].split(",")[0])
err_val = str2flo(li_or[1].split(",")[1])
ema_err_train = ema_err_val = None
if ema:
ema_err_train = li_or[3].split(":")[1].split(",")[0]
ema_err_val = li_or[3].split(",")[1]
if "None" not in ema_err_train:
ema_err_train = str2flo(ema_err_train)
ema_err_val = str2flo(ema_err_val)
else:
ema_err_train = ema_err_val = None
float_epoch = str2flo(iter) / batch_per_epoch
plotter.add_values(float_epoch,
loss_train=loss_train,
loss_val=loss_val,
err_train=err_train,
err_val=err_val,
ema_err_train=ema_err_train,
ema_err_val=ema_err_val,
redraw=False)
i += 1
time_str = "{}\r".format(
calculate_remaining(st_time, time.time(), i, fi_len))
sys.stdout.write(time_str)
sys.stdout.flush()
sys.stdout.write("\n")
log_file2.close()
plotter.redraw() # save as image
def plot_image(title, filename, training, validation):
plotter = LossAccPlotter(title,
save_to_filepath=os.path.join(
input_dir, filename),
show_regressions=False,
show_averages=False,
show_loss_plot=True,
show_acc_plot=True,
show_plot_window=False,
x_label="Iteration")
# Store counter for next validation row to plot
next_validation_row_index_to_plot = 0
for row in range(len(training)):
training_record = training[row]
training_iter = training_record[1]
loss_train = training_record[2]
# Plot Both Training and Validation Record
if len(validation) > next_validation_row_index_to_plot and \
validation[next_validation_row_index_to_plot][1] == training_iter:
val_record = validation[next_validation_row_index_to_plot]
next_validation_row_index_to_plot = next_validation_row_index_to_plot + 1
loss_val = val_record[2]
if len(val_record) == 3:
plotter.add_values(training_iter,
loss_train=loss_train,
loss_val=loss_val,
redraw=False)
elif len(val_record) > 3:
inst_acc_val = val_record[4]
inst_acc_train = training_record[4]
plotter.add_values(training_iter,
loss_train=loss_train,
loss_val=loss_val,
acc_train=inst_acc_train,
acc_val=inst_acc_val,
redraw=False)
else: # Plot Training Record only
if len(training_record) == 3:
plotter.add_values(training_iter,
loss_train=loss_train,
redraw=False)
elif len(training_record) > 3: # Valid value
inst_acc_train = training_record[4]
plotter.add_values(training_iter,
loss_train=loss_train,
acc_train=inst_acc_train,
redraw=False)
plotter.redraw()
plotter.block()
def plotResults(title, a_train, c_train, a_val, c_val):
plotter = LossAccPlotter(title=title,
show_averages=False,
save_to_filepath=plot_loc +
"lossAcc_{}.png".format(title),
show_plot_window=show_not_save)
for e in range(len(a_train)):
plotter.add_values(e,
loss_train=c_train[e],
acc_train=a_train[e],
loss_val=c_val[e],
acc_val=a_val[e],
redraw=False)
plotter.redraw()
plotter.block()
def show_chart(loss_train, loss_val, acc_train, acc_val, lap=None, title=None):
"""Shows a plot using the LossAccPlotter and all provided values.
Args:
loss_train: y-values of the loss function of the training dataset.
loss_val: y-values of the loss function of the validation dataset.
acc_train: y-values of the accuracy of the training dataset.
acc_val: y-values of the accuracy of the validation dataset.
lap: A LossAccPlotter-Instance or None. If None then a new LossAccPlotter
will be instantiated. (Default is None.)
title: The title to use for the plot, i.e. LossAccPlotter.title .
"""
lap = LossAccPlotter() if lap is None else lap
# set the plot title, which will be shown at the very top of the plot
if title is not None:
lap.title = title
# add loss train line/values
for idx in range(loss_train.shape[0]):
lt_val = loss_train[idx] if loss_train[idx] != -1.0 else None
lap.add_values(idx, loss_train=lt_val, redraw=False)
# add loss validation line/values
for idx in range(loss_val.shape[0]):
lv_val = loss_val[idx] if loss_val[idx] != -1.0 else None
lap.add_values(idx, loss_val=lv_val, redraw=False)
# add accuracy training line/values
for idx in range(acc_train.shape[0]):
at_val = acc_train[idx] if acc_train[idx] != -1.0 else None
lap.add_values(idx, acc_train=at_val, redraw=False)
# add accuracy validation line/values
for idx in range(acc_val.shape[0]):
av_val = acc_val[idx] if acc_val[idx] != -1.0 else None
lap.add_values(idx, acc_val=av_val, redraw=False)
# redraw once after adding all values, because that's significantly
# faster than redrawing many times
lap.redraw()
# block at the end so that the plot does not close immediatly.
print("Close the chart to continue.")
lap.block()
def train(instrument, batch_size, latent_dim, epochs, mode=None):
data_loader = load_data(instrument, batch_size)
generator = Generator(latent_dim)
generator.apply(weights_init)
discriminator = Discriminator()
discriminator.apply(weights_init)
# print("Generator's state_dict:")
# for param_tensor in generator.state_dict():
# print(param_tensor, "\t", generator.state_dict()[param_tensor].size())
# print("Discriminator's state_dict:")
# for param_tensor in discriminator.state_dict():
# print(param_tensor, "\t", discriminator.state_dict()[param_tensor].size())
g_optimizer = optim.Adam(generator.parameters(),
lr=0.002,
betas=(0.5, 0.999))
d_optimizer = optim.Adam(discriminator.parameters(),
lr=0.002,
betas=(0.5, 0.999))
loss = nn.BCELoss()
if not os.path.exists("./plots/" + instrument):
os.makedirs("./plots/" + instrument)
plotter = LossAccPlotter(save_to_filepath="./loss/" + instrument +
"/loss.png",
show_regressions=False,
show_acc_plot=False,
show_averages=False,
show_plot_window=True,
x_label="Epoch")
if not os.path.exists("./model/" + instrument):
os.makedirs("./model/" + instrument)
if not os.path.exists("./loss/" + instrument):
os.makedirs("./loss/" + instrument)
epoch = 0
d_loss_list = []
g_loss_list = []
while True:
for num_batch, real_data in enumerate(data_loader):
############################
# (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))
###########################
for i in range(1):
real_data_d = next(iter(data_loader))
size = real_data_d.size(0)
real_data_t = torch.ones(size, 239, 4)
for i in range(size):
for j in range(239):
real_data_t[i][j] = torch.log(
real_data_d[i, j + 1, :] / real_data_d[i, j, 3])
y_real = Variable(torch.ones(size, 1, 1))
y_fake = Variable(torch.zeros(size, 1, 1))
real_data = Variable(real_data_t.float())
fake_data = Variable(
torch.from_numpy(
np.random.normal(0, 0.2,
(size, latent_dim, 1))).float())
fake_gen = generator(fake_data).detach()
prediction_real = discriminator(real_data)
loss_real = loss(prediction_real, y_real)
prediction_fake = discriminator(fake_gen)
loss_fake = loss(prediction_fake, y_fake)
d_loss = loss_real + loss_fake
d_optimizer.zero_grad()
d_loss.backward()
d_optimizer.step()
############################
# (2) Update G network: maximize log(D(G(z)))
###########################
size = real_data.size(0)
y_real = Variable(torch.ones(size, 1, 1))
fake_data = Variable(
torch.from_numpy(
np.random.normal(0, 0.2, (size, latent_dim, 1))).float())
fake_gen = generator(fake_data)
prediction = discriminator(fake_gen)
g_loss = loss(prediction, y_real)
g_optimizer.zero_grad()
g_loss.backward()
g_optimizer.step()
if num_batch % 20 == 0:
tmp = torch.ones(size, 239, 4)
data = real_data if num_batch == 0 else fake_gen
for batch in range(size):
for t in range(238, 0, -1):
data[batch, t, 0] = torch.sum(
data[batch, 0:t, 3]) + data[batch, t, 0]
data[batch, t, 1] = torch.sum(
data[batch, 0:t, 3]) + data[batch, t, 1]
data[batch, t, 2] = torch.sum(
data[batch, 0:t, 3]) + data[batch, t, 2]
data[batch, t, 3] = torch.sum(data[batch, 0:t + 1, 3])
data = torch.exp(data)
tmp = tmp * data
print("epoch: %d, num_batch: %d, d-loss: %.4f, g-loss: %.4f" %
(epoch, num_batch, d_loss.data.numpy(),
g_loss.data.numpy()))
visualize(instrument, tmp, epoch, num_batch)
plotter.add_values(epoch,
loss_train=g_loss.item(),
loss_val=d_loss.item())
d_loss_list.append(d_loss.item())
g_loss_list.append(g_loss.item())
if epoch % 10 == 0:
torch.save(
generator, "./model/" + instrument + "/generator_epoch_" +
str(epoch) + ".model")
torch.save(
discriminator, "./model/" + instrument +
"/discriminator_epoch_" + str(epoch) + ".model")
d_loss_np = np.array(d_loss_list)
np.save(
"./loss/" + instrument + "/d_loss_epoch_" + str(epoch) +
".npy", d_loss_np)
g_loss_np = np.array(g_loss_list)
np.save(
"./loss/" + instrument + "/g_loss_epoch_" + str(epoch) +
".npy", g_loss_np)
epoch += 1
if mode == "test" and epoch == epochs:
break
"500" : 1e-4,
"10000": 1e-5,
"20000": 1e-6
}
lr_decay_power = 4
momentum = 0.9
weight_decay = 0.005
n_classes = 3
n_boxes = 5
start = time.time()
plotter = LossAccPlotter(title="YOLOv2 loss",
save_to_filepath="plot.png",
show_regressions=True,
show_averages=True,
show_loss_plot=True,
show_acc_plot=False,
show_plot_window=True,
x_label="Batch")
# load image generator
#print("loading image generator...")
#generator = ImageGenerator(item_path, background_path)
print("loading ImageNet generator")
imageNet_data = ImageNet_data("./XmlToTxt/water_bottle_img", "./XmlToTxt/water_bottle_bbox", "./XmlToTxt/images_list.txt", n_classes)
# load model
print("loading initial model...")
yolov2 = YOLOv2(n_classes=n_classes, n_boxes=n_boxes)
model = YOLOv2Predictor(yolov2)
serializers.load_hdf5(initial_weight_file, model)
def main():
rand.seed()
#env = gym.make('Asteroids-v0')
env = gym.make('Breakout-v0')
num_actions = env.action_space.n
#print(num_actions)
plotter = LossAccPlotter(title="mem256_perGame",show_acc_plot=True,save_to_filepath="/mem256_perGame.png",show_loss_plot=False)
plotter2 = LossAccPlotter(title="mem256_100",show_acc_plot=True,save_to_filepath="/mem256_100.png",show_loss_plot=False)
#plotter.save_plot("./mem256.png")
observation = env.reset()
#observation = downsample(observation)
#reward = 0
action = 0
total_reward = 0
total_reward2 = 0
env.render()
prev_obs = []
curr_obs = []
D = []
step = 0
rate = 1
sess, output_net, x, cost, trainer, mask, reward, nextQ = initialize()
#load(sess)
startPrinting = False
for i in range(5):
observation, rw, done, info = env.step(action) # pass in 0 for action
observation = convert_to_small_and_grayscale(observation)
prev_obs = deepcopy(curr_obs)
curr_obs = obsUpdate(curr_obs,observation)
#e = [rw, action, deepcopy(prev_obs), deepcopy(curr_obs)]
#D.append(e)
action = 1
#print(i)
print("Entering mini-loop")
for _ in range(10):
step +=1
#print(step)
if done:
observation = env.reset()
if (len(D) > 256):
D.pop()
if step % 1000 == 0:
rate = rate / 2
if (rate < 0.05):
rate=0.05
#if step % 1000 == 0:
#save(sess)
action = magic(curr_obs, sess, output_net, x,step,rate, False) #change this to just take in curr_obs, sess, and False
#action = env.action_space.sample()
env.render()
observation, rw, done, info = env.step(action) # take a random action
#print(action, rw, step)
observation = convert_to_small_and_grayscale(observation)
e = [rw, action, deepcopy(prev_obs), deepcopy(curr_obs)]
D.append(e)
prev_obs = deepcopy(curr_obs)
curr_obs = obsUpdate(curr_obs,observation)
print("Entering full loop")
while step < 10001:
step +=1
#print(step)
if done:
print("saving to plot....")
plot_reward = total_reward
plotter.add_values(step, acc_train=plot_reward)
total_reward = 0
observation = env.reset()
if (len(D) > 256):
D.pop()
if step % 100 == 0:
print(step,"steps have passed")
save(sess, step)
rate = rate / 2
startPrinting = True
if (rate < 0.05):
rate=0.05
print("saving to plot2....")
plot_reward = total_reward2/100
plotter2.add_values(step, acc_train=plot_reward)
total_reward2 = 0
#print(step,"steps have passed")
if step % 500 == 0:
plotter.save_plot("./mem256_perGame.png")
plotter2.save_plot("./mem256_100.png")
action = magic(curr_obs, sess, output_net, x,step,rate, startPrinting) #change this to just take in curr_obs, sess, and False
#action = env.action_space.sample()
env.render()
observation, rw, done, info = env.step(action) # take a random action
#print(action, rw, step)
observation = convert_to_small_and_grayscale(observation)
e = [rw, action, deepcopy(prev_obs), deepcopy(curr_obs)]
D.insert(0,e)
prev_obs = deepcopy(curr_obs)
curr_obs = obsUpdate(curr_obs,observation)
update_q_function(D, sess, output_net, x, cost, trainer, mask, reward, nextQ)
total_reward = total_reward + rw
total_reward2 = total_reward2 + rw
plotter.block()
plotter2.block()
def main():
"""Run various checks on the LossAccPlotter.
They all follow the same pattern: Generate some random data (lines)
to display. Then display them (using various settings).
"""
print("")
print("------------------")
print("1 datapoint")
print("------------------")
# generate example values for: loss train, loss validation, accuracy train
# and accuracy validation
(loss_train, loss_val, acc_train, acc_val) = create_values(1)
# generate a plot showing the example values
show_chart(loss_train,
loss_val,
acc_train,
acc_val,
title="A single datapoint")
print("")
print("------------------")
print("150 datapoints")
print("Saved to file 'plot.png'")
print("------------------")
(loss_train, loss_val, acc_train, acc_val) = create_values(150)
show_chart(loss_train,
loss_val,
acc_train,
acc_val,
lap=LossAccPlotter(save_to_filepath="plot.png"),
title="150 datapoints, saved to file 'plot.png'")
print("")
print("------------------")
print("150 datapoints")
print("No accuracy chart")
print("------------------")
(loss_train, loss_val, _, _) = create_values(150)
show_chart(loss_train,
loss_val,
np.array([]),
np.array([]),
lap=LossAccPlotter(show_acc_plot=False),
title="150 datapoints, no accuracy chart")
print("")
print("------------------")
print("150 datapoints")
print("No loss chart")
print("------------------")
(_, _, acc_train, acc_val) = create_values(150)
show_chart(np.array([]),
np.array([]),
acc_train,
acc_val,
lap=LossAccPlotter(show_loss_plot=False),
title="150 datapoints, no loss chart")
print("")
print("------------------")
print("150 datapoints")
print("No accuracy chart")
print("------------------")
(loss_train, loss_val, _, _) = create_values(150)
show_chart(loss_train,
loss_val,
np.array([]),
np.array([]),
lap=LossAccPlotter(show_acc_plot=False),
title="150 datapoints, no accuracy chart")
print("")
print("------------------")
print("150 datapoints")
print("Only validation values (no training lines)")
print("------------------")
(_, loss_val, _, acc_val) = create_values(150)
show_chart(np.array([]),
loss_val,
np.array([]),
acc_val,
title="150 datapoints, only validation (no training)")
print("")
print("------------------")
print("150 datapoints")
print("No regressions")
print("------------------")
(loss_train, loss_val, acc_train, acc_val) = create_values(150)
show_chart(loss_train,
loss_val,
acc_train,
acc_val,
lap=LossAccPlotter(show_regressions=False),
title="150 datapoints, regressions deactivated")
print("")
print("------------------")
print("150 datapoints")
print("No averages")
print("------------------")
(loss_train, loss_val, acc_train, acc_val) = create_values(150)
show_chart(loss_train,
loss_val,
acc_train,
acc_val,
lap=LossAccPlotter(show_averages=False),
title="150 datapoints, averages deactivated")
print("")
print("------------------")
print("150 datapoints")
print("x-index 5 of loss_train should create a warning as its set to NaN")
print("------------------")
(loss_train, loss_val, acc_train, acc_val) = create_values(150)
# this should create a warning when LossAccPlotter.add_values() gets called.
loss_train[5] = float("nan")
show_chart(
loss_train,
loss_val,
acc_train,
acc_val,
title="150 datapoints, one having value NaN (loss train at x=5)")
print("")
print("------------------")
print("1000 datapoints training")
print("100 datapoints validation")
print("------------------")
nb_points_train = 1000
nb_points_val = 100
(loss_train, loss_val, acc_train, acc_val) = create_values(nb_points_train)
# set 9 out of 10 values of the validation arrays to -1.0 (Which will be
# interpreted as None in show_chart(). Numpy doesnt support None directly,
# only NaN, which is already used before to check whether the Plotter
# correctly creates a warning if any data point is NaN.)
all_indices = np.arange(0, nb_points_train - 1, 1)
keep_indices = np.arange(0, nb_points_train - 1,
int(nb_points_train / nb_points_val))
set_to_none_indices = np.delete(all_indices, keep_indices)
loss_val[set_to_none_indices] = -1.0
acc_val[set_to_none_indices] = -1.0
show_chart(
loss_train,
loss_val,
acc_train,
acc_val,
title="1000 training datapoints, but only 100 validation datapoints")
print("")
print("------------------")
print("5 datapoints")
print("slowly added, one by one")
print("------------------")
(loss_train, loss_val, acc_train, acc_val) = create_values(5)
lap = LossAccPlotter(title="5 datapoints, slowly added one by one")
for idx in range(loss_train.shape[0]):
lap.add_values(idx,
loss_train=loss_train[idx],
loss_val=loss_val[idx],
acc_train=acc_train[idx],
acc_val=acc_val[idx],
redraw=True)
sleep(1.0)
print("Close the chart to continue.")
lap.block()
if option.is_gd():
print("\tOptimization: Gradient Descent")
elif option.is_sgd():
print("\tOptimization: Stochastic Gradient Descent")
if option.is_linear():
print("\tActivation Function: Linear")
elif option.is_sigmoid():
print("\tActivation Function: Sigmoid")
if option.is_l2norm():
print("\tRegularization: L2Norm")
elif option.is_dropout():
print("\tRegularization: Drop out")
net = Network(training, test, option)
if read_weights_from_file == "y":
loaded = np.load('weights.npz')
net.set_hid_weights(loaded['hid_weights']).set_out_weights(
loaded['out_weights']).set_hid_bias(
loaded['hid_bias']).set_out_bias(loaded['out_bias'])
plotter = LossAccPlotter(show_acc_plot=False)
for i in range(NUM_OF_ITER):
training_loss = 0
if read_weights_from_file != "y":
training_loss = net.train(i == NUM_OF_ITER - 1)
test_loss = net.test(i == NUM_OF_ITER - 1)
plotter.add_values(i, loss_train=training_loss, loss_val=test_loss)
plotter.block()
train_acc.append(None)
val_acc_top1.append(None)
val_acc_top5.append(None)
count_acc += 1
print("Start plotting")
# print(len(loss))
# print(train_acc)
# print(val_acc_top1)
# print(val_acc_top5)
plot_title = 'Loss and Accuracy Plotted Using ' + num_classes + ' Keywords (' + num_train_per_class + ' training videos each) and minibatch size of ' + batch_size
plotter = LossAccPlotter(title=plot_title,
save_to_filepath=num_classes + '_' +
num_train_per_class + '_' + batch_size + '.png',
show_regressions=False,
show_averages=False,
show_loss_plot=True,
show_acc_plot=True,
show_plot_window=True,
x_label="Epoch")
for epoch in range(len(loss)):
plotter.add_values(epoch,
loss_train=loss[epoch],
acc_train=train_acc[epoch],
acc_val=val_acc_top1[epoch])
plotter.block()
print("Finished plotting")
def main():
"""Create the example plots in the following way:
1. Generate example data (all plots use more or less the same data)
2. Generate plot 1: "standard" example with loss and accuracy
3. Generate plot 2: Same as 1, but only loss / no accuracy
4. Generate plot 3: Same as 1, but no validation lines (only training dataset)
5. Generate plot 4: Same as 1, but use only every 10th validation datapoint
(i.e. resembles real world scenario where you rarely
validate your machine learning method)
"""
nb_points = 500
loss_train = add_noise(np.linspace(0.9, 0.1, num=nb_points), 0.025)
loss_val = add_noise(np.linspace(0.7, 0.3, num=nb_points), 0.045)
acc_train = add_noise(np.linspace(0.52, 0.95, num=nb_points), 0.025)
acc_val = add_noise(np.linspace(0.65, 0.75, num=nb_points), 0.045)
# Normal example plot
lap = LossAccPlotter(save_to_filepath="example_plot.png")
show_chart(loss_train,
loss_val,
acc_train,
acc_val,
lap=lap,
title="Example Plot with Loss and Accuracy")
# Plot showing only the results of the loss function (accuracy off)
lap = LossAccPlotter(show_acc_plot=False,
save_to_filepath="example_plot_loss.png")
show_chart(loss_train,
loss_val,
acc_train,
acc_val,
lap=lap,
title="Example Plot, only Loss Function")
# Plot showing only training dataset values (but for both loss and accuracy)
lap = LossAccPlotter(save_to_filepath="example_plot_only_training.png")
show_chart(
loss_train,
np.array([]),
acc_train,
np.array([]),
lap=lap,
title="Example Plot, only Training Dataset / no Validation Dataset")
# Plot with a different update interval for training and validation dataset
# (i.e. only one validation value for every 10 training values)
#
# Set 9 out of 10 validation values to -1, which will be transformed into
# None in show_chart(). (same technique as in check_laplotter.py)
nb_points_train = nb_points
nb_points_val = int(nb_points * 0.1)
all_indices = np.arange(0, nb_points_train - 1, 1)
keep_indices = np.arange(0, nb_points_train - 1,
int(nb_points_train / nb_points_val))
set_to_none_indices = np.delete(all_indices, keep_indices)
loss_val[set_to_none_indices] = -1.0
acc_val[set_to_none_indices] = -1.0
lap = LossAccPlotter(show_acc_plot=False,
save_to_filepath="example_plot_update_intervals.png")
show_chart(loss_train, loss_val, acc_train, acc_val, lap=lap,
title="Example Plot with different Update Intervals for Training " \
"and Validation Datasets")
def train(ngram, name, bar, drop_out, dataset, is_cuda=False, edges=False):
plotter = LossAccPlotter(title="This is an example plot",
save_to_filepath="/tmp/my_plot.png",
show_regressions=True,
show_averages=True,
show_loss_plot=True,
show_acc_plot=True,
show_plot_window=False,
x_label="Epoch")
print('load data helper.')
data_helper = DataHelper(mode='train')
b_size = len(data_helper.label)
print('*' * 100)
print('train set total:', b_size)
if os.path.exists(os.path.join('.',
name + '.pkl')) and name != 'temp_model':
print('load model from file')
model = torch.load(os.path.join('.', name + '.pkl'))
else:
print('new model')
if name == 'temp_model':
name == 'temp_model'
edges_weights, edges_mappings, count = cal_PMI()
## -----------------************************** import the datahelper class to get the vocab-5 doc*****************------------------------
## class_num = len(data_helper.labels_str) is changed, consider just a score
model = Model(class_num=data_helper.labels_str,
hidden_size_node=200,
vocab=data_helper.vocab,
n_gram=ngram,
drop_out=drop_out,
edges_matrix=edges_mappings,
edges_num=count,
trainable_edges=edges,
pmi=edges_weights,
cuda=is_cuda)
### --------------------------------------- ###
print(model)
if is_cuda:
print('cuda')
model.cuda()
loss_func = torch.nn.MSELoss()
loss_mae = torch.nn.L1Loss(reduction='sum')
optim = torch.optim.Adam(model.parameters(), weight_decay=1e-3)
iter = 0
if bar:
pbar = tqdm.tqdm(total=NUM_ITER_EVAL)
best_acc = 0.0
last_best_epoch = 0
start_time = time.time()
total_loss = 0.0
total_correct = 0
total = 0
accuracy = 0.0
num_epoch = 500
weight_decays = 1e-4
for content, label, epoch in data_helper.batch_iter(
batch_size=32, num_epoch=num_epoch):
improved = ''
model.train()
pred = model(content)
pred_sq = torch.squeeze(pred, 1)
l2_reg = 0.5 * weight_decays * (
model.seq_edge_w.weight.to('cpu').detach().numpy()**2).sum()
loss = loss_func(pred_sq, label.float()) + l2_reg
#-------------------------------------------#
error = loss_mae(pred_sq.cpu().data, label.cpu())
accuracy += error
total += len(pred) ##batch size = len(label)
total_loss += (loss.item() * len(pred))
total_correct += loss.item()
optim.zero_grad()
loss.backward()
optim.step()
iter += 1
if bar:
pbar.update()
if iter % NUM_ITER_EVAL == 0:
if bar:
pbar.close()
val_acc, val_loss = dev(model)
if val_acc < best_acc:
best_acc = val_acc
last_best_epoch = epoch
improved = '* '
torch.save(model, name + '.pkl')
msg = 'Epoch: {0:>6} Iter: {1:>6}, Train Loss: {5:>7.2}, Train Error: {6:>7.2}' \
+ 'Val Acc: {2:>7.2}, Time: {3}{4}, val error:{7:>7.2}' \
# + ' Time: {5} {6}'
print(
msg.format(epoch, iter, val_acc, get_time_dif(start_time),
improved, total_correct / (NUM_ITER_EVAL),
float(accuracy) / float(total), val_loss))
plotter.add_values(epoch,
loss_train=total_correct / (NUM_ITER_EVAL),
acc_train=float(accuracy) / float(total),
loss_val=val_loss,
acc_val=best_acc)
total_loss = 0.0
total_correct = 0
accuracy = 0.0
total = 0
if bar:
pbar = tqdm.tqdm(total=NUM_ITER_EVAL)
plotter.block()
return name
def train_model(num_tasks,
models,
dataloaders,
dataset_sizes,
criterion,
optimizers,
schedulers,
epochs=15):
since = time.time()
num_tasks = num_tasks
use_gpu = torch.cuda.is_available()
final_outputs = [
] #this is the variable to accumulate the outputs of all columns for each task
middle_outputs = [
] #this is the variable for keeping outputs of current task in each column
#we iterate for each task
for task_id in range(num_tasks):
#everytime we do a new task, we empty the final outputs
final_outputs[:] = []
# we now iterate for each previous column until the one of our task
for i in range(0, task_id + 1):
#we save the weights with best results
best_model_wts = copy.deepcopy(models[task_id][i].state_dict())
model = models[task_id][i]
optimizer = optimizers[i]
scheduler = schedulers[i]
# if it's not the column corresponding to the task, do not train
if task_id != i:
#this is the case for "previous" columns so we only need to pass data, not train
num_epochs = 1
middle_outputs[:] = []
else:
num_epochs = epochs
dataloader = dataloaders[i]
dataset_size = dataset_sizes[i]
best_acc = 0.0
# let's add a plotter for loss and accuracy
plotter = LossAccPlotter()
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
scheduler.step()
model.train(True) # Set model to training mode
else:
model.train(False) # Set model to evaluate mode
running_loss = 0.0
running_corrects = 0
# Iterate over data.
#tqdm shows a progression bar
for data in tqdm(dataloader[phase]):
# get the inputs
inputs, labels = data
if inputs.type() != int and labels.type() != int:
# wrap them in Variable
if use_gpu:
inputs = Variable(inputs.cuda())
labels = Variable(labels.cuda())
else:
inputs, labels = Variable(inputs), Variable(
labels)
# zero the parameter gradients
optimizer.zero_grad()
# forward
outputs, middle_outputs = model(
inputs, final_outputs)
#we save the outputs of this column in middle outputs and we have previous columns in final
_, preds = torch.max(outputs.data, 1)
loss = criterion(outputs, labels)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.data[0] * inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / dataset_size[phase]
epoch_acc = running_corrects / dataset_size[phase]
print('{} Loss: {:.4f} Acc: {:.4f}'.format(
phase, epoch_loss, epoch_acc))
if phase == 'train':
plotter.add_values(epoch,
loss_train=epoch_loss,
acc_train=epoch_acc,
redraw=False)
else:
plotter.add_values(epoch,
loss_val=epoch_loss,
acc_val=epoch_acc,
redraw=False)
# deep copy the model
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
print()
plotter.redraw()
if task_id != i:
final_outputs.append(middle_outputs)
#we add the ouput of this column to final outputs
plotter.save_plot('plots%d.%d.png' % (task_id, i))
print('Best val Acc: {:4f}'.format(best_acc))
# load best model weights
models[task_id][i].load_state_dict(best_model_wts)
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
return models
