def train_model(X_train, y_train, X_valid, y_valid, n_minibatches, batch_size, lr ,num_filters, filter_size, history_length, model_dir="./models", tensorboard_dir="./tensorboard"):
# create result and model folders
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
agent = Model(filter_size=filter_size, num_filters=num_filters, learning_rate=lr, history_length=history_length)
agent.sess.run(tf.global_variables_initializer())
tensorboard_eval = Evaluation(tensorboard_dir)
tf.reset_default_graph()
#X_train, y_train = shuffle(X_train, y_train, random_state = 0) # but if i shuffle I loose history!
X_valid = np.reshape(X_valid, (X_valid.shape[0], 96, 96, 1))
n_samples = X_valid.shape[0]
n_batches = n_samples // batch_size
X_split = np.array_split(X_valid, n_batches)
Y_split = np.array_split(y_valid, n_batches)
for i in range(n_batches):
X_split[i], Y_split[i] = make_history(X_split[i], Y_split[i], history_length)
total = n_minibatches//1000
train = np.zeros(total)
valid = np.zeros(total)
index = uniform_sampling(X_train[history_length-1:], y_train[history_length-1:], 25000)
for i in range(n_minibatches):
x, y = sample_minibatch(X_train, y_train, index, batch_size, history_length)
y_hot = one_hot(y)
agent.sess.run([agent.train_step, agent.cross_entropy], feed_dict={agent.x_image: x, agent.y_: y_hot})
if ((i+1)%1000==0):
step=(i+1)//1000
index = uniform_sampling(X_train[history_length-1:], y_train[history_length-1:], 25000)
print("Training step: " + str(step) + " of " + str(total))
train[step-1] = agent.accuracy.eval(session=agent.sess, feed_dict={agent.x_image:x, agent.y_: y_hot})
for j in range(n_batches):
valid[step-1] += agent.accuracy.eval(session=agent.sess, feed_dict={agent.x_image:X_split[j], agent.y_: Y_split[j]})
valid[step-1] = valid[step-1]/n_batches
print("Train accuracy: "+ str(train[step-1]))
print("Test accuracy: " + str(valid[step-1]))
eval_dict = {"train":train[step-1], "valid":valid[step-1]}
tensorboard_eval.write_episode_data(step, eval_dict)
# TODO: save your agent
model_dir = agent.save(os.path.join(model_dir, "agent.ckpt"))
print("Model saved in file: %s" % model_dir)
agent.sess.close()
def train_model(X_train, y_train, X_valid, n_minibatches, batch_size, lr, model_dir="./models", tensorboard_dir="./tensorboard"):
# create result and model folders
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
# TODO: specify your agent with the neural network in agents/bc_agent.py
# agent = BCAgent(...)
tensorboard_eval = Evaluation(tensorboard_dir)
def train_model(X_train,
y_train,
X_valid,
y_valid,
n_minibatches,
batch_size,
lr,
model_dir="./models",
tensorboard_dir="./tensorboard",
history_length=1):
# create result and model folders
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
# TODO: specify your neural network in model.py
ks = 5
num_kernels = 32
agent = Model(lr, ks, num_kernels, history_length=history_length)
tensorboard_eval = Evaluation(tensorboard_dir)
# TODO: implement the training
#
# 1. write a method sample_minibatch and perform an update step
def sample_minibatch(X, y, batch_size):
batch_start = np.random.randint(X_train.shape[0] - batch_size)
X_minibatch = X[batch_start:batch_start + batch_size]
y_minibatch = y[batch_start:batch_start + batch_size]
return X_minibatch, y_minibatch
def create_probability_mask(y):
distr = "uniform"
num_classes = 5
# get discrete actions
# y_discrete = actions_to_ids(y)
action_counts = action_distribution(y)
# get the probabilities from the frequency of the classes
if distr == "uniform":
for i in range(action_counts.shape[0]):
if action_counts[i] == 0:
num_classes -= 1
else:
action_counts[i] = 1 / action_counts[i]
action_counts = action_counts * 1 / num_classes
# get a mask with the probabilities for choosing each data point
mask = np.zeros(y.shape[0])
for i in range(5):
mask[y[:, i] == 1] = action_counts[i]
return mask
def sample_minibatch_with_uniform_actions(X, y, batch_size, mask):
# now sample indices according to the probabilities
indices = np.arange(y.shape[0])
batch_inds = np.random.choice(indices, size=batch_size, p=mask)
# create batch
X_batch = X[batch_inds, ...]
y_batch = y[batch_inds, ...]
# print(action_distribution(y_batch))
return X_batch, y_batch
# since 0 (straight) is 10 times more likely than any other action
# I choose any
# 2. compute training/ validation accuracy and loss for the batch and visualize them with tensorboard. You can watch the progress of
# your training in your web browser
#
# create probability mask for the sampling
prob_mask = create_probability_mask(y_train)
# training loop
for i in range(n_minibatches):
X_batch, y_batch = sample_minibatch_with_uniform_actions(
X_train, y_train, batch_size, prob_mask)
_, train_loss = agent.sess.run([agent.train, agent.loss], {
agent.inputs: X_batch,
agent.targets: y_batch
})
val_loss = agent.sess.run(agent.loss, {
agent.inputs: X_valid,
agent.targets: y_valid
})
if i % 10 == 0:
train_acc = agent.accuracy(X_batch, y_batch)
val_acc = agent.accuracy(X_valid, y_valid)
print('Step %i, Training Loss: %.4f, Validation Loss: %.4f' %
(i, train_loss, val_loss))
print('Train accuracy: %.2f, Validation accuracy: %.2f' %
(train_acc * 100, val_acc * 100))
tensorboard_eval.write_episode_data(i, {
"loss": train_loss,
"val_loss": val_loss
})
# TODO: save your agent
model_dir = agent.save(os.path.join(model_dir, "agent.ckpt"))
print("Model saved in file: %s" % model_dir)
def train_model(X_train, y_train, X_valid,y_valid,config,model_dir="./models", tensorboard_dir="./tensorboard"):
n_minibatches, batch_size, lr,hidden_units,agent_type = config.n_minibatches, config.batch_size, config.lr,config.hidden_units,config.agent_type
# create result and model folders
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
# TODO: specify your agent with the neural network in agents/bc_agent.py
if agent_type=="FCN":
agent = BCAgent("FCN",lr,hidden_units)
else:
agent= BCAgent("CNN",lr,hidden_units)
#states=['training_accuracy']
states=['training_accuracy','validation_accuracy','loss']
tensorboard_eval = Evaluation(tensorboard_dir,"Learning_curves",states)
# TODO: implement the training
# 1. write a method sample_minibatch and perform an update step
def sample_minibatch(X,y):
length=X.shape[0]
X_batch=[]
y_batch=[]
sampled_idx= rnd.sample(range(1,length),batch_size)
for idx in sampled_idx:
X_batch.append(X[idx])
y_batch.append(y[idx])
return X_batch,y_batch
# 2. compute training/ validation accuracy and loss for the batch and visualize them with tensorboard. You can watch the progress of
# your training *during* the training in your web browser
#
# training loop
for i in range(n_minibatches):
X_batch,y_batch=sample_minibatch(X_train,y_train)
loss=agent.update(X_batch,y_batch)
print(loss)
if i % 10 == 0:
training_correct=0
training_total=0
validation_correct=0
validation_total=0
training_accuracy=0
validation_accuracy=0
with torch.no_grad():
output_training = agent.predict(torch.tensor(X_batch).to(device))
output_validation= agent.predict(torch.tensor(X_valid).to(device))
for idx, label in enumerate(output_training):
if torch.argmax(label) == torch.tensor(y_batch[idx],dtype=torch.long, device=device):
training_correct += 1
training_total += 1
for idx, label in enumerate(output_validation):
if torch.argmax(label) == torch.tensor(y_valid[idx],dtype=torch.long, device=device):
validation_correct += 1
validation_total += 1
training_accuracy=training_correct/training_total
validation_accuracy=validation_correct/validation_total
print("Training accuracy: %f" %training_accuracy)
print("Validation accuracy: %f" %validation_accuracy)
# compute training/ validation accuracy and write it to tensorboard
eval_dic={'training_accuracy':training_accuracy,'validation_accuracy':validation_accuracy,'loss':loss.item()}
#eval_dic={'training_accuracy':training_accuracy}
tensorboard_eval.write_episode_data(i,eval_dic)
# save your agent
save_file=datetime.now().strftime("%Y%m%d-%H%M%S")+"_bc_agent.pt"
model_dir = agent.save(os.path.join(model_dir,save_file))
print("Model saved in file: %s" %model_dir)
def train_model(X_train,
y_train,
X_valid,
y_valid,
history_length,
n_minibatches,
batch_size,
lr,
model_dir="./models",
tensorboard_dir="./tensorboard"):
# create result and model folders
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
# TODO: specify your agent with the neural network in agents/bc_agent.py
# agent = BCAgent(...)
agent = BCAgent(history_length)
#tensorboard_eval = Evaluation(tensorboard_dir)
tensorboard_eval = Evaluation(
tensorboard_dir, "train",
["loss_train", "loss_valid", "accuracy_train", "accuracy_valid"])
# TODO: implement the training
#
# 1. write a method sample_minibatch and perform an update step
# 2. compute training/ validation accuracy and loss for the batch and visualize them with tensorboard. You can watch the progress of
# your training *during* the training in your web browser
#
# training loop
# for i in range(n_minibatches):
# ...
# for i % 10 == 0:
# # compute training/ validation accuracy and write it to tensorboard
# tensorboard_eval.write_episode_data(...)
# TODO: save your agent
# model_dir = agent.save(os.path.join(model_dir, "agent.pt"))
# print("Model saved in file: %s" % model_dir)
def sample_minibatch(X_train, y_train, permutation, i_n_minibatches):
indices = permutation[i_n_minibatches:i_n_minibatches + batch_size]
batch_x, batch_y = X_train[indices], y_train[indices]
return batch_x, batch_y
permutation = torch.randperm(X_train.shape[0])
# training loop
for i in range(n_minibatches):
X, y = sample_minibatch(X_train, y_train, permutation, i)
#print('episode: ', i)
agent.update(X, y)
# output
outputs_train = agent.predict(X)
outputs_valid = agent.predict(X_valid)
# loss
loss_train = torch.nn.CrossEntropyLoss()(
outputs_train, torch.LongTensor(y).cuda().squeeze(1))
loss_valid = torch.nn.CrossEntropyLoss()(
outputs_valid, torch.LongTensor(y_valid).cuda().squeeze(1))
# labels_output
labels_train = torch.nn.functional.softmax(
outputs_train, dim=1).argmax(dim=1) # check dimension
labels_valid = torch.nn.functional.softmax(
outputs_valid, dim=1).argmax(dim=1) # check dimension
# accuracy
if i % 10 == 0:
print('episode: ', i)
corrects_train = (
labels_train == torch.LongTensor(y).cuda().squeeze(1))
corrects_valid = (
labels_valid == torch.LongTensor(y_valid).cuda().squeeze(1))
accuracy_train = corrects_train.sum().float() / float(len(y))
accuracy_valid = corrects_valid.sum().float() / float(len(y_valid))
tensorboard_eval.write_episode_data(i,
eval_dict={
"loss_train":
loss_train.item(),
"loss_valid":
loss_valid.item(),
"accuracy_train":
accuracy_train.item(),
"accuracy_valid":
accuracy_valid.item()
})
#print('episode: ', i, ' acc train: ', accuracy_train.item(), ' loss train: ', loss_train.item(), \
# 'acc val: ', accuracy_valid.item(), ' loss val: ', loss_valid.item())
agent.save(os.path.join(model_dir, "agent.pt"))
def train_model(X_train,
y_train,
X_valid,
y_valid,
epochs,
batch_size,
lr,
history_length=1,
model_dir="./models",
tensorboard_dir="./tensorboard"):
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
agent = BCAgent(lr=lr, history_length=history_length)
tensorboard_eval = Evaluation(tensorboard_dir)
t_losses, t_f1_scores, v_f1_scores = [], [], []
train_batch_size = int(len(X_train) / batches)
val_batch_size = int(len(y_valid) / batches)
best_f1_score = 0
for i in range(epochs):
start = time.time()
all_train_preds, all_val_preds = [], []
for j in range(batches):
X_batch_tr = X_train[j * train_batch_size:(j + 1) *
train_batch_size]
y_batch_tr = y_train[j * train_batch_size:(j + 1) *
train_batch_size]
X_batch_va = X_valid[j * val_batch_size:(j + 1) * val_batch_size]
y_batch_va = y_valid[j * val_batch_size:(j + 1) * val_batch_size]
X_batch_va = torch.Tensor(X_batch_va).cuda()
X_batch_va = X_batch_va.view((-1, 1 + history_length, 96, 96))
t_loss, train_preds = agent.update(X_batch_tr, y_batch_tr)
train_preds = torch.max(train_preds.data, 1)[1]
all_train_preds = np.concatenate(
[all_train_preds, train_preds.cpu().numpy()])
with torch.no_grad():
val_preds = agent.predict(X_batch_va)
val_preds = torch.max(val_preds.data, 1)[1]
all_val_preds = np.concatenate(
[all_val_preds, val_preds.cpu().numpy()])
torch.cuda.empty_cache()
if (j + 1) * train_batch_size < len(X_train):
X_batch_tr = X_train[(j + 1) * train_batch_size:]
y_batch_tr = y_train[(j + 1) * train_batch_size:]
X_batch_va = X_valid[(j + 1) * val_batch_size:]
y_batch_va = y_valid[(j + 1) * val_batch_size:]
X_batch_va = torch.Tensor(X_batch_va).cuda()
X_batch_va = X_batch_va.view((-1, 1 + history_length, 96, 96))
t_loss, train_preds = agent.update(X_batch_tr, y_batch_tr)
train_preds = torch.max(train_preds.data, 1)[1]
all_train_preds = np.concatenate(
[all_train_preds, train_preds.cpu().numpy()])
with torch.no_grad():
val_preds = agent.predict(X_batch_va)
val_preds = torch.max(val_preds.data, 1)[1]
all_val_preds = np.concatenate(
[all_val_preds, val_preds.cpu().numpy()])
train_f1_score = f1_score(all_train_preds, y_train, average='weighted')
val_f1_score = f1_score(all_val_preds, y_valid, average='weighted')
if best_f1_score < val_f1_score:
print(f'Saving model at epoch {i+1}')
agent.save(f'agent1_15k_epoch{i+1}_{batches}.pt')
best_f1_score = val_f1_score
t_losses.append(t_loss)
t_f1_scores.append(train_f1_score)
v_f1_scores.append(val_f1_score)
if i % 10 == 0:
print(
f"Epoch: {i+1}\tTrain Loss: {t_loss:.3f}\tTrain f1_score:{train_f1_score:.3f}\tValidation f1_score:{val_f1_score:.3f}"
)
tensorboard_eval.write_episode(i, {
"Train Accuracy": train_acc,
"Validation Accuracy": valid_acc
})
end = time.time()
print(f'Epoch {i+1} Time: {end - start}s')
return t_losses, t_f1_scores, v_f1_scores
def train_model(X_train,
y_train,
X_valid,
n_minibatches,
batch_size,
lr,
model_dir="./models",
tensorboard_dir="./tensorboard"):
# create result and model folders
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
# TODO: specify your neural network in model.py
agent = Model(lr)
tensorboard_eval = Evaluation(tensorboard_dir)
#train_writer = tf.summary.FileWriter('./logs/1/train ', sess.graph)
batch_size = X_train.shape[0] // n_minibatches
print("Batchsize", batch_size)
# TODO: implement the training
#
# 1. write a method sample_minibatch and perform an update step
saver = tf.train.import_meta_graph('SaveGraph/data-all.meta')
saver.restore(agent.sess, tf.train.latest_checkpoint('./SaveGraph'))
graph = tf.get_default_graph()
x = agent.sess.graph.get_tensor_by_name('input:0')
y = agent.sess.graph.get_tensor_by_name('output:0')
#op = agent.sess.graph.get_operations()
#print("before-----------------opti---opertaions",op)
cost = agent.sess.graph.get_tensor_by_name('my_cost:0')
optimizer = agent.sess.graph.get_operation_by_name('my_adam')
accuracy = agent.sess.graph.get_tensor_by_name('my_acc:0')
correct_prediction = agent.sess.graph.get_tensor_by_name(
'predict_output:0')
summary_writer = tf.summary.FileWriter('./Output', agent.sess.graph)
count = 0
loss = 0
acc_tr = 0
valid_acc = 0
for batch in range(X_train.shape[0] // batch_size):
batch_x = X_train[batch * batch_size:min((batch + 1) *
batch_size, len(X_train))]
batch_y = y_train[batch * batch_size:min((batch + 1) *
batch_size, len(y_train))]
print("BATCH ", batch)
#print("Y BATCH SHAPE",batch_y.shape)
# Run optimization op (backprop).
# Calculate batch loss and accuracy
opt = agent.sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
loss, acc_tr = agent.sess.run([cost, accuracy],
feed_dict={
x: batch_x,
y: batch_y
})
if batch % 20 == 0:
te_dict = {"loss": loss, "training_accuracy": acc_tr}
tensorboard_eval.write_episode_data(batch, te_dict)
for batch in range(X_valid.shape[0] // batch_size):
batch_x = X_valid[batch * batch_size:min((batch + 1) *
batch_size, len(X_valid))]
batch_y = y_valid[batch * batch_size:min((batch + 1) *
batch_size, len(y_valid))]
# Run optimization op (backprop).
# Calculate batch loss and accuracy
cp = agent.sess.run(correct_prediction,
feed_dict={
x: batch_x,
y: batch_y
})
count += len(cp[cp == True])
valid_acc = count / len(X_valid)
#accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#loss, acc = sess.run([cost, accuracy], feed_dict={x: batch_x,y: batch_y})
#print(" Loss= " +\"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))
print(loss, acc_tr)
print("Validation Accuracy :", valid_acc)
# 2. compute training/ validation accuracy and loss for the batch and visualize them with tensorboard. You can watch the progress of
# your training in your web browser
#
# training loop
# for i in range(n_minibatches):
# ...
#tensorboard_eval.write_episode_data(count, te_dict)
# TODO: save your agent
model_dir = agent.save(os.path.join(model_dir, "agent.ckpt"))
print(model_dir)
print("Model saved in file: %s" % model_dir)
agent.sess.close()
def train_model(X_train,
Y_train,
X_valid,
Y_valid,
epochs,
batch_size,
lr,
history_length=1,
model_dir="./models",
tensorboard_dir="./tensorboard"):
# create result and model folders
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
# TODO: specify your neural network in model.py
agent = Model(learning_rate=lr, history_length=history_length)
print('exit from model')
#y = tf.nn.softmax(agent.logits)
acc = []
acc_valid = []
train_cost = np.zeros(epochs)
valid_cost = np.zeros(epochs)
correct_prediction = agent.prediction
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tensorboard_eval = Evaluation(tensorboard_dir)
tf.reset_default_graph()
with agent.sess as sess:
sess.run(tf.global_variables_initializer())
print('inside session')
start_time = time()
for epoch in range(epochs):
correctsum = 0
#x_train_batchsize = 16
x_train_batchsize = len(X_train) // batch_size
print("X_train_batchsize :::::", x_train_batchsize)
correctsum_val = 0
#x_val_batchsize = 12
x_val_batchsize = len(X_valid) // batch_size
train_cost = np.zeros((epochs))
valid_cost = np.zeros((epochs))
for i in range(x_train_batchsize):
x_batch = X_train[i * batch_size:(i + 1) * batch_size]
y_batch = Y_train[i * batch_size:(i + 1) * batch_size]
y_batch = id_to_action(y_batch)
#_, loss = sess.run([agent.optimizer, agent.cost],feed_dict={agent.x_input:x_batch, agent.y_label: y_batch})
# training
for i in range(x_train_batchsize):
x_batch = X_train[i * batch_size:(i + 1) * batch_size]
y_batch = Y_train[i * batch_size:(i + 1) * batch_size]
y_batch = id_to_action(y_batch)
batch_correct_count = sess.run(accuracy,
feed_dict={
agent.x_input: x_batch,
agent.y_label: y_batch
})
correctsum += batch_correct_count
_, loss = sess.run([agent.optimizer, agent.cost],
feed_dict={
agent.x_input: x_batch,
agent.y_label: y_batch
})
train_cost[epoch] += sess.run(agent.cost,
feed_dict={
agent.x_input: x_batch,
agent.y_label: y_batch
})
print('iteration {} , epoch {}, train cost {:.2f}'.format(
i, epoch, train_cost[epoch]))
total_accuracy = correctsum / x_train_batchsize
acc.append(total_accuracy)
print('epoch {}, loss {:.2f}, train accuracy {:.2f}%'.format(
epoch, loss, total_accuracy * 100),
end='\n')
#validation
for i in range(x_val_batchsize):
x_val = X_valid[i * batch_size:(i + 1) * batch_size]
y_val = Y_valid[i * batch_size:(i + 1) * batch_size]
#y_val = id_to_action(y_val)
batch_correct_count = sess.run(accuracy,
feed_dict={
agent.x_input: x_val,
agent.y_label: y_val
})
correctsum_val += batch_correct_count
valid_cost[epoch] += agent.sess.run(agent.cost,
feed_dict={
agent.x_input: x_val,
agent.y_label: y_val
})
print(
'valid iteration{}, epoch{}, valid cost ::: {:.2f}'.format(
i, epoch, valid_cost[epoch]))
total_accuracy_v = correctsum_val / x_val_batchsize
acc_valid.append(total_accuracy_v)
print('epoch {}, validation accuracy {:.2f}%'.format(
epoch, total_accuracy_v * 100),
end='\n')
train_cost[epoch] = train_cost[epoch] / x_train_batchsize
valid_cost[epoch] = valid_cost[epoch] / x_val_batchsize
print("[%d/%d]: train_cost: %.4f, valid_cost: %.4f" %
(epoch + 1, epochs, train_cost[epoch], valid_cost[epoch]))
print('epoch {},train_cost{:.2f}, validation cost{:.2f}'.format(
epoch, train_cost[epoch], valid_cost[epoch]))
eval_dict = {
"train": train_cost[epoch],
"valid": valid_cost[epoch]
}
tensorboard_eval.write_episode_data(epoch, eval_dict)
# TODO: save your agent
agent.save(os.path.join(model_dir, "agent.ckpt"))
print(model_dir)
print("Model saved in file: %s" % model_dir)
agent.sess.close()
def train_model(X_train, y_train, X_valid, n_minibatches, file, batch_size, lr, model_dir="./models", tensorboard_dir="./tensorboard"):
# create result and model folders
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
tensorboard_eval = Evaluation(tensorboard_dir)
# TODO: specify your neural network in model.py
agent = Model(batch_size = batch_size, lr = lr, history_length = history_length)
print("... model created")
train_accuracy = 0
valid_accuracy = 0
with agent.sess:
for epoch in range(n_minibatches):
_loss = 0
for i in range(X_train.shape[0] // batch_size):
first_index = np.random.randint(0, X_train.shape[0] - batch_size - history_length - 1)
last_index = first_index + batch_size
X_train_mini = X_train[first_index : last_index, :, :]
y_train_mini = y_train[first_index : last_index, :]
X_train_mini, y_train_mini = preprocessing(X_train_mini, y_train_mini, history_length)
#print('... done loading training data')
opt, l = agent.sess.run([agent.optimizer, agent.loss], feed_dict = {agent.x_image: X_train_mini, agent.y_: y_train_mini})
_loss += l/(X_train.shape[0] //batch_size)
print("\n\nPREDICTION:", agent.predict.eval(feed_dict={agent.x_image: X_train_mini}))
train_accuracy = agent.accuracy.eval(feed_dict = {agent.x_image: X_train_mini, agent.y_: y_train_mini})
print('... training done')
valid_accuracy = 0
for i in range(X_valid.shape[0] // batch_size):
first_index = np.random.randint(0, X_train.shape[0] - batch_size - history_length - 1)
last_index = first_index + batch_size
X_valid_mini = X_valid[first_index : last_index, :, :]
y_valid_mini = y_valid[first_index : last_index, :]
X_valid_mini, y_valid_mini = preprocessing(X_valid_mini, y_valid_mini, history_length)
#print('... done loading validation data')
ac = agent.accuracy.eval(feed_dict={agent.x_image: X_valid_mini, agent.y_: y_valid_mini})
#print("ac:", ac)
valid_accuracy += ac /(X_valid.shape[0] //batch_size)
print('... validation done')
eval_dict = {"train":train_accuracy, "valid":valid_accuracy, "loss":_loss}
tensorboard_eval.write_episode_data(epoch, eval_dict)
#train_plot = []
#train_plot.append(train_accuracy)
file.write(str(train_accuracy))
print("Epoch:",epoch+1, "Train accuracy:", train_accuracy, "validation accuracy:", valid_accuracy, "loss:", _loss)
'''
plt.plot(train_plot, label = str(train_plot))
plt.ylabel('Train Accuracy')
plt.xlabel('Epochs')
plt.title('Train Accuracy over epochs')
plt.legend()
plt.savefig('Training for history:'+ str(history_length) +'.png')
plt.close()
'''
save_path = os.path.join(model_dir, "agent.ckpt")
agent.save(save_path)
tensorboard_eval.close_session()
print("Model saved in file: %s" % model_dir)
agent.sess.close()
'''SA'''
def train_online(name,
env,
num_episodes=10000,
lr=1e-4,
discount_factor=0.99,
batch_size=64,
epsilon=0.05,
epsilon_decay=0.0,
boltzmann=False,
tau=0.01,
double_q=False,
buffer_capacity=5e5,
history_length=0,
diff_history=False,
skip_frames=0,
big=False,
try_resume=False):
print("AGENT: " + name)
print("\t... creating agent")
# prepare folders
model_path = os.path.join(base_path, name)
ckpt_path = os.path.join(model_path, "ckpt")
tensorboard_path = os.path.join(base_path, "tensorboard")
for d in [model_path, ckpt_path, tensorboard_path]:
if not os.path.exists(d):
os.mkdir(d)
agent = make_pacman_agent(
name,
model_path,
lr,
discount_factor,
batch_size,
epsilon,
epsilon_decay,
boltzmann,
tau,
double_q,
buffer_capacity,
history_length,
diff_history,
big,
save_hypers=True)
print("... training agent")
# todo? make this better
tensorboard = Evaluation(
os.path.join(tensorboard_path, agent.name + "_train"),
["episode_reward", "NOOP", "up", "right", "left", "down"])
tensorboard_test = Evaluation(
os.path.join(tensorboard_path, agent.name + "_test"),
["episode_reward", "NOOP", "up", "right", "left", "down"])
start_episode = 0
if try_resume:
possible_file = os.path.join(model_path, "epstrained.json")
if os.path.exists(possible_file):
# get the last ep trained; start at the next one
with open(possible_file, "r") as fh:
start_episode = json.load(fh) + 1
#load up model from previous training session
agent.load(os.path.join(model_path, 'ckpt', 'dqn_agent.ckpt'))
# training
for i in range(start_episode, num_episodes):
print("episode: ", i)
max_timesteps = 99999
stats = run_episode(
env,
agent,
max_timesteps=max_timesteps,
deterministic=False,
softmax=False,
do_training=True,
rendering=False,
skip_frames=skip_frames,
history_length=history_length,
diff_history=diff_history)
tensorboard.write_episode_data(
i,
eval_dict={
"episode_reward": stats.episode_reward,
"NOOP": stats.get_action_usage(NOOP),
"up": stats.get_action_usage(UP),
"right": stats.get_action_usage(RIGHT),
"left": stats.get_action_usage(LEFT),
"down": stats.get_action_usage(DOWN)
})
if i % MODEL_TEST_INTERVAL == 0 or i >= (num_episodes - 1):
stats_test = run_episode(
env,
agent,
max_timesteps=max_timesteps,
deterministic=True,
softmax=False,
do_training=False,
rendering=False,
skip_frames=2,
history_length=history_length,
diff_history=diff_history)
tensorboard_test.write_episode_data(
i,
eval_dict={
"episode_reward": stats.episode_reward,
"NOOP": stats.get_action_usage(NOOP),
"up": stats.get_action_usage(UP),
"right": stats.get_action_usage(RIGHT),
"left": stats.get_action_usage(LEFT),
"down": stats.get_action_usage(DOWN)
})
# store model every 100 episodes and in the end.
if i % MODEL_SAVE_INTERVAL == 0 or i >= (num_episodes - 1):
agent.saver.save(agent.sess,
os.path.join(ckpt_path, "dqn_agent.ckpt"))
# write an episode counter, so that we can resume training later
with open(os.path.join(model_path, "epstrained.json"), "w") as fh:
json.dump(i, fh)
tensorboard.close_session()
tensorboard_test.close_session()
def train_model(X_train,
y_train,
X_valid,
y_valid,
n_minibatches,
batch_size,
lr,
model_dir="./models",
tensorboard_dir="./tensorboard",
history_length=1,
arq_file='',
ckpt_file='',
net_name='JABnet'):
# create result and model folders
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
# TODO: specify your neural network in model.py
agent = Model(history_length=history_length,
name=net_name,
learning_rate=lr,
from_file=arq_file)
# Get the history length from the actual model, whether it comes from a file or a parameter
history_length = agent.history_length
if ckpt_file != '':
agent.load(ckpt_file)
model_name = agent.name
tensorboard_dir = tensorboard_dir + '/' + model_name
if not os.path.exists(tensorboard_dir):
os.mkdir(tensorboard_dir)
tensorboard_eval = Evaluation(tensorboard_dir, agent.session)
# 1. write a method sample_minibatch and perform an update step
# 2. compute training/ validation accuracy and loss for the batch and visualize them with tensorboard. You can watch the progress of
# your training in your web browser
#
# training loop
# Initialize Parameters
agent.session.run(agent.init)
# Save Architecture (Only if the network was not loaded from one)
if arq_file == '':
agent.save_arq()
for i in range(n_minibatches):
minibatch_indexes = np.random.randint(history_length - 1,
X_train.shape[0], batch_size)
X_minibatch, y_minibatch = resequence(X_train, y_train, history_length,
minibatch_indexes)
agent.train(X_minibatch, y_minibatch)
if i % 1000 == 0:
loss_train, acc_train = evaluate_model(X_train, y_train, agent)
loss_valid, acc_valid = evaluate_model(X_valid, y_valid, agent)
print("Minibatch: ", i, " Train accuracy: ",
'{:.4f}%'.format(acc_train * 100), " Train Loss: ",
'{:.6f}'.format(loss_train), " | Test accuracy: ",
'{:.4f}%'.format(acc_valid * 100), " Test Loss: ",
'{:.6f}'.format(loss_valid))
# Save intermediate checkpoints in case training crashes or for Early Stop
agent.save(suffix='_' + datetime.now().strftime("%Y%m%d-%H%M%S") +
'_i' + str(i) + '_TrAcc_' +
"{:.4f}".format(acc_train * 100),
dump_architecture=False)
eval_dict = {
'loss': loss_train,
'acc': acc_train,
'vloss': loss_valid,
'vacc': acc_valid
}
tensorboard_eval.write_episode_data(i, eval_dict)
agent.save(dump_architecture=False)
def training(self, X_train, y_train, X_valid, y_valid, epochs, batch_size,
model_dir, tensorboard_dir):
tensorboard_eval = Evaluation(tensorboard_dir)
acc = []
acc_valid = []
train_cost = np.zeros(epochs)
valid_cost = np.zeros(epochs)
correct_prediction = self.y_pred
#accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#accuracy = self.accuracy
#tf.reset_default_graph()
with self.sess as sess:
sess.run(tf.global_variables_initializer())
print('inside session')
start_time = time()
for epoch in range(epochs):
correctsum = 0
#x_train_batchsize = 16
x_train_batchsize = len(X_train) // batch_size
print("X_train_batchsize :::::", x_train_batchsize)
correctsum_val = 0
#x_val_batchsize = 12
x_val_batchsize = len(X_valid) // batch_size
train_cost = np.zeros((epochs))
valid_cost = np.zeros((epochs))
# for Training computation
for i in range(x_train_batchsize):
x_batch = X_train[i * batch_size:(i + 1) * batch_size]
y_batch = y_train[i * batch_size:(i + 1) * batch_size]
y_batch = id_to_action(y_batch)
_, loss = sess.run([self.optimizer, self.cost],
feed_dict={
self.x: x_batch,
self.y_: y_batch
})
train_cost[epoch] += sess.run(self.cost,
feed_dict={
self.x: x_batch,
self.y_: y_batch
})
print('iteration {} , epoch {}, train cost {:.2f}'.format(
i, epoch, train_cost[epoch]))
print('epoch {}, loss {:.2f} %'.format(epoch, loss), end='\n')
# for Validation computation
for i in range(x_val_batchsize):
x_val = X_valid[i * batch_size:(i + 1) * batch_size]
y_val = y_valid[i * batch_size:(i + 1) * batch_size]
#y_val = id_to_action(y_val)
valid_cost[epoch] += self.sess.run(self.cost,
feed_dict={
self.x: x_val,
self.y_: y_val
})
print('valid iteration{}, epoch{}, valid cost ::: {:.2f}'.
format(i, epoch, valid_cost[epoch]))
train_cost[epoch] = train_cost[epoch] / x_train_batchsize
valid_cost[epoch] = valid_cost[epoch] / x_val_batchsize
print(
"[%d/%d]: train_cost: %.4f, valid_cost: %.4f" %
(epoch + 1, epochs, train_cost[epoch], valid_cost[epoch]))
print(
'epoch {},train_cost{:.2f}, validation cost{:.2f}'.format(
epoch, train_cost[epoch], valid_cost[epoch]))
eval_dict = {
"train": train_cost[epoch],
"valid": valid_cost[epoch]
}
tensorboard_eval.write_episode_data(epoch, eval_dict)
# TODO: save your self
self.save(os.path.join(model_dir, "self.ckpt"))
print(model_dir)
print("Model saved in file: %s" % model_dir)
self.sess.close()
return eval_dict
def train_model(X_train,
y_train,
X_valid,
y_valid,
n_minibatches,
batch_size,
lr,
model_dir="./models",
tensorboard_dir="./tensorboard",
history_length=1):
# create result and model folders
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
# TODO: specify your agent with the neural network in agents/bc_agent.py
# agent = BCAgent(...)
agent = BCAgent(history_length)
stats = ['train_acc', 'train_loss', 'valid_acc']
tensorboard_eval = Evaluation(tensorboard_dir,
name="history_5_better_diff",
stats=stats)
#tensorboard_eval = Evaluation(tensorboard_dir)
# TODO: implement the training
#
# 1. write a method sample_minibatch and perform an update step
# 2. compute training/ validation accuracy and loss for the batch and visualize them with tensorboard. You can watch the progress of
# your training *during* the training in your web browser
#
# training loop
# for i in range(n_minibatches):
# ...
# for i % 10 == 0:
# # compute training/ validation accuracy and write it to tensorboard
# tensorboard_eval.write_episode_data(...)
# TODO: save your agent
# model_dir = agent.save(os.path.join(model_dir, "agent.pt"))
# print("Model saved in file: %s" % model_dir)
for i in range(n_minibatches):
#torch.cuda.empty_cache()
X_batch, y_batch = sample_minibatch(X_train, y_train, batch_size)
train_loss = agent.update(X_batch, y_batch)
# TODO validation
# calculate train accuracy
if i % 10 == 0:
y_pred = agent.predict(
torch.tensor(X_batch).permute(0, 3, 1, 2).to(device))
y_pred = y_pred.argmax(1)
y_batch = torch.LongTensor(y_batch).view(-1).to(device)
train_accuracy = (torch.sum(torch.eq(y_pred, y_batch)).item() /
batch_size)
print('train acc: ', train_accuracy)
with torch.no_grad():
y_valid_pred = agent.predict(
torch.tensor(X_valid).permute(0, 3, 1, 2).to(device))
y_valid_pred = y_valid_pred.argmax(1)
y_valid = torch.LongTensor(y_valid).view(-1).to(device)
valid_accuracy = torch.sum(torch.eq(
y_valid_pred, y_valid)).item() / y_valid.shape[0]
print('valid acc: ', valid_accuracy)
eval_dict = {
'train_acc': train_accuracy,
'train_loss': train_loss.item(),
'valid_acc': valid_accuracy,
}
tensorboard_eval.write_episode_data(i, eval_dict)
# save your agent
model_dir = agent.save(os.path.join(model_dir, "history_5_better_diff.pt"))
print("Model saved in file: %s" % model_dir)
def train_model(X_train,
y_train,
X_valid,
y_valid,
n_minibatches,
batch_size,
lr,
history_length=1,
model_dir="./models",
tensorboard_dir="./tensorboard"):
# create result and model folders
if not os.path.exists(model_dir):
os.makedirs(model_dir)
print("... train model")
# TODO: specify your agent with the neural network in agents/bc_agent.py
agent = BCAgent(device='cpu',
history_length=history_length,
lr=lr,
n_classes=5)
# print('Loading model......')
# agent.load('models/agent_20190601-112531.pt')
idx = datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_eval = Evaluation(tensorboard_dir,
name='Imitation - Car Racing',
idx=idx,
stats=['loss', 'train_acc', 'valid_acc'])
# TODO: implement the training
#
# 1. write a method sample_minibatch and perform an update step
# 2. compute training/ validation accuracy and loss for the batch and visualize them with tensorboard.
# You can watch the progress of your training *during* the training in your web browser
# weighted sampling
weights = np.array([0.1, 0.6, 0.6, 0.6, 0.9])
# weights = np.array([1,1,1,1,1])
sample_weights = weights[y_train]
sample_weights = sample_weights / sum(sample_weights)
# sampler = WeightedRandomSampler(sample_weights, len(sample_weights))
# training loop
for e in range(n_minibatches):
X_batch, y_batch = sample_batch(X_train,
y_train,
batch_size=batch_size,
history_length=history_length,
weights=sample_weights)
# run update
loss = agent.update(X_batch, y_batch)
if (e + 1) % 10 == 0:
# compute training/ validation accuracy and write it to tensorboard
# predict on a large sample of data
# evaluate training data
tX, ty = sample_batch(X_train,
y_train,
batch_size=batch_size,
history_length=history_length)
train_pred = agent.predict(tX)
# evaluate validation data
vX, vy = sample_batch(X_valid,
y_valid,
batch_size=batch_size,
history_length=history_length)
valid_pred = agent.predict(vX)
# accuracy
train_acc = metrics.accuracy_score(y_true=ty, y_pred=train_pred)
valid_acc = metrics.accuracy_score(y_true=vy, y_pred=valid_pred)
print(
'Epoch [%d/%d] loss: %.4f train & valid accuracies: %.4f %.4f'
% (e + 1, n_minibatches, loss.item(), train_acc, valid_acc))
tensorboard_eval.write_episode_data(
e, {
'loss': loss.item(),
'train_acc': train_acc,
'valid_acc': valid_acc
})
# TODO: save your agent
model_dir = model_dir + "agent_" + idx + ".pt"
agent.save(model_dir)
print("Model saved in file: %s" % model_dir)
def train_model(X_train,
y_train,
X_valid,
y_valid,
epochs=10,
batch_size=64,
lr=0.0003,
history_length=1,
num_uniform_sample=12000,
num_filters=64,
model_dir="./models",
tensorboard_dir="./tensorboard"):
# create result and model folders
if not os.path.exists(model_dir):
os.mkdir(model_dir)
print("... train model")
# TODO: specify your neural network in model.py
agent = Model(learning_rate=lr,
history_length=history_length,
num_filters=num_filters)
init = tf.global_variables_initializer()
agent.sess.run(init)
tensorboard_eval = Evaluation(tensorboard_dir)
tf.reset_default_graph()
# TODO: implement the training
#
# 1. write a method sample_minibatch and perform an update step
# 2. compute training/ validation accuracy and loss for the batch and visualize them with tensorboard. You can watch the progress of
# your training in your web browser
offset = history_length - 1
# training loop
# init training cost
train_cost = np.zeros((epochs))
valid_cost = np.zeros((epochs))
for epoch in range(epochs):
index = uniform_sampling(X_train[offset:], y_train[offset:],
num_uniform_sample)
total_batch_num = (num_uniform_sample - history_length +
1) // batch_size
total_batch_num_valid = (X_valid.shape[0] - history_length +
1) // batch_size
for b in range(total_batch_num):
# select the batch data
batch_index = index[b * batch_size:(b + 1) * batch_size]
X_batch, y_batch = sample_minibatch(X_train, y_train, batch_index,
history_length)
y_batch = id_to_action(y_batch)
# compute the cost
_, temp_cost = agent.sess.run([agent.optimizer, agent.cost],
feed_dict={
agent.x_input: X_batch,
agent.y_label: y_batch
})
# training cost
for b in range(total_batch_num):
batch_index = index[b * batch_size:(b + 1) * batch_size]
X_batch, y_batch = sample_minibatch(X_train, y_train, batch_index,
history_length)
y_batch = id_to_action(y_batch)
train_cost[epoch] += agent.sess.run(agent.cost,
feed_dict={
agent.x_input: X_batch,
agent.y_label: y_batch
})
# validation cost
for b in range(total_batch_num_valid):
batch_index = np.arange(b * batch_size, (b + 1) * batch_size)
X_valid_batch, y_valid_batch = sample_minibatch(
X_valid, y_valid, batch_index, history_length)
valid_cost[epoch] += agent.sess.run(agent.cost,
feed_dict={
agent.x_input:
X_valid_batch,
agent.y_label:
y_valid_batch
})
train_cost[epoch] = train_cost[epoch] / total_batch_num
valid_cost[epoch] = valid_cost[epoch] / total_batch_num_valid
print("[%d/%d]: train_cost: %.4f, valid_cost: %.4f" %
(epoch + 1, epochs, train_cost[epoch], valid_cost[epoch]))
eval_dict = {"train": train_cost[epoch], "valid": valid_cost[epoch]}
tensorboard_eval.write_episode_data(epoch, eval_dict)
# TODO: save your agent
agent.save(os.path.join(model_dir, "agent.ckpt"))
print("Model saved in file: %s" % model_dir)
agent.sess.close()
return train_cost, valid_cost
