def get_experimental_setup(logger_obj, channels_ids, test_ids, train_ids,
val_ids, name, dataset_name):
path = "archives/mts_archive/"
dataset = Dataset(dataset_name, None, logger_obj)
x_test, y_test, sampling_test = dataset.load(path, test_ids, channels_ids)
x_val, y_val, sampling_val = dataset.load(path, val_ids, channels_ids)
x_train, y_train, sampling_train = dataset.load(path, train_ids,
channels_ids)
x_train = [np.expand_dims(np.array(x), 2) for x in x_train]
x_val = [np.expand_dims(np.array(x), 2) for x in x_val]
x_test = [np.expand_dims(np.array(x), 2) for x in x_test]
input_shapes, nb_classes, y_val, y_train, y_test, y_true = prepare_data(
x_train, y_train, y_val, y_test)
ndft_arr = [get_ndft(x) for x in sampling_test]
if len(input_shapes) != len(ndft_arr):
raise Exception("Different sizes of input_shapes and ndft_arr")
for i in range(len(input_shapes)):
if input_shapes[i][0] < ndft_arr[i]:
raise Exception(
f"Too big ndft, i: {i}, ndft_arr[i]: {ndft_arr[i]}, input_shapes[i][0]: {input_shapes[i][0]}"
)
experimental_setup = ExperimentalSetup(name, x_train, y_train, x_val,
y_val, x_test, y_test, input_shapes,
sampling_val, ndft_arr, nb_classes,
lambda x: 150, get_batch_size)
return experimental_setup
def features_testing(self, c):
total_cols = self.base_cols + self.added_cols
if c not in total_cols:
cols = total_cols + [c]
x_df = self.features_df[cols]
x_df = x_df.dropna()
X_train, X_valid, y_train, y_valid, df_train, df_train_y, df_valid, df_valid_y = \
prepare_data(x_df, self.cutoff, self.s2pred, self.merging, is_features=True)
# model = AdaBoost(self.n_est, self.class_weights)
model = RandomForest(self.n_est, self.class_weights)
# model = MLP()
model.fit(X=X_train, y=y_train)
df_valid_y['predictions'] = model.predict(
df_valid.drop('timestamp', axis=1))
return (df_valid_y, c)
def loop(c):
if c not in base_cols:
cols = base_cols + [c]
x_df = features[cols]
x_df = x_df.dropna()
X_train, X_valid, y_train, y_valid, df_train_y, df_valid, df_valid_y = \
prepare_data(x_df, CUTOFF, s2pred, merging, is_features=True)
model.fit(X=X_train, y=y_train)
df_valid_y['predictions'] = model.predict(
df_valid.drop('timestamp', axis=1))
df_valid_y.to_csv('predictions/' + data_intervals +
'/feature_selection/' + c + '.csv',
index=False)
def __init__(self, args):
super(KITTIDataset, self).__init__(args)
directory_train = '../kitti_data/train_data'
directory_valid = '../kitti_data/valid_data'
for filename in os.listdir(directory_train):
if filename.endswith(".p"):
self.datasets_train_filter[filename[:-2]] = [0, None]
t, _, _, _, _ = prepare_data(args, self, filename[:-2])
self.training_dataset_length += t.shape[0]
else:
continue
for filename in os.listdir(directory_valid):
if filename.endswith(".p"):
self.datasets_validatation_filter[filename[:-2]] = [0, None]
t, _, _, _, _ = prepare_data(args, self, filename[:-2])
self.valid_dataset_length += t.shape[0]
else:
continue
print(self.valid_dataset_length, self.training_dataset_length)
def main():
args = get_args()
prepare_data()
word_vocab_config = {
"<UNK>": 0,
"<PAD>": 1,
"<start>": 2,
"<end>": 3,
"insert_start": "<SOS>",
"insert_end": "<EOS>",
"tokenization": "nltk",
"specials": ["<UNK>", "<PAD>", "<SOS>", "<EOS>"],
"embedding_root": os.path.join(args.app_path, "data", "embedding",
"word"),
"embedding_type": "glove.840B",
"embedding_dim": 300
}
print("Reading Vocab", flush=True)
char_vocab_config = word_vocab_config.copy()
char_vocab_config["embedding_root"] = os.path.join(args.app_path, "data",
"embedding", "char")
char_vocab_config["embedding_type"] = "glove_char.840B"
# TODO: build vocab out of dataset
# build vocab
itos, stoi, wv_vec = read_vocab(word_vocab_config)
itoc, ctoi, cv_vec = read_vocab(char_vocab_config)
char_embedding_config = {
"embedding_weights": cv_vec,
"padding_idx": word_vocab_config["<UNK>"],
"update": args.update_char_embedding,
"bidirectional": args.bidirectional,
"cell_type": "gru",
"output_dim": 300
}
word_embedding_config = {
"embedding_weights": wv_vec,
"padding_idx": word_vocab_config["<UNK>"],
"update": args.update_word_embedding
}
sentence_encoding_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": True,
"dropout": args.dropout,
}
pair_encoding_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": args.bidirectional,
"dropout": args.dropout,
"gated": True,
"mode": "GRU",
"rnn_cell": torch.nn.GRUCell,
"attn_size": args.attention_size,
"residual": args.residual
}
self_matching_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": args.bidirectional,
"dropout": args.dropout,
"gated": True,
"mode": "GRU",
"rnn_cell": torch.nn.GRUCell,
"attn_size": args.attention_size,
"residual": args.residual
}
pointer_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"dropout": args.dropout,
"residual": args.residual,
"rnn_cell": torch.nn.GRUCell
}
print("DEBUG Mode is ", "On" if args.debug else "Off", flush=True)
dev_cache = "./data/cache/SQuAD_dev%s.pkl" % ("_debug"
if args.debug else "")
test_json = args.test_json
test = read_dataset(test_json,
itos,
stoi,
itoc,
ctoi,
dev_cache,
args.debug,
split="dev")
test_dataloader = test.get_dataloader(args.batch_size_dev)
tester = Tester(args, test_dataloader, char_embedding_config,
word_embedding_config, sentence_encoding_config,
pair_encoding_config, self_matching_config, pointer_config)
result = tester.test()
json.dump(result, open('prediction.json', 'w'))
pd.DataFrame([[id, ' '.join([str(j) for j in range(ans[0], ans[1])])]
for id, ans in result.items()],
columns=['id', 'answer']).to_csv('prediction.csv',
index=False)
def main(args, _log):
def data_augmentation(input_image, output_image):
# Data augmentation
input_image, output_image = utils.random_crop(input_image,
output_image,
args["crop_height"],
args["crop_width"])
if args["h_flip"] and random.randint(0, 1):
input_image = cv2.flip(input_image, 1)
output_image = cv2.flip(output_image, 1)
if args["v_flip"] and random.randint(0, 1):
input_image = cv2.flip(input_image, 0)
output_image = cv2.flip(output_image, 0)
if args["brightness"]:
factor = 1.0 + random.uniform(-1.0 * args["brightness"],
args["brightness"])
table = np.array([((i / 255.0) * factor) * 255
for i in np.arange(0, 256)]).astype(np.uint8)
input_image = cv2.LUT(input_image, table)
if args["rotation"]:
angle = random.uniform(-1 * args["rotation"], args["rotation"])
if args["rotation"]:
M = cv2.getRotationMatrix2D(
(input_image.shape[1] // 2, input_image.shape[0] // 2), angle,
1.0)
input_image = cv2.warpAffine(
input_image,
M, (input_image.shape[1], input_image.shape[0]),
flags=cv2.INTER_NEAREST)
output_image = cv2.warpAffine(
output_image,
M, (output_image.shape[1], output_image.shape[0]),
flags=cv2.INTER_NEAREST)
return input_image, output_image
print("Args:", args)
# Get the names of the classes so we can record the evaluation results
class_names_list, label_values = helpers.get_label_info(
os.path.join(args["dataset"], "class_dict.csv"))
class_names_string = ""
for class_name in class_names_list:
if not class_name == class_names_list[-1]:
class_names_string = class_names_string + class_name + ", "
else:
class_names_string = class_names_string + class_name
num_classes = len(label_values)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# Compute your softmax cross entropy loss
net_input = tf.placeholder(tf.float32, shape=[None, None, None, 3])
net_output = tf.placeholder(tf.float32,
shape=[None, None, None, num_classes])
network, init_fn = model_builder.build_model(
model_name=args["model"],
frontend=args["frontend"],
net_input=net_input,
num_classes=num_classes,
crop_width=args["crop_width"],
crop_height=args["crop_height"],
is_training=True)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=network,
labels=net_output))
opt = tf.train.RMSPropOptimizer(
learning_rate=0.0001, decay=0.995).minimize(
loss, var_list=[var for var in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=1000)
sess.run(tf.global_variables_initializer())
utils.count_params()
# If a pre-trained ResNet is required, load the weights.
# This must be done AFTER the variables are initialized with sess.run(tf.global_variables_initializer())
if init_fn is not None:
init_fn(sess)
# Load a previous checkpoint if desired
model_checkpoint_name = "checkpoints/latest_model_" + args[
"model"] + "_" + basename(normpath(args["dataset"])) + ".ckpt"
if args["continue_training"]:
_log.info('Loaded latest model checkpoint')
saver.restore(sess, model_checkpoint_name)
# Load the data
_log.info("Loading the data ...")
train_input_names, train_output_names, val_input_names, val_output_names, test_input_names, test_output_names = utils.prepare_data(
dataset_dir=args["dataset"])
_log.info("\n***** Begin training *****")
_log.debug("Dataset -->", args["dataset"])
_log.debug("Model -->", args["model"])
_log.debug("Crop Height -->", args["crop_height"])
_log.debug("Crop Width -->", args["crop_width"])
_log.debug("Num Epochs -->", args["num_epochs"])
_log.debug("Batch Size -->", args["batch_size"])
_log.debug("Num Classes -->", num_classes)
_log.debug("Data Augmentation:")
_log.debug("\tVertical Flip -->", args["v_flip"])
_log.debug("\tHorizontal Flip -->", args["h_flip"])
_log.debug("\tBrightness Alteration -->", args["brightness"])
_log.debug("\tRotation -->", args["rotation"])
avg_loss_per_epoch = []
avg_scores_per_epoch = []
avg_iou_per_epoch = []
# Which validation images do we want
val_indices = []
num_vals = min(args["num_val_images"], len(val_input_names))
# Set random seed to make sure models are validated on the same validation images.
# So you can compare the results of different models more intuitively.
random.seed(16)
val_indices = random.sample(range(0, len(val_input_names)), num_vals)
# Do the training here
for epoch in range(args["epoch_start_i"], args["num_epochs"]):
current_losses = []
cnt = 0
# Equivalent to shuffling
id_list = np.random.permutation(len(train_input_names))
num_iters = int(np.floor(len(id_list) / args["batch_size"]))
st = time.time()
epoch_st = time.time()
for i in range(num_iters):
# st=time.time()
input_image_batch = []
output_image_batch = []
# Collect a batch of images
for j in range(args["batch_size"]):
index = i * args["batch_size"] + j
id = id_list[index]
input_image = utils.load_image(train_input_names[id],
args["crop_width"],
args["crop_height"])
output_image = utils.load_image(train_output_names[id],
args["crop_width"],
args["crop_height"])
with tf.device('/cpu:0'):
input_image, output_image = data_augmentation(
input_image, output_image)
# Prep the data. Make sure the labels are in one-hot format
input_image = np.float32(input_image) / 255.0
output_image = np.float32(
helpers.one_hot_it(label=output_image,
label_values=label_values))
input_image_batch.append(
np.expand_dims(input_image, axis=0))
output_image_batch.append(
np.expand_dims(output_image, axis=0))
if args["batch_size"] == 1:
input_image_batch = input_image_batch[0]
output_image_batch = output_image_batch[0]
else:
input_image_batch = np.squeeze(
np.stack(input_image_batch, axis=1))
output_image_batch = np.squeeze(
np.stack(output_image_batch, axis=1))
# Do the training
_, current = sess.run([opt, loss],
feed_dict={
net_input: input_image_batch,
net_output: output_image_batch
})
current_losses.append(current)
cnt = cnt + args["batch_size"]
if cnt % 20 == 0:
string_print = "Epoch = %d Count = %d Iter:(%d of %d) Current_Loss = %.4f Time = %.2f" % (
epoch, cnt, i, num_iters, current, time.time() - st)
utils.LOG(string_print)
st = time.time()
mean_loss = np.mean(current_losses)
avg_loss_per_epoch.append(mean_loss)
# Create directories if needed
if not os.path.isdir("%s/%04d" % ("checkpoints", epoch)):
os.makedirs("%s/%04d" % ("checkpoints", epoch))
# Save latest checkpoint to same file name
_log.info("Saving latest checkpoint")
saver.save(sess, model_checkpoint_name)
if val_indices != 0 and epoch % args["checkpoint_step"] == 0:
_log.info("Saving checkpoint for this epoch")
saver.save(sess, "%s/%04d/model.ckpt" % ("checkpoints", epoch))
if epoch % args["validation_step"] == 0:
_log.info("Performing validation")
target_path = "%s/%04d/val_scores.csv" % ("checkpoints", epoch)
target = open(target_path, 'w')
target.write(
"val_name, avg_accuracy, precision, recall, f1 score, mean iou, %s\n"
% (class_names_string))
scores_list = []
class_scores_list = []
precision_list = []
recall_list = []
f1_list = []
iou_list = []
# Do the validation on a small set of validation images
for ind in val_indices:
input_image = np.expand_dims(np.float32(
utils.load_image(val_input_names[ind], args["crop_width"],
args["crop_height"])
[:args["crop_height"], :args["crop_width"]]),
axis=0) / 255.0
gt = utils.load_image(
val_output_names[ind], args["crop_width"],
args["crop_height"]
)[:args["crop_height"], :args["crop_width"]]
gt = helpers.reverse_one_hot(
helpers.one_hot_it(gt, label_values))
# st = time.time()
output_image = sess.run(network,
feed_dict={net_input: input_image})
output_image = np.array(output_image[0, :, :, :])
output_image = helpers.reverse_one_hot(output_image)
out_vis_image = helpers.colour_code_segmentation(
output_image, label_values)
accuracy, class_accuracies, prec, rec, f1, iou = utils.evaluate_segmentation(
pred=output_image, label=gt, num_classes=num_classes)
file_name = utils.filepath_to_name(val_input_names[ind])
target.write("%s, %f, %f, %f, %f, %f" %
(file_name, accuracy, prec, rec, f1, iou))
for item in class_accuracies:
target.write(", %f" % (item))
target.write("\n")
scores_list.append(accuracy)
class_scores_list.append(class_accuracies)
precision_list.append(prec)
recall_list.append(rec)
f1_list.append(f1)
iou_list.append(iou)
gt = helpers.colour_code_segmentation(gt, label_values)
file_name = os.path.basename(val_input_names[ind])
file_name = os.path.splitext(file_name)[0]
pred_img_path = "%s/%04d/%s_pred.png" % ("checkpoints", epoch,
file_name)
gt_img_path = "%s/%04d/%s_gt.png" % ("checkpoints", epoch,
file_name)
cv2.imwrite(
pred_img_path,
cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR))
cv2.imwrite(gt_img_path,
cv2.cvtColor(np.uint8(gt), cv2.COLOR_RGB2BGR))
# Send the first 16 images to sacred
if ind in val_indices[:16]:
ex.add_artifact(gt_img_path, "GtImage_%d" % ind)
ex.add_artifact(pred_img_path, "PredImage_%d" % ind)
target.close()
ex.add_artifact(target_path)
avg_score = np.mean(scores_list)
class_avg_scores = np.mean(class_scores_list, axis=0)
avg_scores_per_epoch.append(avg_score)
avg_precision = np.mean(precision_list)
avg_recall = np.mean(recall_list)
avg_f1 = np.mean(f1_list)
avg_iou = np.mean(iou_list)
avg_iou_per_epoch.append(avg_iou)
# Sacred info dict gets sent every heartbeat (10s)
ex.info["avg_score"] = avg_score
ex.info["class_avg_scores"] = class_avg_scores
ex.info["avg_precision"] = avg_precision
ex.info["avg_recall"] = avg_recall
ex.info["avg_f1"] = avg_f1
ex.info["avg_iou"] = avg_iou
_log.debug("\nAverage validation accuracy for epoch # %04d = %f" %
(epoch, avg_score))
_log.debug(
"Average per class validation accuracies for epoch # %04d:" %
(epoch))
for index, item in enumerate(class_avg_scores):
_log.debug("%s = %f" % (class_names_list[index], item))
_log.debug("Validation precision = ", avg_precision)
_log.debug("Validation recall = ", avg_recall)
_log.debug("Validation F1 score = ", avg_f1)
_log.debug("Validation IoU score = ", avg_iou)
epoch_time = time.time() - epoch_st
remain_time = epoch_time * (args["num_epochs"] - 1 - epoch)
m, s = divmod(remain_time, 60)
h, m = divmod(m, 60)
if s != 0:
train_time = "Remaining training time = %d hours %d minutes %d seconds\n" % (
h, m, s)
else:
train_time = "Remaining training time : Training completed.\n"
utils.LOG(train_time)
scores_list = []
fig1, ax1 = plt.subplots(figsize=(11, 8))
ax1.plot(range(epoch + 1), avg_scores_per_epoch)
ax1.set_title("Average validation accuracy vs epochs")
ax1.set_xlabel("Epoch")
ax1.set_ylabel("Avg. val. accuracy")
plt.savefig('accuracy_vs_epochs.png')
ex.add_artifact("accuracy_vs_epochs.png")
plt.clf()
fig2, ax2 = plt.subplots(figsize=(11, 8))
ax2.plot(range(epoch + 1), avg_loss_per_epoch)
ax2.set_title("Average loss vs epochs")
ax2.set_xlabel("Epoch")
ax2.set_ylabel("Current loss")
plt.savefig('loss_vs_epochs.png')
ex.add_artifact("loss_vs_epochs.png")
plt.clf()
fig3, ax3 = plt.subplots(figsize=(11, 8))
ax3.plot(range(epoch + 1), avg_iou_per_epoch)
ax3.set_title("Average IoU vs epochs")
ax3.set_xlabel("Epoch")
ax3.set_ylabel("Current IoU")
plt.savefig('iou_vs_epochs.png')
ex.add_artifact("iou_vs_epochs.png")
from utils.utils import process_image, deprocess_image
from utils.utils import read_and_generate_heatmap, prepare_data,evaluate_distribution_accuracy
from models import create_model
max_features = 20000
maxlen=100
EMBEDDING_DIM = 300
use_distribution = True
use_semantics = False
use_comments=True
use_multigap=True
# X_train, Y_train, X_test, Y_test = prepare_data(use_distribution=use_distribution)
X_train, Y_train,X_test, Y_test,X_train_text, X_test_text,embedding_layer =
prepare_data(use_distribution=use_distribution, use_semantics=use_semantics, use_comments=use_comments)
# X_train, Y_train,X_test, Y_test= prepare_data(use_distribution=use_distribution, use_semantics=False)
# X_train, Y_train, Y_train_semantics, X_test, Y_test, Y_test_semantics, X_train_text, X_test_text, embedding_layer = prepare_data(use_distribution=use_distribution, use_semantics=use_semantics, use_comments=use_comments)
## Without image data
# _, Y_train,_, Y_test,X_train_text, X_test_text,embedding_layer = prepare_data(use_distribution=use_distribution, use_semantics=use_semantics, use_comments=use_comments, imageDataAvailable=False)
# BEST MODEL
model = create_model('weights/2017-01-25 22_56_09 - distribution_2layergru_extra_conv_layer.h5',
use_distribution=use_distribution, use_semantics=use_semantics,use_multigap=use_multigap,use_comments=use_comments,
embedding_layer=embedding_layer,extra_conv_layer=True,textInputMaxLength=maxlen,embedding_dim=EMBEDDING_DIM)
# model = create_model('weights/googlenet_aesthetics_weights.h5',
def standardization_all_data(args, dataset):
nb_data = 0
nb_data_dt = 0
#---------------------Compute X - mean factor----------------------------------------------------------------
for i in range(0, len(dataset)):
#---------------------Load the track----------------------------------------------------------------
dataset_name = dataset.dataset_name(i)
print(dataset_name)
t, ang_gt, _, v_gt, u = prepare_data(args, dataset, dataset_name)
dt = (t[1:] - t[:-1]).unsqueeze(1)
nb_data += u.shape[0]
nb_data_dt += dt.shape[0]
ang_gt = correct_ang_gt(ang_gt)
#---------------------Smoothing ground truth quaternion--------------------------------------------
quat_tensor = torch.zeros(u.shape[0], 4)
for j in range(0, quat_tensor.shape[0]):
quat_tensor[j, :] = euler_to_quaternion(ang_gt[j, 0], ang_gt[j, 1],
ang_gt[j, 2])
quat_tensor[2:-2, :] = (quat_tensor[0:-4, :] + quat_tensor[1:-3, :] +
quat_tensor[2:-2, :] + quat_tensor[3:-1, :] +
quat_tensor[4:, :]) / 5
quat_tensor = torch.div(
quat_tensor,
torch.norm(quat_tensor, dim=1).unsqueeze(1).repeat(1, 4))
if i == 0:
u_input_loc = torch.sum(u[:], dim=0)
v_gt_input_loc = torch.sum(v_gt[:], dim=0)
quat_tensor_loc = torch.sum(quat_tensor[:, 0:4], dim=0)
dt_loc = torch.sum(dt, dim=0)
else:
u_input_loc += torch.sum(u[:], dim=0)
v_gt_input_loc += torch.sum(v_gt[:], dim=0)
quat_tensor_loc += torch.sum(quat_tensor[:, 0:4], dim=0)
dt_loc += torch.sum(dt, dim=0)
u_input_loc = u_input_loc / nb_data
v_gt_input_loc = v_gt_input_loc / nb_data
quat_tensor_loc = quat_tensor_loc / nb_data
dt_loc = dt_loc / nb_data_dt
print(u_input_loc.shape, v_gt_input_loc.shape, quat_tensor_loc.shape,
dt_loc.shape)
input_loc = torch.cat(
(u_input_loc, v_gt_input_loc, quat_tensor_loc, dt_loc), dim=0)
#---------------------Compute the standard deviation factor----------------------------------------------------------------
for i in range(0, len(dataset)):
#---------------------Load the track----------------------------------------------------------------
dataset_name = dataset.dataset_name(i)
print(dataset_name)
t, ang_gt, p_gt, v_gt, u = prepare_data(args, dataset, dataset_name)
dt = (t[1:] - t[:-1]).unsqueeze(1)
ang_gt = correct_ang_gt(ang_gt)
#---------------------Smoothing ground truth quaternion--------------------------------------------
quat_tensor = torch.zeros(u.shape[0], 4)
for j in range(0, quat_tensor.shape[0]):
quat_tensor[j, :] = euler_to_quaternion(ang_gt[j, 0], ang_gt[j, 1],
ang_gt[j, 2])
quat_tensor[2:-2, :] = (quat_tensor[0:-4, :] + quat_tensor[1:-3, :] +
quat_tensor[2:-2, :] + quat_tensor[3:-1, :] +
quat_tensor[4:, :]) / 5
quat_tensor = torch.div(
quat_tensor,
torch.norm(quat_tensor, dim=1).unsqueeze(1).repeat(1, 4))
if i == 0:
u_input_std = ((u[:] - input_loc[0:6])**2).sum(dim=0)
v_gt_std = ((v_gt[:] - input_loc[6:9])**2).sum(dim=0)
quat_tensor_std = ((quat_tensor[:, 0:4] -
input_loc[9:13])**2).sum(dim=0)
dt_std = ((dt - input_loc[13])**2).sum(dim=0)
else:
u_input_std += ((u[:] - input_loc[0:6])**2).sum(dim=0)
v_gt_std += ((v_gt[:] - input_loc[6:9])**2).sum(dim=0)
quat_tensor_std += ((quat_tensor[:, 0:4] -
input_loc[9:13])**2).sum(dim=0)
dt_std += ((dt - input_loc[13])**2).sum(dim=0)
u_input_std = (u_input_std / nb_data).sqrt()
v_gt_std = (v_gt_std / nb_data).sqrt()
quat_tensor_std = (quat_tensor_std / nb_data).sqrt()
dt_std = (dt_std / nb_data_dt).sqrt()
#---------------------Saving the factors in dictionary--------------------------------------------
input_std = torch.cat((u_input_std, v_gt_std, quat_tensor_std, dt_std),
dim=0)
mondict = {'input_loc': input_loc, 'input_std': input_std}
path_data = args.path_standardization_factor
torch.save(mondict, path_data)
return
def create_dataset_Relative_Kinematic(args, dataset, windows_size):
#---------------------Initialization of the tensors--------------------------------------------
number_input = 14
training_data = torch.zeros(
(dataset.training_dataset_length, windows_size, number_input))
training_label_data = torch.zeros(
(dataset.training_dataset_length, windows_size, 7))
valid_data = torch.zeros(
(dataset.valid_dataset_length, windows_size, number_input))
valid_label_data = torch.zeros(
(dataset.valid_dataset_length, windows_size, 7))
training_index = 0
valid_index = 0
training_dataset = dataset.datasets_train_filter
for i in range(0, len(dataset)):
dataset_name = dataset.dataset_name(i)
print(dataset_name)
#---------------------load the track--------------------------------------------
t, ang_gt, _, v_gt, u = prepare_data(args, dataset, dataset_name)
if (not os.path.exists(args.path_normalization_factor)):
normalization_all_data(args, dataset)
if (not os.path.exists(args.path_standardization_factor)):
standardization_all_data(args, dataset)
dt = (t[1:] - t[:-1]).float()
d_vgt = v_gt[1:] - v_gt[:-1]
ang_gt = correct_ang_gt(ang_gt)
#---------------------Smoothing ground truth quaternion--------------------------------------------
quat_tensor = torch.zeros(u.shape[0], 4)
for j in range(0, quat_tensor.shape[0]):
quat_tensor[j, :] = euler_to_quaternion(ang_gt[j, 0], ang_gt[j, 1],
ang_gt[j, 2])
quat_tensor[2:-2, :] = (quat_tensor[0:-4, :] + quat_tensor[1:-3, :] +
quat_tensor[2:-2, :] + quat_tensor[3:-1, :] +
quat_tensor[4:, :]) / 5
quat_tensor = torch.div(
quat_tensor,
torch.norm(quat_tensor, dim=1).unsqueeze(1).repeat(1, 4))
relative_quaternion = relative_rotation(quat_tensor)
relative_quaternion = torch.div(
relative_quaternion,
torch.norm(relative_quaternion, dim=1).unsqueeze(1).repeat(1, 4))
#---------------------Attribution of the values for the training tensor and testing tensor--------------------------------------------
if (dataset_name in training_dataset):
for j in range(windows_size, u.shape[0]):
index = j - windows_size
training_data[training_index, :, 0:3] = u[index:j, 0:3]
training_data[training_index, :, 3:6] = u[index:j, 3:6]
training_data[training_index, :, 6:9] = v_gt[index:j, 0:3]
training_data[training_index, :, 9:13] = quat_tensor[index:j,
0:4]
training_data[training_index, :, 13] = dt[index:j]
training_label_data[training_index, :, 0:3] = d_vgt[index:j]
training_label_data[training_index, :,
3:7] = relative_quaternion[index:j, 0:4]
training_index += 1
else:
for j in range(windows_size, u.shape[0]):
index = j - windows_size
valid_data[valid_index, :, 0:3] = u[index:j, 0:3]
valid_data[valid_index, :, 3:6] = u[index:j, 3:6]
valid_data[valid_index, :, 6:9] = v_gt[index:j, 0:3]
valid_data[valid_index, :, 9:13] = quat_tensor[index:j, 0:4]
valid_data[valid_index, :, 13] = dt[index:j]
valid_label_data[valid_index, :, 0:3] = d_vgt[index:j]
valid_label_data[valid_index, :,
3:7] = relative_quaternion[index:j, 0:4]
valid_index += 1
#---------------------Adjusting the size of the tensor and shuffling--------------------------------------------
training_data = training_data[:training_index]
training_label_data = training_label_data[:training_index]
valid_data = valid_data[:valid_index]
valid_label_data = valid_label_data[:valid_index]
randomize = shuffle_tensor(training_data.shape[0])
training_data = training_data[randomize]
training_label_data = training_label_data[randomize]
randomize = shuffle_tensor(valid_data.shape[0])
valid_data = valid_data[randomize]
valid_label_data = valid_label_data[randomize]
mondict = {
'training_data': training_data,
'training_label_data': training_label_data,
'valid_data': valid_data,
'valid_label_data': valid_label_data
}
path_data = '../Relative_kinematic_training.p'
torch.save(mondict, path_data)
return training_data, training_label_data, valid_data, valid_label_data
def normalization_all_data(args, dataset):
#---------------------Initialization of the normalization factors----------------------------------------------------------------
nb_data = 0
max_quat = [0, 0, 0, 0] #We initialize with a random low values
min_quat = [20, 20, 20, 20] #We initialize with a random high values
max_gyr = [0, 0, 0]
min_gyr = [20, 20, 20]
max_acc = [0, 0, 0]
min_acc = [20, 20, 20]
max_gyr_dt = [0, 0, 0]
min_gyr_dt = [20, 20, 20]
max_acc_dt = [0, 0, 0]
min_acc_dt = [20, 20, 20]
max_acc_rot = [0, 0, 0]
min_acc_rot = [20, 20, 20]
max_prev_vel = 0
min_prev_vel = 20
max_velocity = [0, 0, 0]
min_velocity = [20, 20, 20]
for i in range(0, len(dataset)):
#---------------------Load the track----------------------------------------------------------------
dataset_name = dataset.dataset_name(i)
print(dataset_name)
t, ang_gt, p_gt, v_gt, u = prepare_data(args, dataset, dataset_name)
dt = (t[1:] - t[:-1]).unsqueeze(1)
nb_data += (u.shape[0] - 1)
u_dt = u[1:, 0:6].mul(dt[:])
norm_v_gt = torch.norm(v_gt, dim=1)
ang_gt = correct_ang_gt(ang_gt)
#---------------------Smoothing ground truth quaternion--------------------------------------------
quat_tensor = torch.zeros(u.shape[0], 4)
for j in range(0, quat_tensor.shape[0]):
quat_tensor[j, :] = euler_to_quaternion(ang_gt[j, 0], ang_gt[j, 1],
ang_gt[j, 2])
quat_tensor[2:-2, :] = (quat_tensor[0:-4, :] + quat_tensor[1:-3, :] +
quat_tensor[2:-2, :] + quat_tensor[3:-1, :] +
quat_tensor[4:, :]) / 5
quat_tensor = torch.div(
quat_tensor,
torch.norm(quat_tensor, dim=1).unsqueeze(1).repeat(1, 4))
rot_matrix = as_rotation_matrix(quat_tensor[:-1])
macc = mult_acc(rot_matrix, u[1:, 3:6])
#---------------------Get min-max values--------------------------------------------
velocity_max = [v_gt[:, 0].max(), v_gt[:, 1].max(), v_gt[:, 2].max()]
velocity_min = [v_gt[:, 0].min(), v_gt[:, 1].min(), v_gt[:, 2].min()]
print(velocity_max)
print(velocity_min)
u_values_max = [
u[:, 0].max(), u[:, 1].max(), u[:, 2].max(), u[:, 3].max(),
u[:, 4].max(), u[:, 5].max()
]
u_values_min = [
u[:, 0].min(), u[:, 1].min(), u[:, 2].min(), u[:, 3].min(),
u[:, 4].min(), u[:, 5].min()
]
quat_tensor_max = [
quat_tensor[:, 0].max(), quat_tensor[:, 1].max(),
quat_tensor[:, 2].max(), quat_tensor[:, 3].max()
]
quat_tensor_min = [
quat_tensor[:, 0].min(), quat_tensor[:, 1].min(),
quat_tensor[:, 2].min(), quat_tensor[:, 3].min()
]
u_dt_values_max = [
u_dt[:, 0].max(), u_dt[:, 1].max(), u_dt[:, 2].max(),
u_dt[:, 3].max(), u_dt[:, 4].max(), u_dt[:, 5].max()
]
u_dt_values_min = [
u_dt[:, 0].min(), u_dt[:, 1].min(), u_dt[:, 2].min(),
u_dt[:, 3].min(), u_dt[:, 4].min(), u_dt[:, 5].min()
]
vel_max = norm_v_gt.max()
vel_min = norm_v_gt.min()
rotacc_values_max = [
macc[:, 0].max(), macc[:, 1].max(), macc[:, 2].max()
]
rotacc_values_min = [
macc[:, 0].min(), macc[:, 1].min(), macc[:, 2].min()
]
#print(vel_min,vel_max)
#---------------------Update the min and max values--------------------------------------------
if vel_min < min_prev_vel:
min_prev_vel = vel_min
if vel_max > max_prev_vel:
max_prev_vel = vel_max
if quat_tensor_min[3] < min_quat[3]:
min_quat[3] = quat_tensor_min[3]
if quat_tensor_max[3] > max_quat[3]:
max_quat[3] = quat_tensor_max[3]
for i in range(3):
if u_values_min[i] < min_gyr[i]:
min_gyr[i] = u_values_min[i]
if u_values_max[i] > max_gyr[i]:
max_gyr[i] = u_values_max[i]
if u_values_min[3 + i] < min_acc[i]:
min_acc[i] = u_values_min[3 + i]
if u_values_max[3 + i] > max_acc[i]:
max_acc[i] = u_values_max[3 + i]
if rotacc_values_min[i] < min_acc_rot[i]:
min_acc_rot[i] = rotacc_values_min[i]
if rotacc_values_max[i] > max_acc_rot[i]:
max_acc_rot[i] = rotacc_values_max[i]
if u_dt_values_min[i] < min_gyr_dt[i]:
min_gyr_dt[i] = u_dt_values_min[i]
if u_dt_values_max[i] > max_gyr_dt[i]:
max_gyr_dt[i] = u_dt_values_max[i]
if u_dt_values_min[3 + i] < min_acc_dt[i]:
min_acc_dt[i] = u_dt_values_min[3 + i]
if u_dt_values_max[3 + i] > max_acc_dt[i]:
max_acc_dt[i] = u_dt_values_max[3 + i]
if quat_tensor_min[i] < min_quat[i]:
min_quat[i] = quat_tensor_min[i]
if quat_tensor_max[i] > max_quat[i]:
max_quat[i] = quat_tensor_max[i]
if velocity_min[i] < min_velocity[i]:
min_velocity[i] = velocity_min[i]
if velocity_max[i] > max_velocity[i]:
max_velocity[i] = velocity_max[i]
#---------------------Saving the factors in dictionary--------------------------------------------
mondict = {
'max_quat': max_quat,
'min_quat': min_quat,
'max_gyr': max_gyr,
'min_gyr': min_gyr,
'max_acc': max_acc,
'min_acc': min_acc,
'max_gyr_dt': max_gyr_dt,
'min_gyr_dt': min_gyr_dt,
'max_prev_vel': max_prev_vel,
'min_prev_vel': min_prev_vel,
'max_acc_rot': max_acc_rot,
'min_acc_rot': min_acc_rot,
'max_acc_dt': max_acc_dt,
'min_acc_dt': min_acc_dt,
'min_velocity': min_velocity,
'max_velocity': max_velocity
}
path_data = args.path_normalization_factor
torch.save(mondict, path_data)
print("Test")
print(max_velocity, min_velocity)
return
(MOUNTH_DATA_ROWS /
2)):int(0.9 * l)]
n_est = 0
if 'AdaBoost' in model_name:
n_est = int(model_name.replace('AdaBoost_', ''))
if 'RandomForest' in model_name:
n_est = int(model_name.replace('RandomForest_', ''))
feature_selector = Selector(model, df_to_selector, CUTOFF, s2pred,
merging, n_est, class_weight)
# l = ['soft_upper_dist', 'soft_lower_dist', 'soft_upper_broke', 'soft_lower_broke']
# l = ['NEOUSDT_' + i for i in l]
feature_selector.execute()
cross_data = cross_data[feature_selector.get_cols()]
X_train, X_valid, y_train, y_valid, df_train, df_train_y, df_valid, df_valid_y = \
prepare_data(cross_data, CUTOFF, s2pred, merging, is_features=is_features)
print('d')
if not is_keras:
model.fit(X=X_train, y=y_train)
else:
# model.fit(X=X_train, y=y_train, epochs=EPOCHS)
model.fit(X=X_train,
y=np.array(df_train_y['y']),
epochs=EPOCHS)
df_valid_y['predictions'] = model.predict(
df_valid.drop('timestamp', axis=1))
df_train_y['predictions'] = model.predict(
df_train.drop('timestamp', axis=1))
if not is_keras:
f_imp = [None] + list(
utils.count_params()
# If a pre-trained ResNet is required, load the weights.
# This must be done AFTER the variables are initialized with sess.run(tf.global_variables_initializer())
if init_fn is not None:
init_fn(sess)
# Load a previous checkpoint if desired
model_checkpoint_name = "model/latest_model_" + args.model + "_" + args.dataset + ".ckpt"
if args.continue_training:
print('Loaded latest model checkpoint')
saver.restore(sess, model_checkpoint_name)
# Load the data
print("Loading the data ...")
train_input_names, train_output_names, val_input_names, val_output_names, test_input_names, test_output_names = utils.prepare_data(
dataset_dir=args.dataset)
print("\n***** Begin training *****")
print("Dataset -->", args.dataset)
print("Model -->", args.model)
print("Crop Height -->", args.crop_height)
print("Crop Width -->", args.crop_width)
print("Num Epochs -->", args.num_epochs)
print("Batch Size -->", args.batch_size)
print("Num Classes -->", num_classes)
print("Data Augmentation:")
print("\tVertical Flip -->", args.v_flip)
print("\tHorizontal Flip -->", args.h_flip)
print("\tBrightness Alteration -->", args.brightness)
print("\tRotation -->", args.rotation)
def _main():
# parse command line arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'--train_from_checkpoint',
type=str,
help=
"The path to where a previously trained model's weights are stored. To use the default\n\
coco weights, use the path 'model_weights/coco_pretrained_weights.ckpt'. Otherwise, the model\n\
weights will be initialized randomly. ")
parser.add_argument(
'--class_path',
default='utils/coco_classes.txt',
type=str,
help=
'The path that points towards where the class names for the dataset are stored.\n\
The default path is "utils/coco_classes.txt".')
parser.add_argument(
'--anchors_path',
default='utils/anchors.txt',
type=str,
help=
'The path that points towards where the anchor values for the model are stored.\n\
The default path is "utils/anchors.txt", which contains anchors trained on the coco dataset.'
)
parser.add_argument(
'--data_path',
default='training_data/image_paths_and_box_info.txt',
type=str,
help=
'The path that points towards where the training data text file is stored.\n\
The default path is "training_data/image_paths_and_box_info.txt".')
parser.add_argument(
'--input_height',
default=416,
type=int,
help=
'The input height of the yolov3 model. The height must be a multiple of 32.\n\
The default height is 416.')
parser.add_argument(
'--input_width',
default=416,
type=int,
help=
'The input width of the yolov3 model. The width must be a mutliple of 32.\n\
The default width is 416.')
parser.add_argument(
'--batch_size',
default=32,
type=int,
help=
'The training batch size, whose default value is set to 32 images per batch.'
)
parser.add_argument(
'--max_num_boxes_per_image',
default=20,
type=int,
help=
'The max number of boxes that can be detected within one image. Default is 20.'
)
parser.add_argument(
'--num_training_epochs',
default=150,
type=int,
help='The number of training epochs. The default is 150.')
parser.add_argument(
'--learning_rate',
default=0.001,
type=float,
help='The learning rate of the model. The default is 0.001.')
parser.add_argument(
'--ignore_threshold',
default=0.5,
type=float,
help=
'Impacts how the loss is calculated. Must be between zero and one, and the default is set to 0.5.'
)
parser.add_argument(
'--train_val_data_split',
default=0.9,
type=float,
help=
'The split between the data that will be used for training and data that will be used\n\
for validation. Default value is 0.9.')
parser.add_argument(
'--train_save_path',
default='model_weights/',
help=
"The training model's checkpoint save path. The default path is 'model_weights/'."
)
parser.add_argument(
'--model_name',
default='model.ckpt',
help=
'The name that should be given to the checkpoint file. The default name is "model.ckpt".'
)
parser.add_argument(
'--tensorboard_save_path',
default='tensorboard/tensorboard_train/',
help=
'The path where the event files to be used with tensorboard will be saved at. The default\n\
path is "tensorboard/tensorboard_train/".')
parser.add_argument(
'--test_model_overfit',
nargs='?',
default=False,
type=str2bool,
const=True,
help=
'Whether or not to purposefully overfit the model by training it on only one image.\n\
This option is useful in testing out if the loss function is working correctly.'
)
parser.add_argument(
'--save_every_x_iterations',
default=100,
type=int,
help=
"How frequently the model's training weights are saved. The default value is every\n\
100 iterations.")
parser.add_argument(
'--log_every_x_iterations',
default=5,
type=int,
help=
"How frequently the model's loss is logged for it to be inspected in Tensorboard.\n\
The default value is every 5 iterations.")
args = vars(parser.parse_args())
args[
'data_path'] = '/home/yl/CNN/Yolo/keras-yolo3-fine-tune/dataset/train_label.txt'
# read inputs
h = args['input_height']
w = args['input_width']
ignore_thresh = args['ignore_threshold']
max_num_boxes_per_image = args['max_num_boxes_per_image']
anchors = get_anchors(args['anchors_path'])
lr = args['learning_rate']
num_anchors_per_detector = len(anchors) // 3
num_detectors_per_image = num_anchors_per_detector * (((h / 32) *
(w / 32)) +
((h / 16) *
(w / 16)) +
((h / 8) * (w / 8)))
class_names = get_classes(args['class_path'])
num_classes = len(class_names)
tb_train_path = args['tensorboard_save_path'] + 'train/'
tb_val_path = args['tensorboard_save_path'] + 'val/'
training_data, validation_data, batch_size = prepare_data(
args['data_path'], args['train_val_data_split'], args['batch_size'],
args['test_model_overfit'])
tf.reset_default_graph()
# build graph
with tf.variable_scope('y_true'):
y_true_data = tf.placeholder(
dtype=tf.float32,
shape=[None, num_detectors_per_image, num_classes + 5])
with tf.variable_scope('y_true_boxes'):
y_true_box_data = tf.placeholder(dtype=tf.float32,
shape=[
None, max_num_boxes_per_image *
num_anchors_per_detector, 4
])
with tf.variable_scope('x_input'):
X = tf.placeholder(dtype=tf.float32, shape=[None, h, w, 3])
yolo_outputs = yolo_v3(inputs=X,
num_classes=len(class_names),
anchors=anchors,
h=h,
w=w,
training=True) # output
loss = yolo_v3_loss(yolo_outputs,
y_true_data,
y_true_box_data,
ignore_threshold=ignore_thresh,
anchors=anchors,
num_classes=num_classes,
h=h,
w=w,
batch_size=batch_size)
tf.summary.scalar('loss', loss)
global_step = tf.get_variable(name='global_step',
trainable=False,
initializer=0,
dtype=tf.int32)
# returns a varlist containing only the vars of the conv layers right before the yolo layers
trainable_var_list = tf.trainable_variables()
last_layer_var_list = [
i for i in trainable_var_list
if i.shape[-1] == (5 + num_classes) * num_anchors_per_detector
]
train_op_with_frozen_variables = tf.train.AdamOptimizer(
learning_rate=lr).minimize(loss,
global_step=global_step,
var_list=last_layer_var_list)
train_op_with_all_variables = tf.train.AdamOptimizer(
learning_rate=lr).minimize(loss,
global_step=global_step,
var_list=trainable_var_list)
summ = tf.summary.merge_all()
# info
print('--info--')
print('model weights will be saved with filename: ', args['model_name'])
print('tensorboard event files located at path: ',
args['tensorboard_save_path'])
# build training loop
with tf.Session() as sess:
train_writer = tf.summary.FileWriter(tb_train_path, sess.graph)
val_writer = tf.summary.FileWriter(tb_val_path)
# initialize model weights either randomly or from a saved checkpoint
saver = tf.train.Saver()
if args['train_from_checkpoint'] is None:
print('initializing variables...')
sess.run(tf.global_variables_initializer())
else:
print('restoring weights from checkpoint: ',
args['train_from_checkpoint'])
saver.restore(sess, args['train_from_checkpoint'])
num_iterations = args['num_training_epochs'] * len(training_data)
print('beginning to train the model...')
for i in range(num_iterations):
input_images, y_true, y_true_boxes = get_training_batch(
training_data,
anchors,
num_classes,
batch_size=batch_size,
h=h,
w=w,
random=not args['test_model_overfit'])
# For the first epochs, train with the frozen layers. Then, unfreeze the entire graph.
if i < num_iterations // 3:
sess.run(train_op_with_frozen_variables,
feed_dict={
X: input_images,
y_true_data: y_true,
y_true_box_data: y_true_boxes
})
else:
sess.run(train_op_with_all_variables,
feed_dict={
X: input_images,
y_true_data: y_true,
y_true_box_data: y_true_boxes
})
if i % args['log_every_x_iterations'] == 0:
# write the training loss to tensorboard
lt, st = sess.run(
[loss, summ],
feed_dict={
X: input_images,
y_true_data: y_true,
y_true_box_data: y_true_boxes
})
train_writer.add_summary(st, i)
#write the validation loss to tensorboard if we are not in overfit mode
if not args['test_model_overfit']:
input_images, y_true, y_true_boxes = get_training_batch(
validation_data,
anchors,
num_classes,
batch_size=batch_size,
h=h,
w=w,
random=not args['test_model_overfit'])
lv, sv = sess.run(
[loss, summ],
feed_dict={
X: input_images,
y_true_data: y_true,
y_true_box_data: y_true_boxes
})
val_writer.add_summary(sv, i)
print("iteration: " + str(i) + ", training loss: " +
str(round(lt, 2)) + ", validation loss: " +
str(round(lv, 2)))
else:
print("iteration: " + str(i) + ", training loss: " +
str(round(lt, 2)))
if i % args['save_every_x_iterations'] == 0:
print('saving model weights at path: ',
args['train_save_path'])
saver.save(
sess,
os.path.join(args['train_save_path'], args['model_name']),
global_step)
train_writer.close()
val_writer.close()
def train(self, session, fevals, model, data_loader):
#Saver
saver = tf.train.Saver(tf.all_variables())
for epoch in xrange(self.tr_config['num_epoch']):
start_time = time.time()
source_i, target_i = data_loader.next_batch()
XX, XXM, YY, YYM = prepare_data(source_i, \
target_i, \
maxlen=self.tr_config['max_steps'])
import pdb
pdb.set_trace()
#Hts_enc, Ots_enc = model._init_states(XX.shape[0], model.TT)
pred, logits, Zts = model.fp(XX, XXM, XX.shape[0], model.TT,
model.TT)
#last_indice = tf.cast(tf.reduce_sum(XXM, 1)-1, 'int32')
#Hts_dec, Ots_dec = model._init_states(YY.shape[0], model.TT)
#Hts = tf.pack(Hts_enc[0])
#Ots = tf.pack(Ots_enc[0])
#Hts_init = last_relevant(Hts, last_indice)
#Ots_init = last_relevant(Ots, last_indice)
cost, _ = session.run(fevals, {\
self.inputs : XX,\
self.targets : YY,\
self.input_masks : XXM,\
self.target_masks: YYM})
end_time = time.time()
if epoch % 20 == 0:
epoch_time = end_time - start_time
print("...Epoch %d, Train Cost %f, Time %f" % \
(epoch, cost, epoch_time))
if self.tr_config['summaryF'] and model.summarize is not None:
summary_str = session.run(model.summarize, {\
self.inputs : XX,\
self.targets : YY})
model.summary_writer.add_summary(summary_str, epoch)
prior = "The future of artificial intelligence "
## Getting sample takes up (relative to GPU) a lot of time in CPU
if self.tr_config['sampleF'] and epoch % 200 == 0:
## Gen samples
gen_txt = self.get_samples(session, \
model, \
data_loader,\
prior,\
num_steps=100)
## Save the samples to a log file
gen_text_epoch = '-'*10+'Epoch '+str(epoch)+'-'*10 + '\n' +\
prior + gen_txt + '\n\n'
f = open(self.tr_config['save_gen_text'], 'a')
f.write(gen_text_epoch)
f.close()
## TODO Checkpoint (not working)
#if epoch % 500 == (500 - 1) :
# checkpoint_path = self.tr_config['checkpoint_path']
# saver.save(session, checkpoint_path, global_step \
# = epoch * data_loader.num_batches)
# print("model saved to {}".format(checkpoint_path))
return cost
def test(opt=train_opt):
test_data, test_videos, captions, wordtoix, ixtoword, bias_init_vector = prepare_data(
train_opt.video_data_path, train_opt.video_path, train_opt.video_feat_path,
'val', train_opt.dict_path)
n_words = len(ixtoword)
print('vocabulary size is %d' % n_words)
sess = tf.InteractiveSession()
model = Video_Caption_Generator(
n_words=n_words,
dim_embed=train_opt.dim_embed,
dim_ctx=train_opt.dim_ctx,
dim_hidden=train_opt.dim_hidden,
n_caption_lstm_step=train_opt.n_caption_lstm_step,
batch_size=train_opt.batch_size,
ctx_shape=train_opt.ctx_shape)
saver = tf.train.Saver()
saver.restore(sess, os.path.join(train_opt.model_path, train_opt.test_graph))
def beam_step(logprobsf, beam_size, hypo_size, t, beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec):
ys = np.sort(logprobsf)
ys[0] = ys[0,::-1]
ix = np.argsort(logprobsf)
ix[0] = ix[0,::-1]
candidates = []
cols = ys.shape[1]
rows = beam_size if t > 0 else 1
for col in range(cols):
for row in range(rows):
local_logprob = ys[row, col]
candidate_logprob = beam_logprobs_sum[row] + local_logprob
candidates.append({'c':ix[row,col], 'q':row, 'p':candidate_logprob, 'r':local_logprob})
candidates = sorted(candidates, key=lambda x: -x['p'])
new_h = h_rec.copy()
new_c = c_rec.copy()
if t >= 1:
beam_seq_prev = beam_seq[:t].copy()
beam_seq_logprobs_prev = beam_seq_logprobs[:t].copy()
for vix in range(beam_size):
v = candidates[vix]
if t >= 1:
beam_seq[:t, vix] = beam_seq_prev[:, v['q']]
beam_seq_logprobs[:t, vix] = beam_seq_logprobs_prev[:, v['q']]
for state_ix in range(len(new_h)):
new_h[state_ix][:, vix] = h_rec[state_ix][:, v['q']]
new_c[state_ix][:, vix] = c_rec[state_ix][:, v['q']]
beam_seq[t, vix] = v['c']
beam_seq_logprobs[t, vix] = v['r']
if beam_logprobs_sum[vix] != -1000:
beam_logprobs_sum[vix] = v['p']
else:
beam_logprobs_sum[vix] = beam_logprobs_sum[vix]
h_rec = new_h
c_rec = new_c
return beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec, candidates
file_index = train_opt.test_graph
file_index = file_index[file_index.find('-')+1::]
output_txt_gs_fd = open('./result/greedy_'+file_index+'.txt', 'w')
output_txt_bs_fd = open('./result/beam_'+file_index+'.txt', 'w')
output_txt_bsm_fd = open('./result/beam_mul_'+file_index+'.txt', 'w')
for idx, video_feat_path in enumerate(test_videos):
start_time = time.time()
print(idx, video_feat_path, captions[idx])
video_feat = np.load(video_feat_path)[None,...]
beam_seq = np.zeros((train_opt.n_caption_lstm_step, train_opt.beam_size)).astype(np.int32)
beam_seq_logprobs = np.zeros((train_opt.n_caption_lstm_step, train_opt.beam_size))
beam_logprobs_sum = np.zeros(train_opt.beam_size)
## to record previous h & c
h_rec = np.zeros((train_opt.n_caption_lstm_step, train_opt.dim_hidden, train_opt.beam_size)).astype(np.float32)
c_rec = np.zeros((train_opt.n_caption_lstm_step, train_opt.dim_hidden, train_opt.beam_size)).astype(np.float32)
done_beams = []
for ind in range(train_opt.n_caption_lstm_step):
if ind == 0:
context_tf, log_prob_tf, h_tf, c_tf = model.pred_word(ind)
log_prob, h, c = sess.run([log_prob_tf, h_tf, c_tf], feed_dict={context_tf:video_feat})
h_rec[ind][:, 0] = h
c_rec[ind][:, 0] = c
beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec, candidates = beam_step(
log_prob, train_opt.beam_size, train_opt.hypo_size, ind,
beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec)
print('Elapsed time:', str((time.time()-start_time)))
else:
log_prob = np.zeros((train_opt.beam_size, n_words)).astype(np.float32)
for vix in range(train_opt.beam_size):
v = candidates[vix]
context_tf, log_prob_tf, h_tf, c_tf = model.pred_word(
ind, prev_word=v['c'], h=h_rec[ind-1][:,v['q']], c=c_rec[ind-1][:,v['q']])
log_prob[vix,:], h_rec[ind][:,vix], c_rec[ind][:,vix] = sess.run([log_prob_tf, h_tf, c_tf],
feed_dict={context_tf:video_feat})
## suppress UNK tokens in the decoding
log_prob[:, 3] -= 1000
beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec, candidates = beam_step(
log_prob, train_opt.beam_size, train_opt.hypo_size, ind,
beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec)
for beam_ind in range(train_opt.beam_size):
if beam_seq[ind, beam_ind] == 2 or ind == train_opt.n_caption_lstm_step-1:
final_beam = {
'seq': beam_seq[:, beam_ind],
'logps': beam_seq_logprobs[:, beam_ind],
'p': beam_logprobs_sum[beam_ind]
}
done_beams.append(copy.deepcopy(final_beam))
beam_logprobs_sum[beam_ind] = -1000
if len(done_beams) >= train_opt.hypo_size:
break
print('Elapsed time:', str((time.time()-start_time)))
context_tf, generated_words_tf, logit_list_tf, alpha_list_tf = model.build_generator()
generated_word_index, alpha_list_val = sess.run([generated_words_tf,alpha_list_tf], feed_dict={context_tf:video_feat})
generated_word_index = [x[0] for x in generated_word_index]
generated_sentence_gs = convert2sen(ixtoword, generated_word_index)
print('greedy search: ', generated_sentence_gs)
output_txt_gs_fd.write(video_feat_path + '\n')
output_txt_gs_fd.write(generated_sentence_gs + '\n\n')
print('Elapsed time:', str((time.time()-start_time)))
for beam_ind in range(train_opt.hypo_size):
beam = done_beams[beam_ind]
generated_sentence = convert2sen(ixtoword, beam['seq'])
print('bs {}, p={}: {}'.format(beam_ind, beam['p'], generated_sentence))
output_txt_bsm_fd.write(video_feat_path + '\n')
output_txt_bsm_fd.write(generated_sentence + '\n\n')
if beam_ind == 1:
output_txt_bs_fd.write(video_feat_path + '\n')
output_txt_bs_fd.write(generated_sentence + '\n\n')
output_txt_gs_fd.close()
output_txt_bs_fd.close()
output_txt_bsm_fd.close()
-0.145], [-0.627, 0.614, 0.479],
[-0.354, 0.772, -0.528],
[-0.658, -0.472,
-0.586], [0.423, 0.322, -0.847],
[0.212, -0.754,
-0.622], [0.912, -0.104, 0.398],
[-0.311, 0.947,
-0.077], [0.679, 0.632, -0.374],
[0.135, -0.286, 0.949], [-0.647, 0.230, 0.727],
[0.904, 0.397, 0.158], [-0.757, 0.647, -0.087],
[0.143, 0.284, 0.948]])
# create gradient table from generated bvecs
gtab_generated = ul.create_own_gradient_table(bvecs_generated)
# prepare data
first_30_images, next_60_images, grad_table_first_30, grad_table_next_60 = ul.prepare_data(
test_img, test_bvecs, test_bvals)
# with open('results/masked_30.pickle', 'wb') as file_1:
# pickle.dump(ul.remove_background(first_30_images), file_1)
# with open('results/masked_60.pickle', 'wb') as file_2:
# pickle.dump(ul.remove_background(next_60_images), file_2)
with open('results/masked_30.pickle', 'rb') as file_1:
masked_30 = pickle.load(file_1)
with open('results/masked_60.pickle', 'rb') as file_2:
masked_60 = pickle.load(file_2)
reference_30 = masked_30[:, :, 39, 15]
reference_60 = masked_60[:, :, 39, 15]
# predicting the signal
transformed_all = ul.rotate_3d(masked_60[:, :, 39:41], 5)
def main(args=None):
#If args is None, will parse command line arguments
#Otherwise args needs to be list with arguments, like
# ['--dataset', 'CamVid', '--model', 'FC-DenseNet103']
parser = argparse.ArgumentParser()
parser.add_argument('--num_epochs',
type=int,
default=300,
help='Number of epochs to train for')
parser.add_argument('--epoch_start_i',
type=int,
default=0,
help='Start counting epochs from this number')
parser.add_argument('--checkpoint_step',
type=int,
default=5,
help='How often to save checkpoints (epochs)')
parser.add_argument('--validation_step',
type=int,
default=1,
help='How often to perform validation (epochs)')
parser.add_argument('--continue_training',
type=utils.str2bool,
default=False,
help='Whether to continue training from a checkpoint')
parser.add_argument(
'--continue_from',
type=str,
default='',
help='From which checkpoint to continue. Only relevant with darea_call.'
)
parser.add_argument('--dataset',
type=str,
default="CamVid",
help='Dataset you are using.')
parser.add_argument('--dataset_path',
type=str,
default="",
help='Path to Dataset folder.')
parser.add_argument(
'--image_suffix',
type=str,
default='',
required=False,
help=
'Only files with this extension should be included. You should specify it if some non-image files will be in the same folder'
)
parser.add_argument('--crop_height',
type=int,
default=512,
help='Height of cropped input image to network')
parser.add_argument('--crop_width',
type=int,
default=512,
help='Width of cropped input image to network')
parser.add_argument(
'--biased_crop',
type=float,
default=0,
help=
'Probability of making a biased cropped. Biased crops always contain some foreground portion. Only works if one of the classes is named "Background".'
)
parser.add_argument(
'--downscale_factor',
type=float,
default=0,
required=False,
help=
'Shrink image by this factor. E.g. if image is 1024x1024 and downscale_factor is 0.5, downscaled image will be 512x512. This is applied before cropping.'
)
parser.add_argument('--batch_size',
type=int,
default=1,
help='Number of images in each batch')
parser.add_argument(
'--num_val_images',
type=int,
default=20,
help='The number of images to used for validations. If -1 -> use all')
parser.add_argument(
'--h_flip',
type=utils.str2bool,
default=False,
help=
'Whether to randomly flip the image horizontally for data augmentation'
)
parser.add_argument(
'--v_flip',
type=utils.str2bool,
default=False,
help=
'Whether to randomly flip the image vertically for data augmentation')
parser.add_argument(
'--brightness',
type=float,
default=None,
help=
'Whether to randomly change the image brightness for data augmentation. Specifies the max bightness change as a factor between 0.0 and 1.0. For example, 0.1 represents a max brightness change of 10%% (+-).'
)
parser.add_argument(
'--rotation',
type=float,
default=None,
help=
'DOES NOT WORK! Whether to randomly rotate the image for data augmentation. Specifies the max rotation angle in degrees.'
)
parser.add_argument('--rotation_perpendicular',
type=utils.str2bool,
default=False,
help='Randomly rotates by 0, 90, 180 or 270 degrees')
parser.add_argument(
'--model',
type=str,
default="FC-DenseNet103",
help=
'The model you are using. See model_builder.py for supported models')
parser.add_argument(
'--frontend',
type=str,
default="ResNet101",
help=
'The frontend you are using. See frontend_builder.py for supported models'
)
parser.add_argument(
'--save_best',
type=utils.str2bool,
default=False,
help='Saves model with smallest loss rather than last model')
parser.add_argument('--learn_rate',
type=float,
default=0.0001,
help='The learning rate')
parser.add_argument(
'--chkpt_prefix',
type=str,
default='',
help=
'Prefix in front of checkpoint (intended for distinguishing same models ran with different parameters)'
)
parser.add_argument(
'--darea_call',
type=utils.str2bool,
default=False,
required=False,
help='Set to true when you call it from Darea software')
parser.add_argument(
'--save_path',
type=str,
default='checkpoints',
required=False,
help='Name of saved model. Only used when darea_call is true.')
parser.add_argument('--makePlots',
type=utils.str2bool,
default=True,
required=False,
help='Whether plots should be made')
if args is None:
args = parser.parse_args()
else:
args = parser.parse_args(args)
if args.darea_call: os.chdir(os.path.dirname(os.path.realpath(__file__)))
# Get the names of the classes so we can record the evaluation results
class_names_list, label_values = helpers.get_label_info(
os.path.join(args.dataset_path, args.dataset, "class_dict.csv"))
class_names_string = ', '.join(class_names_list)
if 'Background' not in class_names_list:
args.biased_crop = 0
backgroundValue = None
else:
backgroundValue = label_values[class_names_list.index('Background')]
num_classes = len(label_values)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# Compute your softmax cross entropy loss
net_input = tf.placeholder(tf.float32, shape=[
None, None, None, 3
]) #Try setting to 1 for grayscale, think theres no other changes needed.
net_output = tf.placeholder(tf.float32,
shape=[None, None, None, num_classes])
network, init_fn = model_builder.build_model(model_name=args.model,
frontend=args.frontend,
net_input=net_input,
num_classes=num_classes,
crop_width=args.crop_width,
crop_height=args.crop_height,
is_training=True)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(logits=network,
labels=net_output))
opt = tf.train.RMSPropOptimizer(
learning_rate=args.learn_rate, decay=0.995).minimize(
loss, var_list=[var for var in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=500)
sess.run(tf.global_variables_initializer())
utils.count_params()
# If a pre-trained ResNet is required, load the weights.
# This must be done AFTER the variables are initialized with sess.run(tf.global_variables_initializer())
if init_fn is not None:
init_fn(sess)
# Load a previous checkpoint if desired
if args.darea_call:
if args.continue_training and args.continue_from:
model_checkpoint_name = '../../deepLearning/checkpoints/' + args.continue_from + '.ckpt'
if os.path.isfile(model_checkpoint_name + '.index'):
print('Loading latest model checkpoint...')
saver.restore(sess, model_checkpoint_name)
print('successfully loaded {}'.format(model_checkpoint_name))
else:
print('Specified checkpoint {} not found. Starting fresh one'.
format(os.path.abspath(model_checkpoint_name)))
if args.save_path:
model_checkpoint_name = os.path.join(args.save_path,
args.dataset + '.ckpt')
else:
if args.continue_training:
if args.continue_from:
model_checkpoint_name = args.continue_from
if not model_checkpoint_name.endswith('.ckpt'):
model_checkpoint_name += '.ckpt'
else:
model_checkpoint_name = os.path.join(
args.save_path, "latest_model_" + args.chkpt_prefix +
args.model + "_" + args.dataset + ".ckpt")
if os.path.isfile(model_checkpoint_name + '.index'):
print('Loading latest model checkpoint...')
saver.restore(sess, model_checkpoint_name)
print('successfully loaded {}'.format(model_checkpoint_name))
else:
print('{} not found. Starting a fresh training'.format(
args.continue_from))
model_checkpoint_name = os.path.join(
args.save_path, "latest_model_" + args.chkpt_prefix + args.model +
"_" + args.dataset + ".ckpt")
# Load the data
print("Loading the data ...")
train_input_names,train_output_names, val_input_names, val_output_names, test_input_names, test_output_names = \
utils.prepare_data(dataset_dir=os.path.join(args.dataset_path,args.dataset),image_suffix=args.image_suffix)
print("\n***** Begin training *****")
print("Dataset -->", args.dataset)
print("Model -->", args.model)
if args.downscale_factor:
print("Downscale Factor -->", args.downscale_factor)
print("Crop Height -->", args.crop_height)
print("Crop Width -->", args.crop_width)
print("Num Epochs -->", args.num_epochs)
print("Batch Size -->", args.batch_size)
print("Num Classes -->", num_classes)
print("Learn Rate -->", args.learn_rate)
print("Validation Step -->", args.validation_step)
print("Data Augmentation:")
print("\tVertical Flip -->", args.v_flip)
print("\tHorizontal Flip -->", args.h_flip)
print("\tBrightness Alteration -->", args.brightness)
print("\tRotation -->", args.rotation)
print("\tPerpendicular Rotation -->", args.rotation_perpendicular)
print("", flush=True)
avg_loss_per_epoch = []
avg_scores_per_epoch = []
avg_iou_per_epoch = []
if args.num_val_images == -1:
#Use all validation images if so wanted by users
val_indices = range(0, len(val_input_names))
else:
# Which validation images do we want
val_indices = []
num_vals = min(args.num_val_images, len(val_input_names))
# Set random seed to make sure models are validated on the same validation images.
# So you can compare the results of different models more intuitively.
random.seed(16)
val_indices = random.sample(range(0, len(val_input_names)), num_vals)
#Copy class file with same name as checkpoint
shutil.copyfile(
os.path.join(args.dataset_path, args.dataset, "class_dict.csv"),
os.path.splitext(model_checkpoint_name)[0] + '.classes')
# Do the training here
for epoch in range(args.epoch_start_i, args.num_epochs):
current_losses = []
cnt = 0
# Equivalent to shuffling
id_list = np.random.permutation(len(train_input_names))
num_iters = int(np.floor(len(id_list) / args.batch_size))
st = time.time()
epoch_st = time.time()
for i in range(num_iters):
# st=time.time()
input_image_batch = []
output_image_batch = []
# Collect a batch of images
for j in range(args.batch_size):
index = i * args.batch_size + j
id = id_list[index]
input_image = utils.load_image(train_input_names[id])
output_image = utils.load_image(train_output_names[id])
with tf.device('/cpu:0'):
input_image, output_image = data_augmentation(
input_image, output_image, args, backgroundValue)
if 0:
#Debugging:
try:
os.mkdir('imagesinTrain')
'kj'
except:
pass
try:
os.mkdir('imagesinTrain/%04d' % (epoch))
except:
pass
file_name = utils.filepath_to_name(
train_input_names[id])
file_name = os.path.splitext(file_name)[0]
cv2.imwrite(
"%s/%04d/%s_im.png" %
("imagesinTrain", epoch, file_name),
cv2.cvtColor(np.uint8(input_image),
cv2.COLOR_RGB2BGR))
cv2.imwrite(
"%s/%04d/%s_gt.png" %
("imagesinTrain", epoch, file_name),
cv2.cvtColor(np.uint8(output_image),
cv2.COLOR_RGB2BGR))
# Prep the data. Make sure the labels are in one-hot format
input_image = np.float32(input_image) / 255.0
output_image = np.float32(
helpers.one_hot_it(label=output_image,
label_values=label_values))
if 0:
#Debugging:
try:
os.mkdir('imagesinTrain')
except:
pass
try:
os.mkdir('imagesinTrain/%04d' % (epoch))
except:
pass
file_name = utils.filepath_to_name(
train_input_names[id])
file_name = os.path.splitext(file_name)[0]
print("%s/%04d/%s_im.png" %
("imagesinTrain", epoch, file_name))
cv2.imwrite(
"%s/%04d/%s_im.png" %
("imagesinTrain", epoch, file_name),
cv2.cvtColor(np.uint8(input_image),
cv2.COLOR_RGB2BGR))
#cv2.imwrite("%s/%04d/%s_gt.png"%("imagesinTrain",epoch, file_name),cv2.cvtColor(np.uint8(output_image), cv2.COLOR_RGB2BGR))
input_image_batch.append(
np.expand_dims(input_image, axis=0))
output_image_batch.append(
np.expand_dims(output_image, axis=0))
if args.batch_size == 1:
input_image_batch = input_image_batch[0]
output_image_batch = output_image_batch[0]
else:
input_image_batch = np.squeeze(
np.stack(input_image_batch, axis=1))
output_image_batch = np.squeeze(
np.stack(output_image_batch, axis=1))
# Do the training
_, current, output_image = sess.run([opt, loss, network],
feed_dict={
net_input:
input_image_batch,
net_output:
output_image_batch
})
current_losses.append(current)
cnt = cnt + args.batch_size
if cnt % 25 == 0:
string_print = "Epoch = %d Count = %d Current_Loss = %.4f Time = %.2f" % (
epoch, cnt, current, time.time() - st)
utils.LOG(string_print)
st = time.time()
if 0:
#For Debugging
output_image = np.array(output_image[0, :, :, :])
output_image = helpers.reverse_one_hot(output_image)
out_vis_image = helpers.colour_code_segmentation(
output_image, label_values)
cv2.imwrite(
"%s/%04d/%s_pred.png" %
("imagesinTrain", epoch, file_name),
cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR))
mean_loss = np.mean(current_losses)
avg_loss_per_epoch.append(mean_loss)
# Create directories if needed
if not os.path.isdir("%s/%04d" % ("checkpoints", epoch)):
os.makedirs("%s/%04d" % ("checkpoints", epoch))
# If latest checkpoint should be saved, save now to same file name,
if not args.save_best:
print("Saving latest checkpoint")
saver.save(sess, model_checkpoint_name)
if val_indices != 0 and epoch % args.checkpoint_step == 0:
print("Saving checkpoint for this epoch")
saver.save(sess, "%s/%04d/model.ckpt" % (args.save_path, epoch))
if epoch % args.validation_step == 0:
if epoch == 0:
best_avg_iou = 0
print("Performing validation")
target = open("%s/%04d/val_scores.csv" % ("checkpoints", epoch),
'w')
target.write(
"val_name, avg_accuracy, precision, recall, f1 score, mean iou, %s\n"
% (class_names_string))
scores_list = []
class_scores_list = []
precision_list = []
recall_list = []
f1_list = []
iou_list = []
# Do the validation on a small set of validation images
for ind in val_indices:
input_image = utils.load_image(val_input_names[ind])
gt = utils.load_image(val_output_names[ind])
if args.downscale_factor and args.downscale_factor != 1:
dim = (int(input_image.shape[0] * args.downscale_factor),
int(input_image.shape[1] * args.downscale_factor))
input_image = cv2.resize(input_image,
dim,
interpolation=cv2.INTER_CUBIC)
gt = cv2.resize(gt, dim, interpolation=cv2.INTER_NEAREST)
#input_image, gt = data_augmentation(input_image, gt, args)
gt = helpers.reverse_one_hot(
helpers.one_hot_it(gt, label_values))
crop_height = args.crop_height
crop_width = args.crop_width
if input_image.shape[0] > crop_height or input_image.shape[
1] > crop_width:
#rectangle in bottom right corner smaller than cropped im will not be used
nrCroppingsY = input_image.shape[0] // crop_height
nrCroppingsX = input_image.shape[1] // crop_width
output_image = np.zeros([
nrCroppingsY * crop_height, nrCroppingsX * crop_width
])
gt = gt[0:nrCroppingsY * crop_height,
0:nrCroppingsX * crop_width]
for yi in range(nrCroppingsY):
row = np.zeros(
[crop_height, nrCroppingsX * crop_width])
for xi in range(nrCroppingsX):
inputIm = input_image[yi * crop_height:(1 + yi) *
crop_height,
xi * crop_width:(1 + xi) *
crop_width, :]
inputIm = np.expand_dims(np.float32(inputIm) /
255.0,
axis=0)
out = sess.run(network,
feed_dict={net_input: inputIm})
out = np.array(out[0, :, :, :])
out = helpers.reverse_one_hot(out)
row[:, xi * crop_width:(1 + xi) * crop_width] = out
output_image[yi * crop_height:(1 + yi) *
crop_height, :] = row
# st = time.time()
else:
input_image = np.expand_dims(np.float32(input_image) /
255.0,
axis=0)
output_image = sess.run(network,
feed_dict={net_input: input_image})
output_image = np.array(output_image[0, :, :, :])
output_image = helpers.reverse_one_hot(output_image)
out_vis_image = helpers.colour_code_segmentation(
output_image, label_values)
accuracy, class_accuracies, prec, rec, f1, iou, class_iou = utils.evaluate_segmentation(
pred=output_image, label=gt, num_classes=num_classes)
file_name = utils.filepath_to_name(val_input_names[ind])
target.write("%s, %f, %f, %f, %f, %f" %
(file_name, accuracy, prec, rec, f1, iou))
for item in class_accuracies:
target.write(", %f" % (item))
target.write("\n")
scores_list.append(accuracy)
class_scores_list.append(class_accuracies)
precision_list.append(prec)
recall_list.append(rec)
f1_list.append(f1)
iou_list.append(iou)
gt = helpers.colour_code_segmentation(gt, label_values)
file_name = os.path.basename(val_input_names[ind])
file_name = os.path.splitext(file_name)[0]
cv2.imwrite(
"%s/%04d/%s_pred.png" % ("checkpoints", epoch, file_name),
cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR))
cv2.imwrite(
"%s/%04d/%s_gt.png" % ("checkpoints", epoch, file_name),
cv2.cvtColor(np.uint8(gt), cv2.COLOR_RGB2BGR))
target.close()
avg_score = np.mean(scores_list)
class_avg_scores = np.mean(class_scores_list, axis=0)
avg_scores_per_epoch.append(avg_score)
avg_precision = np.mean(precision_list)
avg_recall = np.mean(recall_list)
avg_f1 = np.mean(f1_list)
avg_iou = np.mean(iou_list)
avg_iou_per_epoch.append(avg_iou)
print("\nAverage validation accuracy for epoch # %04d = %f" %
(epoch, avg_score))
print("Average per class validation accuracies for epoch # %04d:" %
(epoch))
for index, item in enumerate(class_avg_scores):
print("%s = %f" % (class_names_list[index], item))
print("Validation precision = ", avg_precision)
print("Validation recall = ", avg_recall)
print("Validation F1 score = ", avg_f1)
print("Validation IoU score = ", avg_iou, flush=True)
if args.save_best and avg_iou > best_avg_iou:
#Save model if best model is wanted and it is the best or if it is first evaluation
print("Saving best checkpoint with iou {}".format(avg_iou))
saver.save(sess, model_checkpoint_name)
best_avg_iou = avg_iou
#Save an info file
with open(model_checkpoint_name[:-5] + '.info', 'w') as f:
f.write('Epoch\t{}\nValidation IoU score\t{}'.format(
epoch, avg_iou))
epoch_time = time.time() - epoch_st
remain_time = epoch_time * (args.num_epochs - 1 - epoch)
m, s = divmod(remain_time, 60)
h, m = divmod(m, 60)
if s != 0:
train_time = "Remaining training time = %d hours %d minutes %d seconds\n" % (
h, m, s)
else:
train_time = "Remaining training time : Training completed.\n"
utils.LOG(train_time)
scores_list = []
with open(model_checkpoint_name[:-5] + '.info', 'a+') as f:
#Save some info on filesizes
f.write('\nimageSize\t{}'.format([args.crop_height, args.crop_width]))
f.write('\nTraining completed\t{}'.format(
datetime.datetime.now().strftime("%d-%b-%Y %H:%M:%S")))
if not args.darea_call and args.makePlots:
fig1, ax1 = plt.subplots(figsize=(11, 8))
ax1.plot(range(epoch + 1), avg_scores_per_epoch)
ax1.set_title("Average validation accuracy vs epochs")
ax1.set_xlabel("Epoch")
ax1.set_ylabel("Avg. val. accuracy")
plt.savefig('accuracy_vs_epochs.png')
plt.clf()
fig2, ax2 = plt.subplots(figsize=(11, 8))
ax2.plot(range(epoch + 1), avg_loss_per_epoch)
ax2.set_title("Average loss vs epochs")
ax2.set_xlabel("Epoch")
ax2.set_ylabel("Current loss")
plt.savefig('loss_vs_epochs.png')
plt.clf()
fig3, ax3 = plt.subplots(figsize=(11, 8))
ax3.plot(range(epoch + 1), avg_iou_per_epoch)
ax3.set_title("Average IoU vs epochs")
ax3.set_xlabel("Epoch")
ax3.set_ylabel("Current IoU")
plt.savefig('iou_vs_epochs.png')
def _main():
# parse command line arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'--train-from-checkpoint', type=str,
help="the path to where a previously trained model's weights are stored")
parser.add_argument('--class-path', default='utils/coco_classes.txt', type=str,
help='the path that points to the class names for the dataset')
parser.add_argument(
'--anchors-path', default='utils/anchors.txt', type=str,
help='the path that points to the anchor values for the models')
parser.add_argument(
'--data-path', default='data/image_paths_and_box_info.txt', type=str,
help='the path that points to the training data text file')
parser.add_argument(
'--input-height', default=416, type=int,
help='the input height of the yolov3 model. The height must be a multiple of 32')
parser.add_argument(
'--input-width', default=416, type=int,
help='The input width of the yolov3 model. The width must be a multiple of 32')
parser.add_argument(
'--batch-size', default=32, type=int,
help='the training batch size')
parser.add_argument(
'--max-num-boxes-per-image', default=20, type=int,
help='the max number of boxes that can be detected within one image')
parser.add_argument(
'--num-training-epochs', default=150, type=int,
help='the number of training epochs')
parser.add_argument(
'--learning-rate', default=0.001, type=float,
help='the learning rate')
parser.add_argument(
'--ignore-threshold', default=0.5, type=float,
help='impacts how the loss is calculated. Must be between zero and one')
parser.add_argument(
'--train-val-data-split', default=0.9, type=float,
help='the split between the data that will be used for training and data that will be used\n\
for validation')
parser.add_argument(
'--train-save-path', default='model-weights/',
help="the training model's checkpoint save path")
parser.add_argument(
'--model-name', default='model.ckpt',
help='the name that should be given to the checkpoint file')
parser.add_argument(
'--tensorboard-save-path', default='tb-logs/tb-train/',
help='the path where the event files to be used with tensorboard will be saved at')
parser.add_argument(
'--test-model-overfit', nargs='?', default=False, type=str2bool, const=True,
help='this option is useful in testing out if the loss function is working correctly')
parser.add_argument(
'--save-iterations', default=100, type=int,
help="how frequently the model's training weights are saved")
parser.add_argument(
'--log-iterations', default=5, type=int,
help="how frequently the model's loss is logged for it to be inspected in Tensorboard")
args = parser.parse_args()
# args info
print('[i] checkpoint: ', args.train_from_checkpoint)
print('[i] path of class file: ', args.class_path)
print('[i] path of anchors file: ', args.anchors_path)
# read inputs
h = args.input_height
w = args.input_width
ignore_thresh = args.ignore_threshold
max_num_boxes_per_image = args.max_num_boxes_per_image
anchors = get_anchors(args.anchors_path)
lr = args.learning_rate
num_anchors_per_detector = len(anchors) // 3
num_detectors_per_image = num_anchors_per_detector * (
((h / 32) * (w / 32)) + ((h / 16) * (w / 16)) + ((h / 8) * (w / 8)))
class_names = get_classes(args.class_path)
num_classes = len(class_names)
# tensorboard path
tb_train_path = args.tensorboard_save_path + 'train/'
tb_val_path = args.tensorboard_save_path + 'val/'
training_data, validation_data, batch_size = prepare_data(
args.data_path,
args.train_val_data_split,
args.batch_size,
args.test_model_overfit)
tf.reset_default_graph()
# build graph
with tf.variable_scope('y_true'):
y_true_data = tf.placeholder(dtype=tf.float32, shape=[None, num_detectors_per_image, num_classes + 5])
with tf.variable_scope('y_true_boxes'):
y_true_box_data = tf.placeholder(dtype=tf.float32,
shape=[None, max_num_boxes_per_image * num_anchors_per_detector, 4])
with tf.variable_scope('x_input'):
X = tf.placeholder(dtype=tf.float32, shape=[None, h, w, 3])
yolo_outputs = yolo_v3(inputs=X, num_classes=len(class_names), anchors=anchors, h=h, w=w, training=True) # output
loss = yolo_v3_loss(yolo_outputs, y_true_data, y_true_box_data, ignore_threshold=ignore_thresh, anchors=anchors,
num_classes=num_classes, h=h, w=w, batch_size=batch_size)
tf.summary.scalar('loss', loss)
global_step = tf.get_variable(name='global_step', trainable=False, initializer=0, dtype=tf.int32)
# returns a varlist containing only the vars of the conv layers right before the yolo layers
trainable_var_list = tf.trainable_variables()
last_layer_var_list = [i for i in trainable_var_list if i.shape[-1] == (5 + num_classes) * num_anchors_per_detector]
train_op_with_frozen_variables = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss, global_step=global_step,
var_list=last_layer_var_list)
train_op_with_all_variables = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss, global_step=global_step,
var_list=trainable_var_list)
summ = tf.summary.merge_all()
# info
print('-------info-------')
print('[i] model weights will be saved with filename: ', args.model_name)
print('[i] tensorboard event files located at path: ', args.tensorboard_save_path)
# build training loop
with tf.Session() as sess:
train_writer = tf.summary.FileWriter(tb_train_path, sess.graph)
val_writer = tf.summary.FileWriter(tb_val_path)
# initialize model weights either randomly or from a saved checkpoint
saver = tf.train.Saver()
if args['train_from_checkpoint'] is None:
print('[i] initializing variables...')
sess.run(tf.global_variables_initializer())
else:
print('[i] restoring weights from checkpoint: ', args.train_from_checkpoint)
saver.restore(sess, args.train_from_checkpoint)
num_iterations = args.num_epochs * len(training_data)
print('[i] beginning to train the model...')
for i in range(num_iterations):
input_images, y_true, y_true_boxes = get_training_batch(training_data, anchors, num_classes,
batch_size=batch_size, h=h, w=w,
random=not args.test_model_overfit)
# For the first epochs, train with the frozen layers. Then, unfreeze the entire graph.
if i < num_iterations // 3:
sess.run(train_op_with_frozen_variables,
feed_dict={X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes})
else:
sess.run(train_op_with_all_variables,
feed_dict={X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes})
if i % args.log_iterations == 0:
# write the training loss to tensorboard
lt, st = sess.run([loss, summ],
feed_dict={X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes})
train_writer.add_summary(st, i)
# write the validation loss to tensorboard if we are not in overfit mode
if not args.test_model_overfit:
input_images, y_true, y_true_boxes = get_training_batch(validation_data, anchors, num_classes,
batch_size=batch_size, h=h, w=w,
random=not args.test_model_overfit)
lv, sv = sess.run([loss, summ],
feed_dict={X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes})
val_writer.add_summary(sv, i)
print(
"| iteration: " + str(i) + ", training loss: " + str(round(lt, 2)) + ", validation loss: " + str(
round(lv, 2)))
else:
print("| iteration: " + str(i) + ", training loss: " + str(round(lt, 2)))
if i % args.save_iterations == 0:
print('[i] saving model weights at path: ', args.train_save_path)
saver.save(sess, os.path.join(args.train_save_path, args.model_name), global_step)
print('################################################################################')
train_writer.close()
val_writer.close()
def simple_test_Velonet(args, dataset):
#---------------------Load the NN model--------------------------------------------
model = torch.load('./saved_NN/neural_network.p')
windows_size = 11
device = torch.device('cpu')
network = get_Relative_Kinematic().to(device)
network.load_state_dict(model.get('model_state_dict'))
fig_x = 4
fig_y = 4
fig, axes = plt.subplots(fig_x, ncols=fig_y)
compute_mean = torch.zeros((windows_size, 1))
network.eval()
for i in range(0, len(dataset)):
dataset_name = dataset.dataset_name(i)
print("Test filter on sequence: " + dataset_name)
#---------------------Create a multi display--------------------------------------------
if (i % (fig_x * fig_y) == 0 and i != 0):
plt.subplots_adjust(left=0.03,
bottom=0.03,
right=0.97,
top=0.97,
wspace=0.12,
hspace=0.34)
plt.show()
fig, axes = plt.subplots(fig_x, ncols=fig_y)
#---------------------Load the data--------------------------------------------
t, ang_gt, _, v_gt, u = prepare_data(args, dataset, dataset_name)
dt = (t[1:] - t[:-1]).float()
d_vgt = v_gt[1:] - v_gt[:-1]
ang_gt = correct_ang_gt(ang_gt)
#---------------------Smoothing ground truth quaternion--------------------------------------------
quat_tensor = torch.zeros(u.shape[0], 4)
for j in range(0, quat_tensor.shape[0]):
quat_tensor[j, :] = euler_to_quaternion(ang_gt[j, 0], ang_gt[j, 1],
ang_gt[j, 2])
quat_tensor[2:-2, :] = (quat_tensor[0:-4, :] + quat_tensor[1:-3, :] +
quat_tensor[2:-2, :] + quat_tensor[3:-1, :] +
quat_tensor[4:, :]) / 5
quat_tensor = torch.div(
quat_tensor,
torch.norm(quat_tensor, dim=1).unsqueeze(1).repeat(1, 4))
relative_quaternion = relative_rotation(quat_tensor)
relative_quaternion = torch.div(
relative_quaternion,
torch.norm(relative_quaternion, dim=1).unsqueeze(1).repeat(1, 4))
test_dataset = torch.zeros(u.shape[0], windows_size, 14)
label_dataset = torch.zeros(u.shape[0], windows_size, 7)
data_index = 0
#---------------------Prepare the testing data--------------------------------------------
for j in range(windows_size, u.shape[0]):
index = j - windows_size
test_dataset[data_index, :, 0:3] = u[index:j, 0:3]
test_dataset[data_index, :, 3:6] = u[index:j, 3:6]
test_dataset[data_index, :, 6:9] = v_gt[index:j, 0:3]
test_dataset[data_index, :, 9:13] = quat_tensor[index:j, 0:4]
test_dataset[data_index, :, 13] = dt[index:j]
label_dataset[data_index, :, 0:3] = d_vgt[index:j]
label_dataset[data_index, :, 3:7] = relative_quaternion[index:j,
0:4]
data_index += 1
test_dataset = test_dataset[:data_index]
label_dataset = label_dataset[:data_index]
#test_dataset = quick_norm(test_dataset)
test_dataset = quick_std(test_dataset)
feat_gyr = (test_dataset[:, :, 0:3]).to(device)
feat_acc = (test_dataset[:, :, 3:6]).to(device)
feat_dt = (test_dataset[:, :, 13].unsqueeze(2)).to(device)
feat_prev_quat = (test_dataset[:, :, 9:13]).to(device)
#---------------------Use the NN to predict relative quaternion and relative velocity--------------------------------------------
with torch.no_grad():
measurements_vel, measurements_ori = network(
feat_gyr, feat_acc, feat_dt, feat_prev_quat)
measurements_vel = measurements_vel[:, 5, :]
measurements_ori = measurements_ori[:, 5, :]
label_dataset = label_dataset[:, 5, :]
measurements_ori = torch.div(
measurements_ori,
torch.norm(measurements_ori, dim=1).unsqueeze(1).repeat(1, 4))
#---------------------Compute the prediction errors--------------------------------------------
conj_target = conjugate(label_dataset[:, 3:7])
qprod = hamilton_product(measurements_ori, conj_target)
w, x, y, z = torch.chunk(qprod, 4, dim=-1)
quat_error = 2 * torch.sum(torch.abs(torch.cat(
(x, y, z), axis=-1)),
dim=1)
mse_vel = torch.mean(torch.pow(
(label_dataset[:, 0:3] - measurements_vel), 2),
dim=0)
mse_ori = torch.mean(torch.pow(
(label_dataset[:, 3:7] - measurements_ori), 2),
dim=0)
print('Error prediction')
print(mse_vel, mse_ori, torch.mean(quat_error, dim=0))
measurements_vel = measurements_vel.detach().numpy()
measurements_ori = measurements_ori.detach().numpy()
#---------------------Multi display--------------------------------------------
x_ax = int((i % (fig_x * fig_y)) / fig_x)
y_ax = i % fig_y
axes[x_ax, y_ax].grid(axis='y')
if dataset_name in dataset.datasets_train_filter:
axes[x_ax, y_ax].set_title(dataset_name, color='red')
else:
axes[x_ax, y_ax].set_title(dataset_name)
axes[x_ax, y_ax].plot(label_dataset[:, 0], color='orange')
axes[x_ax, y_ax].plot(label_dataset[:, 1], color='blue')
axes[x_ax, y_ax].plot(label_dataset[:, 2], color='green')
axes[x_ax, y_ax].plot(measurements_vel[:, 0], color='red')
axes[x_ax, y_ax].plot(measurements_vel[:, 1], color='c')
axes[x_ax, y_ax].plot(measurements_vel[:, 2], color='purple')
axes[x_ax, y_ax].set_xlabel('frame')
axes[x_ax, y_ax].set_ylabel('Relative velocity')
plt.subplots_adjust(left=0.03,
bottom=0.03,
right=0.97,
top=0.97,
wspace=0.12,
hspace=0.34)
plt.show()
def main():
args = get_args()
prepare_data()
word_vocab_config = {
"<UNK>": 0,
"<PAD>": 1,
"<start>": 2,
"<end>": 3,
"insert_start": "<SOS>",
"insert_end": "<EOS>",
"tokenization": "nltk",
"specials": ["<UNK>", "<PAD>", "<SOS>", "<EOS>"],
"embedding_root": os.path.join(args.app_path, "data", "embedding",
"word"),
"embedding_type": "glove.840B",
"embedding_dim": 300
}
print("Reading Vocab", flush=True)
char_vocab_config = word_vocab_config.copy()
char_vocab_config["embedding_root"] = os.path.join(args.app_path, "data",
"embedding", "char")
char_vocab_config["embedding_type"] = "glove_char.840B"
# TODO: build vocab out of dataset
# build vocab
itos, stoi, wv_vec = read_vocab(word_vocab_config)
itoc, ctoi, cv_vec = read_vocab(char_vocab_config)
char_embedding_config = {
"embedding_weights": cv_vec,
"padding_idx": word_vocab_config["<UNK>"],
"update": args.update_char_embedding,
"bidirectional": args.bidirectional,
"cell_type": "gru",
"output_dim": 300
}
word_embedding_config = {
"embedding_weights": wv_vec,
"padding_idx": word_vocab_config["<UNK>"],
"update": args.update_word_embedding
}
sentence_encoding_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": True,
"dropout": args.dropout,
}
pair_encoding_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": args.bidirectional,
"dropout": args.dropout,
"gated": True,
"mode": "GRU",
"rnn_cell": torch.nn.GRUCell,
"attn_size": args.attention_size,
"residual": args.residual
}
self_matching_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": args.bidirectional,
"dropout": args.dropout,
"gated": True,
"mode": "GRU",
"rnn_cell": torch.nn.GRUCell,
"attn_size": args.attention_size,
"residual": args.residual
}
pointer_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"dropout": args.dropout,
"residual": args.residual,
"rnn_cell": torch.nn.GRUCell
}
print("DEBUG Mode is ", "On" if args.debug else "Off", flush=True)
train_cache = "./data/cache/SQuAD%s.pkl" % ("_debug" if args.debug else "")
dev_cache = "./data/cache/SQuAD_dev%s.pkl" % ("_debug"
if args.debug else "")
train_json = args.train_json
dev_json = args.dev_json
train = read_dataset(train_json, itos, stoi, itoc, ctoi, train_cache,
args.debug)
dev = read_dataset(dev_json,
itos,
stoi,
itoc,
ctoi,
dev_cache,
args.debug,
split="dev")
dev_dataloader = dev.get_dataloader(args.batch_size_dev)
train_dataloader = train.get_dataloader(args.batch_size,
shuffle=True,
pin_memory=args.pin_memory)
trainer = Trainer(args, train_dataloader, dev_dataloader,
char_embedding_config, word_embedding_config,
sentence_encoding_config, pair_encoding_config,
self_matching_config, pointer_config)
trainer.train(args.epoch_num)
def train(train_opt):
train_data, captions, wordtoix, ixtoword, bias_init_vector = prepare_data(
train_opt.video_data_path, train_opt.video_path,
train_opt.video_feat_path, 'train', train_opt.dict_path)
n_words = len(wordtoix)
print('vocabulary size is %d' % n_words)
sess = tf.InteractiveSession()
caption_generator = Video_Caption_Generator(
n_words=n_words,
dim_embed=train_opt.dim_embed,
dim_ctx=train_opt.dim_ctx,
dim_hidden=train_opt.dim_hidden,
n_caption_lstm_step=train_opt.n_caption_lstm_step,
batch_size=train_opt.batch_size,
ctx_shape=train_opt.ctx_shape,
bias_init_vector=bias_init_vector)
loss_m, context_m, sentence_m, mask_m = caption_generator.build_model_m()
loss_n, context_n, sentence_n, mask_n = caption_generator.build_model_n()
loss_cl, context_cl, sentence_cl_p, sentence_cl_n, mask_cl, h_seq_pos_cl, h_seq_neg_cl = caption_generator.build_model_cl(
)
saver = tf.train.Saver(max_to_keep=50)
train_op_m = tf.train.AdamOptimizer(
train_opt.learning_rate).minimize(loss_m)
train_op_n = tf.train.AdamOptimizer(train_opt.learning_rate).minimize(
loss_n,
var_list=[
caption_generator.lstm_W_n, caption_generator.lstm_U_n,
caption_generator.lstm_b_n, caption_generator.decode_lstm_W_n,
caption_generator.decode_lstm_W_n
])
train_op_cl = tf.train.AdamOptimizer(train_opt.learning_rate).minimize(
loss_cl,
var_list=[
caption_generator.lstm_W_m, caption_generator.lstm_U_m,
caption_generator.lstm_b_m
])
tf.global_variables_initializer().run()
#---------------------------------------------------------------------------
# first train p_m with pure positive samples for epochs_m
pretrained_model_m = os.path.join(train_opt.model_path,
train_opt.pretrained_graph_m)
if os.path.isfile(pretrained_model_m):
print("Target model has already been trained...")
else:
print("Start training target model...")
for epoch in range(train_opt.epochs_m + 1):
index = list(train_data.index)
np.random.shuffle(index)
train_data_m = train_data.ix[index]
current_train_data = train_data_m.groupby('video_feat_path').apply(
lambda x: x.iloc[0])
current_train_data = current_train_data.reset_index(drop=True)
for start, end in zip(
range(0, len(current_train_data), train_opt.batch_size),
range(train_opt.batch_size, len(current_train_data),
train_opt.batch_size)):
start_time = time.time()
current_caption_matrix, current_caption_masks, current_feats = fetch_batch_data(
current_train_data, start, end, train_opt, wordtoix,
'target')
_, loss_value = sess.run(
[train_op_m, loss_m],
feed_dict={
context_m: current_feats,
sentence_m: current_caption_matrix,
mask_m: current_caption_masks
})
print('idx:', start, 'Epoch:', epoch, 'loss:', loss_value,
'Elapsed time:', str((time.time() - start_time)))
if np.mod(epoch, 100) == 0 and epoch > 0:
print("Epoch ", epoch, "is done. Saving the model...")
saver.save(sess,
os.path.join(train_opt.model_path, 'model'),
global_step=epoch)
#---------------------------------------------------------------------------
# then train p_n with pure negative samples for epochs_n, load the pre-trained p_m model
pretrained_model_n = os.path.join(train_opt.model_path,
train_opt.pretrained_graph_n)
if os.path.isfile(pretrained_model_n):
print("Reference model has already been trained...")
else:
print("Start training reference model...")
saver = tf.train.import_meta_graph(pretrained_model_m)
saver.restore(sess, os.path.join(train_opt.model_path, 'model-900'))
for epoch in range(train_opt.epochs_m + 1,
train_opt.epochs_m + train_opt.epochs_n + 1):
current_train_data = generate_neg_samples(data=train_data,
phase='reference')
for start, end in zip(
range(0, len(current_train_data), train_opt.batch_size),
range(train_opt.batch_size, len(current_train_data),
train_opt.batch_size)):
start_time = time.time()
current_caption_matrix, current_caption_masks, current_feats = fetch_batch_data(
current_train_data, start, end, train_opt, wordtoix,
'reference')
_, loss_value = sess.run(
[train_op_n, loss_n],
feed_dict={
context_n: current_feats,
sentence_n: current_caption_matrix,
mask_n: current_caption_masks
})
print('idx:', start, 'Epoch:', epoch, 'loss:', loss_value,
'Elapsed time:', str((time.time() - start_time)))
if np.mod(epoch, 100) == 0 and epoch > 0:
print("Epoch ", epoch, "is done. Saving the model...")
saver.save(sess,
os.path.join(train_opt.model_path, 'model'),
global_step=epoch)
#---------------------------------------------------------------------------
# next finetune p_m with p_n model for epochs_cl, load pre-trained p_m and p_n model
pretrained_model_cl = os.path.join(train_opt.model_path,
train_opt.pretrained_graph_cl)
if os.path.isfile(pretrained_model_cl):
print("CL model has already been trained...")
else:
print("Start training CL model...")
saver = tf.train.import_meta_graph(
os.path.join(train_opt.model_path, 'model-1800.meta'))
saver.restore(sess, os.path.join(train_opt.model_path, 'model-1800'))
for epoch in range(
train_opt.epochs_m + train_opt.epochs_n + 1,
train_opt.epochs_m + train_opt.epochs_n + train_opt.epochs_cl +
1):
current_train_data_p, current_train_data_n = generate_cl_samples(
train_data, train_opt.neg_K)
for start, end in zip(
range(0, len(current_train_data_p),
train_opt.batch_size * train_opt.neg_K),
range(train_opt.batch_size * train_opt.neg_K,
len(current_train_data_p),
train_opt.batch_size * train_opt.neg_K)):
start_time = time.time()
current_caption_matrix_p, current_caption_masks_p, current_feats = fetch_batch_data(
current_train_data_p, start, end, train_opt, wordtoix,
'CL')
current_caption_matrix_n, current_caption_masks_n, _ = fetch_batch_data(
current_train_data_n, start, end, train_opt, wordtoix,
'CL')
_, loss_value = sess.run(
[train_op_cl, loss_cl],
feed_dict={
context_cl: current_feats,
sentence_cl_p: current_caption_matrix_p,
sentence_cl_n: current_caption_matrix_n,
mask_cl: current_caption_masks_p
})
print('idx:', start, 'Epoch:', epoch, 'loss:', loss_value,
'Elapsed time:', str((time.time() - start_time)))
if np.mod(epoch, 10) == 0 and epoch > 0:
print("Epoch ", epoch, "is done. Saving the model...")
saver.save(sess,
os.path.join(train_opt.model_path, 'model'),
global_step=epoch)
network, _ = model_builder.build_model(args.model, net_input=net_input,
num_classes=num_classes,
crop_width=args.crop_width,
crop_height=args.crop_height,
is_training=False)
sess.run(tf.global_variables_initializer())
print('Loading model checkpoint weights ...')
saver=tf.train.Saver(max_to_keep=1000)
saver.restore(sess, args.checkpoint_path)
# Load the data
print("Loading the data ...")
train_input_names,train_output_names, val_input_names, val_output_names, test_input_names, test_output_names = utils.prepare_data(dataset_dir=os.path.join(args.dataset_path,args.dataset))
# Create directories if needed
if args.output_path!='' and not os.path.isdir(args.output_path):
os.mkdir(args.output_path)
target=open(os.path.join(args.output_path, 'test.csv'),'w')
target.write("test_name,test_accuracy,precision,recall,f1 score,mean iou,%s,%s" % (','.join(class_names_acc), ','.join(class_names_iou)))
scores_list = []
class_scores_list = []
precision_list = []
recall_list = []
f1_list = []
iou_list = []
run_times_list = []