def get_experimental_setup(logger_obj, channels_ids, test_ids, train_ids, val_ids, name, dataset_name): path = "archives/mts_archive/" dataset = Dataset(dataset_name, None, logger_obj) x_test, y_test, sampling_test = dataset.load(path, test_ids, channels_ids) x_val, y_val, sampling_val = dataset.load(path, val_ids, channels_ids) x_train, y_train, sampling_train = dataset.load(path, train_ids, channels_ids) x_train = [np.expand_dims(np.array(x), 2) for x in x_train] x_val = [np.expand_dims(np.array(x), 2) for x in x_val] x_test = [np.expand_dims(np.array(x), 2) for x in x_test] input_shapes, nb_classes, y_val, y_train, y_test, y_true = prepare_data( x_train, y_train, y_val, y_test) ndft_arr = [get_ndft(x) for x in sampling_test] if len(input_shapes) != len(ndft_arr): raise Exception("Different sizes of input_shapes and ndft_arr") for i in range(len(input_shapes)): if input_shapes[i][0] < ndft_arr[i]: raise Exception( f"Too big ndft, i: {i}, ndft_arr[i]: {ndft_arr[i]}, input_shapes[i][0]: {input_shapes[i][0]}" ) experimental_setup = ExperimentalSetup(name, x_train, y_train, x_val, y_val, x_test, y_test, input_shapes, sampling_val, ndft_arr, nb_classes, lambda x: 150, get_batch_size) return experimental_setup
def features_testing(self, c): total_cols = self.base_cols + self.added_cols if c not in total_cols: cols = total_cols + [c] x_df = self.features_df[cols] x_df = x_df.dropna() X_train, X_valid, y_train, y_valid, df_train, df_train_y, df_valid, df_valid_y = \ prepare_data(x_df, self.cutoff, self.s2pred, self.merging, is_features=True) # model = AdaBoost(self.n_est, self.class_weights) model = RandomForest(self.n_est, self.class_weights) # model = MLP() model.fit(X=X_train, y=y_train) df_valid_y['predictions'] = model.predict( df_valid.drop('timestamp', axis=1)) return (df_valid_y, c)
def loop(c): if c not in base_cols: cols = base_cols + [c] x_df = features[cols] x_df = x_df.dropna() X_train, X_valid, y_train, y_valid, df_train_y, df_valid, df_valid_y = \ prepare_data(x_df, CUTOFF, s2pred, merging, is_features=True) model.fit(X=X_train, y=y_train) df_valid_y['predictions'] = model.predict( df_valid.drop('timestamp', axis=1)) df_valid_y.to_csv('predictions/' + data_intervals + '/feature_selection/' + c + '.csv', index=False)
def __init__(self, args): super(KITTIDataset, self).__init__(args) directory_train = '../kitti_data/train_data' directory_valid = '../kitti_data/valid_data' for filename in os.listdir(directory_train): if filename.endswith(".p"): self.datasets_train_filter[filename[:-2]] = [0, None] t, _, _, _, _ = prepare_data(args, self, filename[:-2]) self.training_dataset_length += t.shape[0] else: continue for filename in os.listdir(directory_valid): if filename.endswith(".p"): self.datasets_validatation_filter[filename[:-2]] = [0, None] t, _, _, _, _ = prepare_data(args, self, filename[:-2]) self.valid_dataset_length += t.shape[0] else: continue print(self.valid_dataset_length, self.training_dataset_length)
def main(): args = get_args() prepare_data() word_vocab_config = { "<UNK>": 0, "<PAD>": 1, "<start>": 2, "<end>": 3, "insert_start": "<SOS>", "insert_end": "<EOS>", "tokenization": "nltk", "specials": ["<UNK>", "<PAD>", "<SOS>", "<EOS>"], "embedding_root": os.path.join(args.app_path, "data", "embedding", "word"), "embedding_type": "glove.840B", "embedding_dim": 300 } print("Reading Vocab", flush=True) char_vocab_config = word_vocab_config.copy() char_vocab_config["embedding_root"] = os.path.join(args.app_path, "data", "embedding", "char") char_vocab_config["embedding_type"] = "glove_char.840B" # TODO: build vocab out of dataset # build vocab itos, stoi, wv_vec = read_vocab(word_vocab_config) itoc, ctoi, cv_vec = read_vocab(char_vocab_config) char_embedding_config = { "embedding_weights": cv_vec, "padding_idx": word_vocab_config["<UNK>"], "update": args.update_char_embedding, "bidirectional": args.bidirectional, "cell_type": "gru", "output_dim": 300 } word_embedding_config = { "embedding_weights": wv_vec, "padding_idx": word_vocab_config["<UNK>"], "update": args.update_word_embedding } sentence_encoding_config = { "hidden_size": args.hidden_size, "num_layers": args.num_layers, "bidirectional": True, "dropout": args.dropout, } pair_encoding_config = { "hidden_size": args.hidden_size, "num_layers": args.num_layers, "bidirectional": args.bidirectional, "dropout": args.dropout, "gated": True, "mode": "GRU", "rnn_cell": torch.nn.GRUCell, "attn_size": args.attention_size, "residual": args.residual } self_matching_config = { "hidden_size": args.hidden_size, "num_layers": args.num_layers, "bidirectional": args.bidirectional, "dropout": args.dropout, "gated": True, "mode": "GRU", "rnn_cell": torch.nn.GRUCell, "attn_size": args.attention_size, "residual": args.residual } pointer_config = { "hidden_size": args.hidden_size, "num_layers": args.num_layers, "dropout": args.dropout, "residual": args.residual, "rnn_cell": torch.nn.GRUCell } print("DEBUG Mode is ", "On" if args.debug else "Off", flush=True) dev_cache = "./data/cache/SQuAD_dev%s.pkl" % ("_debug" if args.debug else "") test_json = args.test_json test = read_dataset(test_json, itos, stoi, itoc, ctoi, dev_cache, args.debug, split="dev") test_dataloader = test.get_dataloader(args.batch_size_dev) tester = Tester(args, test_dataloader, char_embedding_config, word_embedding_config, sentence_encoding_config, pair_encoding_config, self_matching_config, pointer_config) result = tester.test() json.dump(result, open('prediction.json', 'w')) pd.DataFrame([[id, ' '.join([str(j) for j in range(ans[0], ans[1])])] for id, ans in result.items()], columns=['id', 'answer']).to_csv('prediction.csv', index=False)
def main(args, _log): def data_augmentation(input_image, output_image): # Data augmentation input_image, output_image = utils.random_crop(input_image, output_image, args["crop_height"], args["crop_width"]) if args["h_flip"] and random.randint(0, 1): input_image = cv2.flip(input_image, 1) output_image = cv2.flip(output_image, 1) if args["v_flip"] and random.randint(0, 1): input_image = cv2.flip(input_image, 0) output_image = cv2.flip(output_image, 0) if args["brightness"]: factor = 1.0 + random.uniform(-1.0 * args["brightness"], args["brightness"]) table = np.array([((i / 255.0) * factor) * 255 for i in np.arange(0, 256)]).astype(np.uint8) input_image = cv2.LUT(input_image, table) if args["rotation"]: angle = random.uniform(-1 * args["rotation"], args["rotation"]) if args["rotation"]: M = cv2.getRotationMatrix2D( (input_image.shape[1] // 2, input_image.shape[0] // 2), angle, 1.0) input_image = cv2.warpAffine( input_image, M, (input_image.shape[1], input_image.shape[0]), flags=cv2.INTER_NEAREST) output_image = cv2.warpAffine( output_image, M, (output_image.shape[1], output_image.shape[0]), flags=cv2.INTER_NEAREST) return input_image, output_image print("Args:", args) # Get the names of the classes so we can record the evaluation results class_names_list, label_values = helpers.get_label_info( os.path.join(args["dataset"], "class_dict.csv")) class_names_string = "" for class_name in class_names_list: if not class_name == class_names_list[-1]: class_names_string = class_names_string + class_name + ", " else: class_names_string = class_names_string + class_name num_classes = len(label_values) config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) # Compute your softmax cross entropy loss net_input = tf.placeholder(tf.float32, shape=[None, None, None, 3]) net_output = tf.placeholder(tf.float32, shape=[None, None, None, num_classes]) network, init_fn = model_builder.build_model( model_name=args["model"], frontend=args["frontend"], net_input=net_input, num_classes=num_classes, crop_width=args["crop_width"], crop_height=args["crop_height"], is_training=True) loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=network, labels=net_output)) opt = tf.train.RMSPropOptimizer( learning_rate=0.0001, decay=0.995).minimize( loss, var_list=[var for var in tf.trainable_variables()]) saver = tf.train.Saver(max_to_keep=1000) sess.run(tf.global_variables_initializer()) utils.count_params() # If a pre-trained ResNet is required, load the weights. # This must be done AFTER the variables are initialized with sess.run(tf.global_variables_initializer()) if init_fn is not None: init_fn(sess) # Load a previous checkpoint if desired model_checkpoint_name = "checkpoints/latest_model_" + args[ "model"] + "_" + basename(normpath(args["dataset"])) + ".ckpt" if args["continue_training"]: _log.info('Loaded latest model checkpoint') saver.restore(sess, model_checkpoint_name) # Load the data _log.info("Loading the data ...") train_input_names, train_output_names, val_input_names, val_output_names, test_input_names, test_output_names = utils.prepare_data( dataset_dir=args["dataset"]) _log.info("\n***** Begin training *****") _log.debug("Dataset -->", args["dataset"]) _log.debug("Model -->", args["model"]) _log.debug("Crop Height -->", args["crop_height"]) _log.debug("Crop Width -->", args["crop_width"]) _log.debug("Num Epochs -->", args["num_epochs"]) _log.debug("Batch Size -->", args["batch_size"]) _log.debug("Num Classes -->", num_classes) _log.debug("Data Augmentation:") _log.debug("\tVertical Flip -->", args["v_flip"]) _log.debug("\tHorizontal Flip -->", args["h_flip"]) _log.debug("\tBrightness Alteration -->", args["brightness"]) _log.debug("\tRotation -->", args["rotation"]) avg_loss_per_epoch = [] avg_scores_per_epoch = [] avg_iou_per_epoch = [] # Which validation images do we want val_indices = [] num_vals = min(args["num_val_images"], len(val_input_names)) # Set random seed to make sure models are validated on the same validation images. # So you can compare the results of different models more intuitively. random.seed(16) val_indices = random.sample(range(0, len(val_input_names)), num_vals) # Do the training here for epoch in range(args["epoch_start_i"], args["num_epochs"]): current_losses = [] cnt = 0 # Equivalent to shuffling id_list = np.random.permutation(len(train_input_names)) num_iters = int(np.floor(len(id_list) / args["batch_size"])) st = time.time() epoch_st = time.time() for i in range(num_iters): # st=time.time() input_image_batch = [] output_image_batch = [] # Collect a batch of images for j in range(args["batch_size"]): index = i * args["batch_size"] + j id = id_list[index] input_image = utils.load_image(train_input_names[id], args["crop_width"], args["crop_height"]) output_image = utils.load_image(train_output_names[id], args["crop_width"], args["crop_height"]) with tf.device('/cpu:0'): input_image, output_image = data_augmentation( input_image, output_image) # Prep the data. Make sure the labels are in one-hot format input_image = np.float32(input_image) / 255.0 output_image = np.float32( helpers.one_hot_it(label=output_image, label_values=label_values)) input_image_batch.append( np.expand_dims(input_image, axis=0)) output_image_batch.append( np.expand_dims(output_image, axis=0)) if args["batch_size"] == 1: input_image_batch = input_image_batch[0] output_image_batch = output_image_batch[0] else: input_image_batch = np.squeeze( np.stack(input_image_batch, axis=1)) output_image_batch = np.squeeze( np.stack(output_image_batch, axis=1)) # Do the training _, current = sess.run([opt, loss], feed_dict={ net_input: input_image_batch, net_output: output_image_batch }) current_losses.append(current) cnt = cnt + args["batch_size"] if cnt % 20 == 0: string_print = "Epoch = %d Count = %d Iter:(%d of %d) Current_Loss = %.4f Time = %.2f" % ( epoch, cnt, i, num_iters, current, time.time() - st) utils.LOG(string_print) st = time.time() mean_loss = np.mean(current_losses) avg_loss_per_epoch.append(mean_loss) # Create directories if needed if not os.path.isdir("%s/%04d" % ("checkpoints", epoch)): os.makedirs("%s/%04d" % ("checkpoints", epoch)) # Save latest checkpoint to same file name _log.info("Saving latest checkpoint") saver.save(sess, model_checkpoint_name) if val_indices != 0 and epoch % args["checkpoint_step"] == 0: _log.info("Saving checkpoint for this epoch") saver.save(sess, "%s/%04d/model.ckpt" % ("checkpoints", epoch)) if epoch % args["validation_step"] == 0: _log.info("Performing validation") target_path = "%s/%04d/val_scores.csv" % ("checkpoints", epoch) target = open(target_path, 'w') target.write( "val_name, avg_accuracy, precision, recall, f1 score, mean iou, %s\n" % (class_names_string)) scores_list = [] class_scores_list = [] precision_list = [] recall_list = [] f1_list = [] iou_list = [] # Do the validation on a small set of validation images for ind in val_indices: input_image = np.expand_dims(np.float32( utils.load_image(val_input_names[ind], args["crop_width"], args["crop_height"]) [:args["crop_height"], :args["crop_width"]]), axis=0) / 255.0 gt = utils.load_image( val_output_names[ind], args["crop_width"], args["crop_height"] )[:args["crop_height"], :args["crop_width"]] gt = helpers.reverse_one_hot( helpers.one_hot_it(gt, label_values)) # st = time.time() output_image = sess.run(network, feed_dict={net_input: input_image}) output_image = np.array(output_image[0, :, :, :]) output_image = helpers.reverse_one_hot(output_image) out_vis_image = helpers.colour_code_segmentation( output_image, label_values) accuracy, class_accuracies, prec, rec, f1, iou = utils.evaluate_segmentation( pred=output_image, label=gt, num_classes=num_classes) file_name = utils.filepath_to_name(val_input_names[ind]) target.write("%s, %f, %f, %f, %f, %f" % (file_name, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f" % (item)) target.write("\n") scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec) f1_list.append(f1) iou_list.append(iou) gt = helpers.colour_code_segmentation(gt, label_values) file_name = os.path.basename(val_input_names[ind]) file_name = os.path.splitext(file_name)[0] pred_img_path = "%s/%04d/%s_pred.png" % ("checkpoints", epoch, file_name) gt_img_path = "%s/%04d/%s_gt.png" % ("checkpoints", epoch, file_name) cv2.imwrite( pred_img_path, cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR)) cv2.imwrite(gt_img_path, cv2.cvtColor(np.uint8(gt), cv2.COLOR_RGB2BGR)) # Send the first 16 images to sacred if ind in val_indices[:16]: ex.add_artifact(gt_img_path, "GtImage_%d" % ind) ex.add_artifact(pred_img_path, "PredImage_%d" % ind) target.close() ex.add_artifact(target_path) avg_score = np.mean(scores_list) class_avg_scores = np.mean(class_scores_list, axis=0) avg_scores_per_epoch.append(avg_score) avg_precision = np.mean(precision_list) avg_recall = np.mean(recall_list) avg_f1 = np.mean(f1_list) avg_iou = np.mean(iou_list) avg_iou_per_epoch.append(avg_iou) # Sacred info dict gets sent every heartbeat (10s) ex.info["avg_score"] = avg_score ex.info["class_avg_scores"] = class_avg_scores ex.info["avg_precision"] = avg_precision ex.info["avg_recall"] = avg_recall ex.info["avg_f1"] = avg_f1 ex.info["avg_iou"] = avg_iou _log.debug("\nAverage validation accuracy for epoch # %04d = %f" % (epoch, avg_score)) _log.debug( "Average per class validation accuracies for epoch # %04d:" % (epoch)) for index, item in enumerate(class_avg_scores): _log.debug("%s = %f" % (class_names_list[index], item)) _log.debug("Validation precision = ", avg_precision) _log.debug("Validation recall = ", avg_recall) _log.debug("Validation F1 score = ", avg_f1) _log.debug("Validation IoU score = ", avg_iou) epoch_time = time.time() - epoch_st remain_time = epoch_time * (args["num_epochs"] - 1 - epoch) m, s = divmod(remain_time, 60) h, m = divmod(m, 60) if s != 0: train_time = "Remaining training time = %d hours %d minutes %d seconds\n" % ( h, m, s) else: train_time = "Remaining training time : Training completed.\n" utils.LOG(train_time) scores_list = [] fig1, ax1 = plt.subplots(figsize=(11, 8)) ax1.plot(range(epoch + 1), avg_scores_per_epoch) ax1.set_title("Average validation accuracy vs epochs") ax1.set_xlabel("Epoch") ax1.set_ylabel("Avg. val. accuracy") plt.savefig('accuracy_vs_epochs.png') ex.add_artifact("accuracy_vs_epochs.png") plt.clf() fig2, ax2 = plt.subplots(figsize=(11, 8)) ax2.plot(range(epoch + 1), avg_loss_per_epoch) ax2.set_title("Average loss vs epochs") ax2.set_xlabel("Epoch") ax2.set_ylabel("Current loss") plt.savefig('loss_vs_epochs.png') ex.add_artifact("loss_vs_epochs.png") plt.clf() fig3, ax3 = plt.subplots(figsize=(11, 8)) ax3.plot(range(epoch + 1), avg_iou_per_epoch) ax3.set_title("Average IoU vs epochs") ax3.set_xlabel("Epoch") ax3.set_ylabel("Current IoU") plt.savefig('iou_vs_epochs.png') ex.add_artifact("iou_vs_epochs.png")
from utils.utils import process_image, deprocess_image from utils.utils import read_and_generate_heatmap, prepare_data,evaluate_distribution_accuracy from models import create_model max_features = 20000 maxlen=100 EMBEDDING_DIM = 300 use_distribution = True use_semantics = False use_comments=True use_multigap=True # X_train, Y_train, X_test, Y_test = prepare_data(use_distribution=use_distribution) X_train, Y_train,X_test, Y_test,X_train_text, X_test_text,embedding_layer = prepare_data(use_distribution=use_distribution, use_semantics=use_semantics, use_comments=use_comments) # X_train, Y_train,X_test, Y_test= prepare_data(use_distribution=use_distribution, use_semantics=False) # X_train, Y_train, Y_train_semantics, X_test, Y_test, Y_test_semantics, X_train_text, X_test_text, embedding_layer = prepare_data(use_distribution=use_distribution, use_semantics=use_semantics, use_comments=use_comments) ## Without image data # _, Y_train,_, Y_test,X_train_text, X_test_text,embedding_layer = prepare_data(use_distribution=use_distribution, use_semantics=use_semantics, use_comments=use_comments, imageDataAvailable=False) # BEST MODEL model = create_model('weights/2017-01-25 22_56_09 - distribution_2layergru_extra_conv_layer.h5', use_distribution=use_distribution, use_semantics=use_semantics,use_multigap=use_multigap,use_comments=use_comments, embedding_layer=embedding_layer,extra_conv_layer=True,textInputMaxLength=maxlen,embedding_dim=EMBEDDING_DIM) # model = create_model('weights/googlenet_aesthetics_weights.h5',
def standardization_all_data(args, dataset): nb_data = 0 nb_data_dt = 0 #---------------------Compute X - mean factor---------------------------------------------------------------- for i in range(0, len(dataset)): #---------------------Load the track---------------------------------------------------------------- dataset_name = dataset.dataset_name(i) print(dataset_name) t, ang_gt, _, v_gt, u = prepare_data(args, dataset, dataset_name) dt = (t[1:] - t[:-1]).unsqueeze(1) nb_data += u.shape[0] nb_data_dt += dt.shape[0] ang_gt = correct_ang_gt(ang_gt) #---------------------Smoothing ground truth quaternion-------------------------------------------- quat_tensor = torch.zeros(u.shape[0], 4) for j in range(0, quat_tensor.shape[0]): quat_tensor[j, :] = euler_to_quaternion(ang_gt[j, 0], ang_gt[j, 1], ang_gt[j, 2]) quat_tensor[2:-2, :] = (quat_tensor[0:-4, :] + quat_tensor[1:-3, :] + quat_tensor[2:-2, :] + quat_tensor[3:-1, :] + quat_tensor[4:, :]) / 5 quat_tensor = torch.div( quat_tensor, torch.norm(quat_tensor, dim=1).unsqueeze(1).repeat(1, 4)) if i == 0: u_input_loc = torch.sum(u[:], dim=0) v_gt_input_loc = torch.sum(v_gt[:], dim=0) quat_tensor_loc = torch.sum(quat_tensor[:, 0:4], dim=0) dt_loc = torch.sum(dt, dim=0) else: u_input_loc += torch.sum(u[:], dim=0) v_gt_input_loc += torch.sum(v_gt[:], dim=0) quat_tensor_loc += torch.sum(quat_tensor[:, 0:4], dim=0) dt_loc += torch.sum(dt, dim=0) u_input_loc = u_input_loc / nb_data v_gt_input_loc = v_gt_input_loc / nb_data quat_tensor_loc = quat_tensor_loc / nb_data dt_loc = dt_loc / nb_data_dt print(u_input_loc.shape, v_gt_input_loc.shape, quat_tensor_loc.shape, dt_loc.shape) input_loc = torch.cat( (u_input_loc, v_gt_input_loc, quat_tensor_loc, dt_loc), dim=0) #---------------------Compute the standard deviation factor---------------------------------------------------------------- for i in range(0, len(dataset)): #---------------------Load the track---------------------------------------------------------------- dataset_name = dataset.dataset_name(i) print(dataset_name) t, ang_gt, p_gt, v_gt, u = prepare_data(args, dataset, dataset_name) dt = (t[1:] - t[:-1]).unsqueeze(1) ang_gt = correct_ang_gt(ang_gt) #---------------------Smoothing ground truth quaternion-------------------------------------------- quat_tensor = torch.zeros(u.shape[0], 4) for j in range(0, quat_tensor.shape[0]): quat_tensor[j, :] = euler_to_quaternion(ang_gt[j, 0], ang_gt[j, 1], ang_gt[j, 2]) quat_tensor[2:-2, :] = (quat_tensor[0:-4, :] + quat_tensor[1:-3, :] + quat_tensor[2:-2, :] + quat_tensor[3:-1, :] + quat_tensor[4:, :]) / 5 quat_tensor = torch.div( quat_tensor, torch.norm(quat_tensor, dim=1).unsqueeze(1).repeat(1, 4)) if i == 0: u_input_std = ((u[:] - input_loc[0:6])**2).sum(dim=0) v_gt_std = ((v_gt[:] - input_loc[6:9])**2).sum(dim=0) quat_tensor_std = ((quat_tensor[:, 0:4] - input_loc[9:13])**2).sum(dim=0) dt_std = ((dt - input_loc[13])**2).sum(dim=0) else: u_input_std += ((u[:] - input_loc[0:6])**2).sum(dim=0) v_gt_std += ((v_gt[:] - input_loc[6:9])**2).sum(dim=0) quat_tensor_std += ((quat_tensor[:, 0:4] - input_loc[9:13])**2).sum(dim=0) dt_std += ((dt - input_loc[13])**2).sum(dim=0) u_input_std = (u_input_std / nb_data).sqrt() v_gt_std = (v_gt_std / nb_data).sqrt() quat_tensor_std = (quat_tensor_std / nb_data).sqrt() dt_std = (dt_std / nb_data_dt).sqrt() #---------------------Saving the factors in dictionary-------------------------------------------- input_std = torch.cat((u_input_std, v_gt_std, quat_tensor_std, dt_std), dim=0) mondict = {'input_loc': input_loc, 'input_std': input_std} path_data = args.path_standardization_factor torch.save(mondict, path_data) return
def create_dataset_Relative_Kinematic(args, dataset, windows_size): #---------------------Initialization of the tensors-------------------------------------------- number_input = 14 training_data = torch.zeros( (dataset.training_dataset_length, windows_size, number_input)) training_label_data = torch.zeros( (dataset.training_dataset_length, windows_size, 7)) valid_data = torch.zeros( (dataset.valid_dataset_length, windows_size, number_input)) valid_label_data = torch.zeros( (dataset.valid_dataset_length, windows_size, 7)) training_index = 0 valid_index = 0 training_dataset = dataset.datasets_train_filter for i in range(0, len(dataset)): dataset_name = dataset.dataset_name(i) print(dataset_name) #---------------------load the track-------------------------------------------- t, ang_gt, _, v_gt, u = prepare_data(args, dataset, dataset_name) if (not os.path.exists(args.path_normalization_factor)): normalization_all_data(args, dataset) if (not os.path.exists(args.path_standardization_factor)): standardization_all_data(args, dataset) dt = (t[1:] - t[:-1]).float() d_vgt = v_gt[1:] - v_gt[:-1] ang_gt = correct_ang_gt(ang_gt) #---------------------Smoothing ground truth quaternion-------------------------------------------- quat_tensor = torch.zeros(u.shape[0], 4) for j in range(0, quat_tensor.shape[0]): quat_tensor[j, :] = euler_to_quaternion(ang_gt[j, 0], ang_gt[j, 1], ang_gt[j, 2]) quat_tensor[2:-2, :] = (quat_tensor[0:-4, :] + quat_tensor[1:-3, :] + quat_tensor[2:-2, :] + quat_tensor[3:-1, :] + quat_tensor[4:, :]) / 5 quat_tensor = torch.div( quat_tensor, torch.norm(quat_tensor, dim=1).unsqueeze(1).repeat(1, 4)) relative_quaternion = relative_rotation(quat_tensor) relative_quaternion = torch.div( relative_quaternion, torch.norm(relative_quaternion, dim=1).unsqueeze(1).repeat(1, 4)) #---------------------Attribution of the values for the training tensor and testing tensor-------------------------------------------- if (dataset_name in training_dataset): for j in range(windows_size, u.shape[0]): index = j - windows_size training_data[training_index, :, 0:3] = u[index:j, 0:3] training_data[training_index, :, 3:6] = u[index:j, 3:6] training_data[training_index, :, 6:9] = v_gt[index:j, 0:3] training_data[training_index, :, 9:13] = quat_tensor[index:j, 0:4] training_data[training_index, :, 13] = dt[index:j] training_label_data[training_index, :, 0:3] = d_vgt[index:j] training_label_data[training_index, :, 3:7] = relative_quaternion[index:j, 0:4] training_index += 1 else: for j in range(windows_size, u.shape[0]): index = j - windows_size valid_data[valid_index, :, 0:3] = u[index:j, 0:3] valid_data[valid_index, :, 3:6] = u[index:j, 3:6] valid_data[valid_index, :, 6:9] = v_gt[index:j, 0:3] valid_data[valid_index, :, 9:13] = quat_tensor[index:j, 0:4] valid_data[valid_index, :, 13] = dt[index:j] valid_label_data[valid_index, :, 0:3] = d_vgt[index:j] valid_label_data[valid_index, :, 3:7] = relative_quaternion[index:j, 0:4] valid_index += 1 #---------------------Adjusting the size of the tensor and shuffling-------------------------------------------- training_data = training_data[:training_index] training_label_data = training_label_data[:training_index] valid_data = valid_data[:valid_index] valid_label_data = valid_label_data[:valid_index] randomize = shuffle_tensor(training_data.shape[0]) training_data = training_data[randomize] training_label_data = training_label_data[randomize] randomize = shuffle_tensor(valid_data.shape[0]) valid_data = valid_data[randomize] valid_label_data = valid_label_data[randomize] mondict = { 'training_data': training_data, 'training_label_data': training_label_data, 'valid_data': valid_data, 'valid_label_data': valid_label_data } path_data = '../Relative_kinematic_training.p' torch.save(mondict, path_data) return training_data, training_label_data, valid_data, valid_label_data
def normalization_all_data(args, dataset): #---------------------Initialization of the normalization factors---------------------------------------------------------------- nb_data = 0 max_quat = [0, 0, 0, 0] #We initialize with a random low values min_quat = [20, 20, 20, 20] #We initialize with a random high values max_gyr = [0, 0, 0] min_gyr = [20, 20, 20] max_acc = [0, 0, 0] min_acc = [20, 20, 20] max_gyr_dt = [0, 0, 0] min_gyr_dt = [20, 20, 20] max_acc_dt = [0, 0, 0] min_acc_dt = [20, 20, 20] max_acc_rot = [0, 0, 0] min_acc_rot = [20, 20, 20] max_prev_vel = 0 min_prev_vel = 20 max_velocity = [0, 0, 0] min_velocity = [20, 20, 20] for i in range(0, len(dataset)): #---------------------Load the track---------------------------------------------------------------- dataset_name = dataset.dataset_name(i) print(dataset_name) t, ang_gt, p_gt, v_gt, u = prepare_data(args, dataset, dataset_name) dt = (t[1:] - t[:-1]).unsqueeze(1) nb_data += (u.shape[0] - 1) u_dt = u[1:, 0:6].mul(dt[:]) norm_v_gt = torch.norm(v_gt, dim=1) ang_gt = correct_ang_gt(ang_gt) #---------------------Smoothing ground truth quaternion-------------------------------------------- quat_tensor = torch.zeros(u.shape[0], 4) for j in range(0, quat_tensor.shape[0]): quat_tensor[j, :] = euler_to_quaternion(ang_gt[j, 0], ang_gt[j, 1], ang_gt[j, 2]) quat_tensor[2:-2, :] = (quat_tensor[0:-4, :] + quat_tensor[1:-3, :] + quat_tensor[2:-2, :] + quat_tensor[3:-1, :] + quat_tensor[4:, :]) / 5 quat_tensor = torch.div( quat_tensor, torch.norm(quat_tensor, dim=1).unsqueeze(1).repeat(1, 4)) rot_matrix = as_rotation_matrix(quat_tensor[:-1]) macc = mult_acc(rot_matrix, u[1:, 3:6]) #---------------------Get min-max values-------------------------------------------- velocity_max = [v_gt[:, 0].max(), v_gt[:, 1].max(), v_gt[:, 2].max()] velocity_min = [v_gt[:, 0].min(), v_gt[:, 1].min(), v_gt[:, 2].min()] print(velocity_max) print(velocity_min) u_values_max = [ u[:, 0].max(), u[:, 1].max(), u[:, 2].max(), u[:, 3].max(), u[:, 4].max(), u[:, 5].max() ] u_values_min = [ u[:, 0].min(), u[:, 1].min(), u[:, 2].min(), u[:, 3].min(), u[:, 4].min(), u[:, 5].min() ] quat_tensor_max = [ quat_tensor[:, 0].max(), quat_tensor[:, 1].max(), quat_tensor[:, 2].max(), quat_tensor[:, 3].max() ] quat_tensor_min = [ quat_tensor[:, 0].min(), quat_tensor[:, 1].min(), quat_tensor[:, 2].min(), quat_tensor[:, 3].min() ] u_dt_values_max = [ u_dt[:, 0].max(), u_dt[:, 1].max(), u_dt[:, 2].max(), u_dt[:, 3].max(), u_dt[:, 4].max(), u_dt[:, 5].max() ] u_dt_values_min = [ u_dt[:, 0].min(), u_dt[:, 1].min(), u_dt[:, 2].min(), u_dt[:, 3].min(), u_dt[:, 4].min(), u_dt[:, 5].min() ] vel_max = norm_v_gt.max() vel_min = norm_v_gt.min() rotacc_values_max = [ macc[:, 0].max(), macc[:, 1].max(), macc[:, 2].max() ] rotacc_values_min = [ macc[:, 0].min(), macc[:, 1].min(), macc[:, 2].min() ] #print(vel_min,vel_max) #---------------------Update the min and max values-------------------------------------------- if vel_min < min_prev_vel: min_prev_vel = vel_min if vel_max > max_prev_vel: max_prev_vel = vel_max if quat_tensor_min[3] < min_quat[3]: min_quat[3] = quat_tensor_min[3] if quat_tensor_max[3] > max_quat[3]: max_quat[3] = quat_tensor_max[3] for i in range(3): if u_values_min[i] < min_gyr[i]: min_gyr[i] = u_values_min[i] if u_values_max[i] > max_gyr[i]: max_gyr[i] = u_values_max[i] if u_values_min[3 + i] < min_acc[i]: min_acc[i] = u_values_min[3 + i] if u_values_max[3 + i] > max_acc[i]: max_acc[i] = u_values_max[3 + i] if rotacc_values_min[i] < min_acc_rot[i]: min_acc_rot[i] = rotacc_values_min[i] if rotacc_values_max[i] > max_acc_rot[i]: max_acc_rot[i] = rotacc_values_max[i] if u_dt_values_min[i] < min_gyr_dt[i]: min_gyr_dt[i] = u_dt_values_min[i] if u_dt_values_max[i] > max_gyr_dt[i]: max_gyr_dt[i] = u_dt_values_max[i] if u_dt_values_min[3 + i] < min_acc_dt[i]: min_acc_dt[i] = u_dt_values_min[3 + i] if u_dt_values_max[3 + i] > max_acc_dt[i]: max_acc_dt[i] = u_dt_values_max[3 + i] if quat_tensor_min[i] < min_quat[i]: min_quat[i] = quat_tensor_min[i] if quat_tensor_max[i] > max_quat[i]: max_quat[i] = quat_tensor_max[i] if velocity_min[i] < min_velocity[i]: min_velocity[i] = velocity_min[i] if velocity_max[i] > max_velocity[i]: max_velocity[i] = velocity_max[i] #---------------------Saving the factors in dictionary-------------------------------------------- mondict = { 'max_quat': max_quat, 'min_quat': min_quat, 'max_gyr': max_gyr, 'min_gyr': min_gyr, 'max_acc': max_acc, 'min_acc': min_acc, 'max_gyr_dt': max_gyr_dt, 'min_gyr_dt': min_gyr_dt, 'max_prev_vel': max_prev_vel, 'min_prev_vel': min_prev_vel, 'max_acc_rot': max_acc_rot, 'min_acc_rot': min_acc_rot, 'max_acc_dt': max_acc_dt, 'min_acc_dt': min_acc_dt, 'min_velocity': min_velocity, 'max_velocity': max_velocity } path_data = args.path_normalization_factor torch.save(mondict, path_data) print("Test") print(max_velocity, min_velocity) return
(MOUNTH_DATA_ROWS / 2)):int(0.9 * l)] n_est = 0 if 'AdaBoost' in model_name: n_est = int(model_name.replace('AdaBoost_', '')) if 'RandomForest' in model_name: n_est = int(model_name.replace('RandomForest_', '')) feature_selector = Selector(model, df_to_selector, CUTOFF, s2pred, merging, n_est, class_weight) # l = ['soft_upper_dist', 'soft_lower_dist', 'soft_upper_broke', 'soft_lower_broke'] # l = ['NEOUSDT_' + i for i in l] feature_selector.execute() cross_data = cross_data[feature_selector.get_cols()] X_train, X_valid, y_train, y_valid, df_train, df_train_y, df_valid, df_valid_y = \ prepare_data(cross_data, CUTOFF, s2pred, merging, is_features=is_features) print('d') if not is_keras: model.fit(X=X_train, y=y_train) else: # model.fit(X=X_train, y=y_train, epochs=EPOCHS) model.fit(X=X_train, y=np.array(df_train_y['y']), epochs=EPOCHS) df_valid_y['predictions'] = model.predict( df_valid.drop('timestamp', axis=1)) df_train_y['predictions'] = model.predict( df_train.drop('timestamp', axis=1)) if not is_keras: f_imp = [None] + list(
utils.count_params() # If a pre-trained ResNet is required, load the weights. # This must be done AFTER the variables are initialized with sess.run(tf.global_variables_initializer()) if init_fn is not None: init_fn(sess) # Load a previous checkpoint if desired model_checkpoint_name = "model/latest_model_" + args.model + "_" + args.dataset + ".ckpt" if args.continue_training: print('Loaded latest model checkpoint') saver.restore(sess, model_checkpoint_name) # Load the data print("Loading the data ...") train_input_names, train_output_names, val_input_names, val_output_names, test_input_names, test_output_names = utils.prepare_data( dataset_dir=args.dataset) print("\n***** Begin training *****") print("Dataset -->", args.dataset) print("Model -->", args.model) print("Crop Height -->", args.crop_height) print("Crop Width -->", args.crop_width) print("Num Epochs -->", args.num_epochs) print("Batch Size -->", args.batch_size) print("Num Classes -->", num_classes) print("Data Augmentation:") print("\tVertical Flip -->", args.v_flip) print("\tHorizontal Flip -->", args.h_flip) print("\tBrightness Alteration -->", args.brightness) print("\tRotation -->", args.rotation)
def _main(): # parse command line arguments parser = argparse.ArgumentParser() parser.add_argument( '--train_from_checkpoint', type=str, help= "The path to where a previously trained model's weights are stored. To use the default\n\ coco weights, use the path 'model_weights/coco_pretrained_weights.ckpt'. Otherwise, the model\n\ weights will be initialized randomly. ") parser.add_argument( '--class_path', default='utils/coco_classes.txt', type=str, help= 'The path that points towards where the class names for the dataset are stored.\n\ The default path is "utils/coco_classes.txt".') parser.add_argument( '--anchors_path', default='utils/anchors.txt', type=str, help= 'The path that points towards where the anchor values for the model are stored.\n\ The default path is "utils/anchors.txt", which contains anchors trained on the coco dataset.' ) parser.add_argument( '--data_path', default='training_data/image_paths_and_box_info.txt', type=str, help= 'The path that points towards where the training data text file is stored.\n\ The default path is "training_data/image_paths_and_box_info.txt".') parser.add_argument( '--input_height', default=416, type=int, help= 'The input height of the yolov3 model. The height must be a multiple of 32.\n\ The default height is 416.') parser.add_argument( '--input_width', default=416, type=int, help= 'The input width of the yolov3 model. The width must be a mutliple of 32.\n\ The default width is 416.') parser.add_argument( '--batch_size', default=32, type=int, help= 'The training batch size, whose default value is set to 32 images per batch.' ) parser.add_argument( '--max_num_boxes_per_image', default=20, type=int, help= 'The max number of boxes that can be detected within one image. Default is 20.' ) parser.add_argument( '--num_training_epochs', default=150, type=int, help='The number of training epochs. The default is 150.') parser.add_argument( '--learning_rate', default=0.001, type=float, help='The learning rate of the model. The default is 0.001.') parser.add_argument( '--ignore_threshold', default=0.5, type=float, help= 'Impacts how the loss is calculated. Must be between zero and one, and the default is set to 0.5.' ) parser.add_argument( '--train_val_data_split', default=0.9, type=float, help= 'The split between the data that will be used for training and data that will be used\n\ for validation. Default value is 0.9.') parser.add_argument( '--train_save_path', default='model_weights/', help= "The training model's checkpoint save path. The default path is 'model_weights/'." ) parser.add_argument( '--model_name', default='model.ckpt', help= 'The name that should be given to the checkpoint file. The default name is "model.ckpt".' ) parser.add_argument( '--tensorboard_save_path', default='tensorboard/tensorboard_train/', help= 'The path where the event files to be used with tensorboard will be saved at. The default\n\ path is "tensorboard/tensorboard_train/".') parser.add_argument( '--test_model_overfit', nargs='?', default=False, type=str2bool, const=True, help= 'Whether or not to purposefully overfit the model by training it on only one image.\n\ This option is useful in testing out if the loss function is working correctly.' ) parser.add_argument( '--save_every_x_iterations', default=100, type=int, help= "How frequently the model's training weights are saved. The default value is every\n\ 100 iterations.") parser.add_argument( '--log_every_x_iterations', default=5, type=int, help= "How frequently the model's loss is logged for it to be inspected in Tensorboard.\n\ The default value is every 5 iterations.") args = vars(parser.parse_args()) args[ 'data_path'] = '/home/yl/CNN/Yolo/keras-yolo3-fine-tune/dataset/train_label.txt' # read inputs h = args['input_height'] w = args['input_width'] ignore_thresh = args['ignore_threshold'] max_num_boxes_per_image = args['max_num_boxes_per_image'] anchors = get_anchors(args['anchors_path']) lr = args['learning_rate'] num_anchors_per_detector = len(anchors) // 3 num_detectors_per_image = num_anchors_per_detector * (((h / 32) * (w / 32)) + ((h / 16) * (w / 16)) + ((h / 8) * (w / 8))) class_names = get_classes(args['class_path']) num_classes = len(class_names) tb_train_path = args['tensorboard_save_path'] + 'train/' tb_val_path = args['tensorboard_save_path'] + 'val/' training_data, validation_data, batch_size = prepare_data( args['data_path'], args['train_val_data_split'], args['batch_size'], args['test_model_overfit']) tf.reset_default_graph() # build graph with tf.variable_scope('y_true'): y_true_data = tf.placeholder( dtype=tf.float32, shape=[None, num_detectors_per_image, num_classes + 5]) with tf.variable_scope('y_true_boxes'): y_true_box_data = tf.placeholder(dtype=tf.float32, shape=[ None, max_num_boxes_per_image * num_anchors_per_detector, 4 ]) with tf.variable_scope('x_input'): X = tf.placeholder(dtype=tf.float32, shape=[None, h, w, 3]) yolo_outputs = yolo_v3(inputs=X, num_classes=len(class_names), anchors=anchors, h=h, w=w, training=True) # output loss = yolo_v3_loss(yolo_outputs, y_true_data, y_true_box_data, ignore_threshold=ignore_thresh, anchors=anchors, num_classes=num_classes, h=h, w=w, batch_size=batch_size) tf.summary.scalar('loss', loss) global_step = tf.get_variable(name='global_step', trainable=False, initializer=0, dtype=tf.int32) # returns a varlist containing only the vars of the conv layers right before the yolo layers trainable_var_list = tf.trainable_variables() last_layer_var_list = [ i for i in trainable_var_list if i.shape[-1] == (5 + num_classes) * num_anchors_per_detector ] train_op_with_frozen_variables = tf.train.AdamOptimizer( learning_rate=lr).minimize(loss, global_step=global_step, var_list=last_layer_var_list) train_op_with_all_variables = tf.train.AdamOptimizer( learning_rate=lr).minimize(loss, global_step=global_step, var_list=trainable_var_list) summ = tf.summary.merge_all() # info print('--info--') print('model weights will be saved with filename: ', args['model_name']) print('tensorboard event files located at path: ', args['tensorboard_save_path']) # build training loop with tf.Session() as sess: train_writer = tf.summary.FileWriter(tb_train_path, sess.graph) val_writer = tf.summary.FileWriter(tb_val_path) # initialize model weights either randomly or from a saved checkpoint saver = tf.train.Saver() if args['train_from_checkpoint'] is None: print('initializing variables...') sess.run(tf.global_variables_initializer()) else: print('restoring weights from checkpoint: ', args['train_from_checkpoint']) saver.restore(sess, args['train_from_checkpoint']) num_iterations = args['num_training_epochs'] * len(training_data) print('beginning to train the model...') for i in range(num_iterations): input_images, y_true, y_true_boxes = get_training_batch( training_data, anchors, num_classes, batch_size=batch_size, h=h, w=w, random=not args['test_model_overfit']) # For the first epochs, train with the frozen layers. Then, unfreeze the entire graph. if i < num_iterations // 3: sess.run(train_op_with_frozen_variables, feed_dict={ X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes }) else: sess.run(train_op_with_all_variables, feed_dict={ X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes }) if i % args['log_every_x_iterations'] == 0: # write the training loss to tensorboard lt, st = sess.run( [loss, summ], feed_dict={ X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes }) train_writer.add_summary(st, i) #write the validation loss to tensorboard if we are not in overfit mode if not args['test_model_overfit']: input_images, y_true, y_true_boxes = get_training_batch( validation_data, anchors, num_classes, batch_size=batch_size, h=h, w=w, random=not args['test_model_overfit']) lv, sv = sess.run( [loss, summ], feed_dict={ X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes }) val_writer.add_summary(sv, i) print("iteration: " + str(i) + ", training loss: " + str(round(lt, 2)) + ", validation loss: " + str(round(lv, 2))) else: print("iteration: " + str(i) + ", training loss: " + str(round(lt, 2))) if i % args['save_every_x_iterations'] == 0: print('saving model weights at path: ', args['train_save_path']) saver.save( sess, os.path.join(args['train_save_path'], args['model_name']), global_step) train_writer.close() val_writer.close()
def train(self, session, fevals, model, data_loader): #Saver saver = tf.train.Saver(tf.all_variables()) for epoch in xrange(self.tr_config['num_epoch']): start_time = time.time() source_i, target_i = data_loader.next_batch() XX, XXM, YY, YYM = prepare_data(source_i, \ target_i, \ maxlen=self.tr_config['max_steps']) import pdb pdb.set_trace() #Hts_enc, Ots_enc = model._init_states(XX.shape[0], model.TT) pred, logits, Zts = model.fp(XX, XXM, XX.shape[0], model.TT, model.TT) #last_indice = tf.cast(tf.reduce_sum(XXM, 1)-1, 'int32') #Hts_dec, Ots_dec = model._init_states(YY.shape[0], model.TT) #Hts = tf.pack(Hts_enc[0]) #Ots = tf.pack(Ots_enc[0]) #Hts_init = last_relevant(Hts, last_indice) #Ots_init = last_relevant(Ots, last_indice) cost, _ = session.run(fevals, {\ self.inputs : XX,\ self.targets : YY,\ self.input_masks : XXM,\ self.target_masks: YYM}) end_time = time.time() if epoch % 20 == 0: epoch_time = end_time - start_time print("...Epoch %d, Train Cost %f, Time %f" % \ (epoch, cost, epoch_time)) if self.tr_config['summaryF'] and model.summarize is not None: summary_str = session.run(model.summarize, {\ self.inputs : XX,\ self.targets : YY}) model.summary_writer.add_summary(summary_str, epoch) prior = "The future of artificial intelligence " ## Getting sample takes up (relative to GPU) a lot of time in CPU if self.tr_config['sampleF'] and epoch % 200 == 0: ## Gen samples gen_txt = self.get_samples(session, \ model, \ data_loader,\ prior,\ num_steps=100) ## Save the samples to a log file gen_text_epoch = '-'*10+'Epoch '+str(epoch)+'-'*10 + '\n' +\ prior + gen_txt + '\n\n' f = open(self.tr_config['save_gen_text'], 'a') f.write(gen_text_epoch) f.close() ## TODO Checkpoint (not working) #if epoch % 500 == (500 - 1) : # checkpoint_path = self.tr_config['checkpoint_path'] # saver.save(session, checkpoint_path, global_step \ # = epoch * data_loader.num_batches) # print("model saved to {}".format(checkpoint_path)) return cost
def test(opt=train_opt): test_data, test_videos, captions, wordtoix, ixtoword, bias_init_vector = prepare_data( train_opt.video_data_path, train_opt.video_path, train_opt.video_feat_path, 'val', train_opt.dict_path) n_words = len(ixtoword) print('vocabulary size is %d' % n_words) sess = tf.InteractiveSession() model = Video_Caption_Generator( n_words=n_words, dim_embed=train_opt.dim_embed, dim_ctx=train_opt.dim_ctx, dim_hidden=train_opt.dim_hidden, n_caption_lstm_step=train_opt.n_caption_lstm_step, batch_size=train_opt.batch_size, ctx_shape=train_opt.ctx_shape) saver = tf.train.Saver() saver.restore(sess, os.path.join(train_opt.model_path, train_opt.test_graph)) def beam_step(logprobsf, beam_size, hypo_size, t, beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec): ys = np.sort(logprobsf) ys[0] = ys[0,::-1] ix = np.argsort(logprobsf) ix[0] = ix[0,::-1] candidates = [] cols = ys.shape[1] rows = beam_size if t > 0 else 1 for col in range(cols): for row in range(rows): local_logprob = ys[row, col] candidate_logprob = beam_logprobs_sum[row] + local_logprob candidates.append({'c':ix[row,col], 'q':row, 'p':candidate_logprob, 'r':local_logprob}) candidates = sorted(candidates, key=lambda x: -x['p']) new_h = h_rec.copy() new_c = c_rec.copy() if t >= 1: beam_seq_prev = beam_seq[:t].copy() beam_seq_logprobs_prev = beam_seq_logprobs[:t].copy() for vix in range(beam_size): v = candidates[vix] if t >= 1: beam_seq[:t, vix] = beam_seq_prev[:, v['q']] beam_seq_logprobs[:t, vix] = beam_seq_logprobs_prev[:, v['q']] for state_ix in range(len(new_h)): new_h[state_ix][:, vix] = h_rec[state_ix][:, v['q']] new_c[state_ix][:, vix] = c_rec[state_ix][:, v['q']] beam_seq[t, vix] = v['c'] beam_seq_logprobs[t, vix] = v['r'] if beam_logprobs_sum[vix] != -1000: beam_logprobs_sum[vix] = v['p'] else: beam_logprobs_sum[vix] = beam_logprobs_sum[vix] h_rec = new_h c_rec = new_c return beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec, candidates file_index = train_opt.test_graph file_index = file_index[file_index.find('-')+1::] output_txt_gs_fd = open('./result/greedy_'+file_index+'.txt', 'w') output_txt_bs_fd = open('./result/beam_'+file_index+'.txt', 'w') output_txt_bsm_fd = open('./result/beam_mul_'+file_index+'.txt', 'w') for idx, video_feat_path in enumerate(test_videos): start_time = time.time() print(idx, video_feat_path, captions[idx]) video_feat = np.load(video_feat_path)[None,...] beam_seq = np.zeros((train_opt.n_caption_lstm_step, train_opt.beam_size)).astype(np.int32) beam_seq_logprobs = np.zeros((train_opt.n_caption_lstm_step, train_opt.beam_size)) beam_logprobs_sum = np.zeros(train_opt.beam_size) ## to record previous h & c h_rec = np.zeros((train_opt.n_caption_lstm_step, train_opt.dim_hidden, train_opt.beam_size)).astype(np.float32) c_rec = np.zeros((train_opt.n_caption_lstm_step, train_opt.dim_hidden, train_opt.beam_size)).astype(np.float32) done_beams = [] for ind in range(train_opt.n_caption_lstm_step): if ind == 0: context_tf, log_prob_tf, h_tf, c_tf = model.pred_word(ind) log_prob, h, c = sess.run([log_prob_tf, h_tf, c_tf], feed_dict={context_tf:video_feat}) h_rec[ind][:, 0] = h c_rec[ind][:, 0] = c beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec, candidates = beam_step( log_prob, train_opt.beam_size, train_opt.hypo_size, ind, beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec) print('Elapsed time:', str((time.time()-start_time))) else: log_prob = np.zeros((train_opt.beam_size, n_words)).astype(np.float32) for vix in range(train_opt.beam_size): v = candidates[vix] context_tf, log_prob_tf, h_tf, c_tf = model.pred_word( ind, prev_word=v['c'], h=h_rec[ind-1][:,v['q']], c=c_rec[ind-1][:,v['q']]) log_prob[vix,:], h_rec[ind][:,vix], c_rec[ind][:,vix] = sess.run([log_prob_tf, h_tf, c_tf], feed_dict={context_tf:video_feat}) ## suppress UNK tokens in the decoding log_prob[:, 3] -= 1000 beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec, candidates = beam_step( log_prob, train_opt.beam_size, train_opt.hypo_size, ind, beam_seq, beam_seq_logprobs, beam_logprobs_sum, h_rec, c_rec) for beam_ind in range(train_opt.beam_size): if beam_seq[ind, beam_ind] == 2 or ind == train_opt.n_caption_lstm_step-1: final_beam = { 'seq': beam_seq[:, beam_ind], 'logps': beam_seq_logprobs[:, beam_ind], 'p': beam_logprobs_sum[beam_ind] } done_beams.append(copy.deepcopy(final_beam)) beam_logprobs_sum[beam_ind] = -1000 if len(done_beams) >= train_opt.hypo_size: break print('Elapsed time:', str((time.time()-start_time))) context_tf, generated_words_tf, logit_list_tf, alpha_list_tf = model.build_generator() generated_word_index, alpha_list_val = sess.run([generated_words_tf,alpha_list_tf], feed_dict={context_tf:video_feat}) generated_word_index = [x[0] for x in generated_word_index] generated_sentence_gs = convert2sen(ixtoword, generated_word_index) print('greedy search: ', generated_sentence_gs) output_txt_gs_fd.write(video_feat_path + '\n') output_txt_gs_fd.write(generated_sentence_gs + '\n\n') print('Elapsed time:', str((time.time()-start_time))) for beam_ind in range(train_opt.hypo_size): beam = done_beams[beam_ind] generated_sentence = convert2sen(ixtoword, beam['seq']) print('bs {}, p={}: {}'.format(beam_ind, beam['p'], generated_sentence)) output_txt_bsm_fd.write(video_feat_path + '\n') output_txt_bsm_fd.write(generated_sentence + '\n\n') if beam_ind == 1: output_txt_bs_fd.write(video_feat_path + '\n') output_txt_bs_fd.write(generated_sentence + '\n\n') output_txt_gs_fd.close() output_txt_bs_fd.close() output_txt_bsm_fd.close()
-0.145], [-0.627, 0.614, 0.479], [-0.354, 0.772, -0.528], [-0.658, -0.472, -0.586], [0.423, 0.322, -0.847], [0.212, -0.754, -0.622], [0.912, -0.104, 0.398], [-0.311, 0.947, -0.077], [0.679, 0.632, -0.374], [0.135, -0.286, 0.949], [-0.647, 0.230, 0.727], [0.904, 0.397, 0.158], [-0.757, 0.647, -0.087], [0.143, 0.284, 0.948]]) # create gradient table from generated bvecs gtab_generated = ul.create_own_gradient_table(bvecs_generated) # prepare data first_30_images, next_60_images, grad_table_first_30, grad_table_next_60 = ul.prepare_data( test_img, test_bvecs, test_bvals) # with open('results/masked_30.pickle', 'wb') as file_1: # pickle.dump(ul.remove_background(first_30_images), file_1) # with open('results/masked_60.pickle', 'wb') as file_2: # pickle.dump(ul.remove_background(next_60_images), file_2) with open('results/masked_30.pickle', 'rb') as file_1: masked_30 = pickle.load(file_1) with open('results/masked_60.pickle', 'rb') as file_2: masked_60 = pickle.load(file_2) reference_30 = masked_30[:, :, 39, 15] reference_60 = masked_60[:, :, 39, 15] # predicting the signal transformed_all = ul.rotate_3d(masked_60[:, :, 39:41], 5)
def main(args=None): #If args is None, will parse command line arguments #Otherwise args needs to be list with arguments, like # ['--dataset', 'CamVid', '--model', 'FC-DenseNet103'] parser = argparse.ArgumentParser() parser.add_argument('--num_epochs', type=int, default=300, help='Number of epochs to train for') parser.add_argument('--epoch_start_i', type=int, default=0, help='Start counting epochs from this number') parser.add_argument('--checkpoint_step', type=int, default=5, help='How often to save checkpoints (epochs)') parser.add_argument('--validation_step', type=int, default=1, help='How often to perform validation (epochs)') parser.add_argument('--continue_training', type=utils.str2bool, default=False, help='Whether to continue training from a checkpoint') parser.add_argument( '--continue_from', type=str, default='', help='From which checkpoint to continue. Only relevant with darea_call.' ) parser.add_argument('--dataset', type=str, default="CamVid", help='Dataset you are using.') parser.add_argument('--dataset_path', type=str, default="", help='Path to Dataset folder.') parser.add_argument( '--image_suffix', type=str, default='', required=False, help= 'Only files with this extension should be included. You should specify it if some non-image files will be in the same folder' ) parser.add_argument('--crop_height', type=int, default=512, help='Height of cropped input image to network') parser.add_argument('--crop_width', type=int, default=512, help='Width of cropped input image to network') parser.add_argument( '--biased_crop', type=float, default=0, help= 'Probability of making a biased cropped. Biased crops always contain some foreground portion. Only works if one of the classes is named "Background".' ) parser.add_argument( '--downscale_factor', type=float, default=0, required=False, help= 'Shrink image by this factor. E.g. if image is 1024x1024 and downscale_factor is 0.5, downscaled image will be 512x512. This is applied before cropping.' ) parser.add_argument('--batch_size', type=int, default=1, help='Number of images in each batch') parser.add_argument( '--num_val_images', type=int, default=20, help='The number of images to used for validations. If -1 -> use all') parser.add_argument( '--h_flip', type=utils.str2bool, default=False, help= 'Whether to randomly flip the image horizontally for data augmentation' ) parser.add_argument( '--v_flip', type=utils.str2bool, default=False, help= 'Whether to randomly flip the image vertically for data augmentation') parser.add_argument( '--brightness', type=float, default=None, help= 'Whether to randomly change the image brightness for data augmentation. Specifies the max bightness change as a factor between 0.0 and 1.0. For example, 0.1 represents a max brightness change of 10%% (+-).' ) parser.add_argument( '--rotation', type=float, default=None, help= 'DOES NOT WORK! Whether to randomly rotate the image for data augmentation. Specifies the max rotation angle in degrees.' ) parser.add_argument('--rotation_perpendicular', type=utils.str2bool, default=False, help='Randomly rotates by 0, 90, 180 or 270 degrees') parser.add_argument( '--model', type=str, default="FC-DenseNet103", help= 'The model you are using. See model_builder.py for supported models') parser.add_argument( '--frontend', type=str, default="ResNet101", help= 'The frontend you are using. See frontend_builder.py for supported models' ) parser.add_argument( '--save_best', type=utils.str2bool, default=False, help='Saves model with smallest loss rather than last model') parser.add_argument('--learn_rate', type=float, default=0.0001, help='The learning rate') parser.add_argument( '--chkpt_prefix', type=str, default='', help= 'Prefix in front of checkpoint (intended for distinguishing same models ran with different parameters)' ) parser.add_argument( '--darea_call', type=utils.str2bool, default=False, required=False, help='Set to true when you call it from Darea software') parser.add_argument( '--save_path', type=str, default='checkpoints', required=False, help='Name of saved model. Only used when darea_call is true.') parser.add_argument('--makePlots', type=utils.str2bool, default=True, required=False, help='Whether plots should be made') if args is None: args = parser.parse_args() else: args = parser.parse_args(args) if args.darea_call: os.chdir(os.path.dirname(os.path.realpath(__file__))) # Get the names of the classes so we can record the evaluation results class_names_list, label_values = helpers.get_label_info( os.path.join(args.dataset_path, args.dataset, "class_dict.csv")) class_names_string = ', '.join(class_names_list) if 'Background' not in class_names_list: args.biased_crop = 0 backgroundValue = None else: backgroundValue = label_values[class_names_list.index('Background')] num_classes = len(label_values) config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) # Compute your softmax cross entropy loss net_input = tf.placeholder(tf.float32, shape=[ None, None, None, 3 ]) #Try setting to 1 for grayscale, think theres no other changes needed. net_output = tf.placeholder(tf.float32, shape=[None, None, None, num_classes]) network, init_fn = model_builder.build_model(model_name=args.model, frontend=args.frontend, net_input=net_input, num_classes=num_classes, crop_width=args.crop_width, crop_height=args.crop_height, is_training=True) loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2(logits=network, labels=net_output)) opt = tf.train.RMSPropOptimizer( learning_rate=args.learn_rate, decay=0.995).minimize( loss, var_list=[var for var in tf.trainable_variables()]) saver = tf.train.Saver(max_to_keep=500) sess.run(tf.global_variables_initializer()) utils.count_params() # If a pre-trained ResNet is required, load the weights. # This must be done AFTER the variables are initialized with sess.run(tf.global_variables_initializer()) if init_fn is not None: init_fn(sess) # Load a previous checkpoint if desired if args.darea_call: if args.continue_training and args.continue_from: model_checkpoint_name = '../../deepLearning/checkpoints/' + args.continue_from + '.ckpt' if os.path.isfile(model_checkpoint_name + '.index'): print('Loading latest model checkpoint...') saver.restore(sess, model_checkpoint_name) print('successfully loaded {}'.format(model_checkpoint_name)) else: print('Specified checkpoint {} not found. Starting fresh one'. format(os.path.abspath(model_checkpoint_name))) if args.save_path: model_checkpoint_name = os.path.join(args.save_path, args.dataset + '.ckpt') else: if args.continue_training: if args.continue_from: model_checkpoint_name = args.continue_from if not model_checkpoint_name.endswith('.ckpt'): model_checkpoint_name += '.ckpt' else: model_checkpoint_name = os.path.join( args.save_path, "latest_model_" + args.chkpt_prefix + args.model + "_" + args.dataset + ".ckpt") if os.path.isfile(model_checkpoint_name + '.index'): print('Loading latest model checkpoint...') saver.restore(sess, model_checkpoint_name) print('successfully loaded {}'.format(model_checkpoint_name)) else: print('{} not found. Starting a fresh training'.format( args.continue_from)) model_checkpoint_name = os.path.join( args.save_path, "latest_model_" + args.chkpt_prefix + args.model + "_" + args.dataset + ".ckpt") # Load the data print("Loading the data ...") train_input_names,train_output_names, val_input_names, val_output_names, test_input_names, test_output_names = \ utils.prepare_data(dataset_dir=os.path.join(args.dataset_path,args.dataset),image_suffix=args.image_suffix) print("\n***** Begin training *****") print("Dataset -->", args.dataset) print("Model -->", args.model) if args.downscale_factor: print("Downscale Factor -->", args.downscale_factor) print("Crop Height -->", args.crop_height) print("Crop Width -->", args.crop_width) print("Num Epochs -->", args.num_epochs) print("Batch Size -->", args.batch_size) print("Num Classes -->", num_classes) print("Learn Rate -->", args.learn_rate) print("Validation Step -->", args.validation_step) print("Data Augmentation:") print("\tVertical Flip -->", args.v_flip) print("\tHorizontal Flip -->", args.h_flip) print("\tBrightness Alteration -->", args.brightness) print("\tRotation -->", args.rotation) print("\tPerpendicular Rotation -->", args.rotation_perpendicular) print("", flush=True) avg_loss_per_epoch = [] avg_scores_per_epoch = [] avg_iou_per_epoch = [] if args.num_val_images == -1: #Use all validation images if so wanted by users val_indices = range(0, len(val_input_names)) else: # Which validation images do we want val_indices = [] num_vals = min(args.num_val_images, len(val_input_names)) # Set random seed to make sure models are validated on the same validation images. # So you can compare the results of different models more intuitively. random.seed(16) val_indices = random.sample(range(0, len(val_input_names)), num_vals) #Copy class file with same name as checkpoint shutil.copyfile( os.path.join(args.dataset_path, args.dataset, "class_dict.csv"), os.path.splitext(model_checkpoint_name)[0] + '.classes') # Do the training here for epoch in range(args.epoch_start_i, args.num_epochs): current_losses = [] cnt = 0 # Equivalent to shuffling id_list = np.random.permutation(len(train_input_names)) num_iters = int(np.floor(len(id_list) / args.batch_size)) st = time.time() epoch_st = time.time() for i in range(num_iters): # st=time.time() input_image_batch = [] output_image_batch = [] # Collect a batch of images for j in range(args.batch_size): index = i * args.batch_size + j id = id_list[index] input_image = utils.load_image(train_input_names[id]) output_image = utils.load_image(train_output_names[id]) with tf.device('/cpu:0'): input_image, output_image = data_augmentation( input_image, output_image, args, backgroundValue) if 0: #Debugging: try: os.mkdir('imagesinTrain') 'kj' except: pass try: os.mkdir('imagesinTrain/%04d' % (epoch)) except: pass file_name = utils.filepath_to_name( train_input_names[id]) file_name = os.path.splitext(file_name)[0] cv2.imwrite( "%s/%04d/%s_im.png" % ("imagesinTrain", epoch, file_name), cv2.cvtColor(np.uint8(input_image), cv2.COLOR_RGB2BGR)) cv2.imwrite( "%s/%04d/%s_gt.png" % ("imagesinTrain", epoch, file_name), cv2.cvtColor(np.uint8(output_image), cv2.COLOR_RGB2BGR)) # Prep the data. Make sure the labels are in one-hot format input_image = np.float32(input_image) / 255.0 output_image = np.float32( helpers.one_hot_it(label=output_image, label_values=label_values)) if 0: #Debugging: try: os.mkdir('imagesinTrain') except: pass try: os.mkdir('imagesinTrain/%04d' % (epoch)) except: pass file_name = utils.filepath_to_name( train_input_names[id]) file_name = os.path.splitext(file_name)[0] print("%s/%04d/%s_im.png" % ("imagesinTrain", epoch, file_name)) cv2.imwrite( "%s/%04d/%s_im.png" % ("imagesinTrain", epoch, file_name), cv2.cvtColor(np.uint8(input_image), cv2.COLOR_RGB2BGR)) #cv2.imwrite("%s/%04d/%s_gt.png"%("imagesinTrain",epoch, file_name),cv2.cvtColor(np.uint8(output_image), cv2.COLOR_RGB2BGR)) input_image_batch.append( np.expand_dims(input_image, axis=0)) output_image_batch.append( np.expand_dims(output_image, axis=0)) if args.batch_size == 1: input_image_batch = input_image_batch[0] output_image_batch = output_image_batch[0] else: input_image_batch = np.squeeze( np.stack(input_image_batch, axis=1)) output_image_batch = np.squeeze( np.stack(output_image_batch, axis=1)) # Do the training _, current, output_image = sess.run([opt, loss, network], feed_dict={ net_input: input_image_batch, net_output: output_image_batch }) current_losses.append(current) cnt = cnt + args.batch_size if cnt % 25 == 0: string_print = "Epoch = %d Count = %d Current_Loss = %.4f Time = %.2f" % ( epoch, cnt, current, time.time() - st) utils.LOG(string_print) st = time.time() if 0: #For Debugging output_image = np.array(output_image[0, :, :, :]) output_image = helpers.reverse_one_hot(output_image) out_vis_image = helpers.colour_code_segmentation( output_image, label_values) cv2.imwrite( "%s/%04d/%s_pred.png" % ("imagesinTrain", epoch, file_name), cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR)) mean_loss = np.mean(current_losses) avg_loss_per_epoch.append(mean_loss) # Create directories if needed if not os.path.isdir("%s/%04d" % ("checkpoints", epoch)): os.makedirs("%s/%04d" % ("checkpoints", epoch)) # If latest checkpoint should be saved, save now to same file name, if not args.save_best: print("Saving latest checkpoint") saver.save(sess, model_checkpoint_name) if val_indices != 0 and epoch % args.checkpoint_step == 0: print("Saving checkpoint for this epoch") saver.save(sess, "%s/%04d/model.ckpt" % (args.save_path, epoch)) if epoch % args.validation_step == 0: if epoch == 0: best_avg_iou = 0 print("Performing validation") target = open("%s/%04d/val_scores.csv" % ("checkpoints", epoch), 'w') target.write( "val_name, avg_accuracy, precision, recall, f1 score, mean iou, %s\n" % (class_names_string)) scores_list = [] class_scores_list = [] precision_list = [] recall_list = [] f1_list = [] iou_list = [] # Do the validation on a small set of validation images for ind in val_indices: input_image = utils.load_image(val_input_names[ind]) gt = utils.load_image(val_output_names[ind]) if args.downscale_factor and args.downscale_factor != 1: dim = (int(input_image.shape[0] * args.downscale_factor), int(input_image.shape[1] * args.downscale_factor)) input_image = cv2.resize(input_image, dim, interpolation=cv2.INTER_CUBIC) gt = cv2.resize(gt, dim, interpolation=cv2.INTER_NEAREST) #input_image, gt = data_augmentation(input_image, gt, args) gt = helpers.reverse_one_hot( helpers.one_hot_it(gt, label_values)) crop_height = args.crop_height crop_width = args.crop_width if input_image.shape[0] > crop_height or input_image.shape[ 1] > crop_width: #rectangle in bottom right corner smaller than cropped im will not be used nrCroppingsY = input_image.shape[0] // crop_height nrCroppingsX = input_image.shape[1] // crop_width output_image = np.zeros([ nrCroppingsY * crop_height, nrCroppingsX * crop_width ]) gt = gt[0:nrCroppingsY * crop_height, 0:nrCroppingsX * crop_width] for yi in range(nrCroppingsY): row = np.zeros( [crop_height, nrCroppingsX * crop_width]) for xi in range(nrCroppingsX): inputIm = input_image[yi * crop_height:(1 + yi) * crop_height, xi * crop_width:(1 + xi) * crop_width, :] inputIm = np.expand_dims(np.float32(inputIm) / 255.0, axis=0) out = sess.run(network, feed_dict={net_input: inputIm}) out = np.array(out[0, :, :, :]) out = helpers.reverse_one_hot(out) row[:, xi * crop_width:(1 + xi) * crop_width] = out output_image[yi * crop_height:(1 + yi) * crop_height, :] = row # st = time.time() else: input_image = np.expand_dims(np.float32(input_image) / 255.0, axis=0) output_image = sess.run(network, feed_dict={net_input: input_image}) output_image = np.array(output_image[0, :, :, :]) output_image = helpers.reverse_one_hot(output_image) out_vis_image = helpers.colour_code_segmentation( output_image, label_values) accuracy, class_accuracies, prec, rec, f1, iou, class_iou = utils.evaluate_segmentation( pred=output_image, label=gt, num_classes=num_classes) file_name = utils.filepath_to_name(val_input_names[ind]) target.write("%s, %f, %f, %f, %f, %f" % (file_name, accuracy, prec, rec, f1, iou)) for item in class_accuracies: target.write(", %f" % (item)) target.write("\n") scores_list.append(accuracy) class_scores_list.append(class_accuracies) precision_list.append(prec) recall_list.append(rec) f1_list.append(f1) iou_list.append(iou) gt = helpers.colour_code_segmentation(gt, label_values) file_name = os.path.basename(val_input_names[ind]) file_name = os.path.splitext(file_name)[0] cv2.imwrite( "%s/%04d/%s_pred.png" % ("checkpoints", epoch, file_name), cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR)) cv2.imwrite( "%s/%04d/%s_gt.png" % ("checkpoints", epoch, file_name), cv2.cvtColor(np.uint8(gt), cv2.COLOR_RGB2BGR)) target.close() avg_score = np.mean(scores_list) class_avg_scores = np.mean(class_scores_list, axis=0) avg_scores_per_epoch.append(avg_score) avg_precision = np.mean(precision_list) avg_recall = np.mean(recall_list) avg_f1 = np.mean(f1_list) avg_iou = np.mean(iou_list) avg_iou_per_epoch.append(avg_iou) print("\nAverage validation accuracy for epoch # %04d = %f" % (epoch, avg_score)) print("Average per class validation accuracies for epoch # %04d:" % (epoch)) for index, item in enumerate(class_avg_scores): print("%s = %f" % (class_names_list[index], item)) print("Validation precision = ", avg_precision) print("Validation recall = ", avg_recall) print("Validation F1 score = ", avg_f1) print("Validation IoU score = ", avg_iou, flush=True) if args.save_best and avg_iou > best_avg_iou: #Save model if best model is wanted and it is the best or if it is first evaluation print("Saving best checkpoint with iou {}".format(avg_iou)) saver.save(sess, model_checkpoint_name) best_avg_iou = avg_iou #Save an info file with open(model_checkpoint_name[:-5] + '.info', 'w') as f: f.write('Epoch\t{}\nValidation IoU score\t{}'.format( epoch, avg_iou)) epoch_time = time.time() - epoch_st remain_time = epoch_time * (args.num_epochs - 1 - epoch) m, s = divmod(remain_time, 60) h, m = divmod(m, 60) if s != 0: train_time = "Remaining training time = %d hours %d minutes %d seconds\n" % ( h, m, s) else: train_time = "Remaining training time : Training completed.\n" utils.LOG(train_time) scores_list = [] with open(model_checkpoint_name[:-5] + '.info', 'a+') as f: #Save some info on filesizes f.write('\nimageSize\t{}'.format([args.crop_height, args.crop_width])) f.write('\nTraining completed\t{}'.format( datetime.datetime.now().strftime("%d-%b-%Y %H:%M:%S"))) if not args.darea_call and args.makePlots: fig1, ax1 = plt.subplots(figsize=(11, 8)) ax1.plot(range(epoch + 1), avg_scores_per_epoch) ax1.set_title("Average validation accuracy vs epochs") ax1.set_xlabel("Epoch") ax1.set_ylabel("Avg. val. accuracy") plt.savefig('accuracy_vs_epochs.png') plt.clf() fig2, ax2 = plt.subplots(figsize=(11, 8)) ax2.plot(range(epoch + 1), avg_loss_per_epoch) ax2.set_title("Average loss vs epochs") ax2.set_xlabel("Epoch") ax2.set_ylabel("Current loss") plt.savefig('loss_vs_epochs.png') plt.clf() fig3, ax3 = plt.subplots(figsize=(11, 8)) ax3.plot(range(epoch + 1), avg_iou_per_epoch) ax3.set_title("Average IoU vs epochs") ax3.set_xlabel("Epoch") ax3.set_ylabel("Current IoU") plt.savefig('iou_vs_epochs.png')
def _main(): # parse command line arguments parser = argparse.ArgumentParser() parser.add_argument( '--train-from-checkpoint', type=str, help="the path to where a previously trained model's weights are stored") parser.add_argument('--class-path', default='utils/coco_classes.txt', type=str, help='the path that points to the class names for the dataset') parser.add_argument( '--anchors-path', default='utils/anchors.txt', type=str, help='the path that points to the anchor values for the models') parser.add_argument( '--data-path', default='data/image_paths_and_box_info.txt', type=str, help='the path that points to the training data text file') parser.add_argument( '--input-height', default=416, type=int, help='the input height of the yolov3 model. The height must be a multiple of 32') parser.add_argument( '--input-width', default=416, type=int, help='The input width of the yolov3 model. The width must be a multiple of 32') parser.add_argument( '--batch-size', default=32, type=int, help='the training batch size') parser.add_argument( '--max-num-boxes-per-image', default=20, type=int, help='the max number of boxes that can be detected within one image') parser.add_argument( '--num-training-epochs', default=150, type=int, help='the number of training epochs') parser.add_argument( '--learning-rate', default=0.001, type=float, help='the learning rate') parser.add_argument( '--ignore-threshold', default=0.5, type=float, help='impacts how the loss is calculated. Must be between zero and one') parser.add_argument( '--train-val-data-split', default=0.9, type=float, help='the split between the data that will be used for training and data that will be used\n\ for validation') parser.add_argument( '--train-save-path', default='model-weights/', help="the training model's checkpoint save path") parser.add_argument( '--model-name', default='model.ckpt', help='the name that should be given to the checkpoint file') parser.add_argument( '--tensorboard-save-path', default='tb-logs/tb-train/', help='the path where the event files to be used with tensorboard will be saved at') parser.add_argument( '--test-model-overfit', nargs='?', default=False, type=str2bool, const=True, help='this option is useful in testing out if the loss function is working correctly') parser.add_argument( '--save-iterations', default=100, type=int, help="how frequently the model's training weights are saved") parser.add_argument( '--log-iterations', default=5, type=int, help="how frequently the model's loss is logged for it to be inspected in Tensorboard") args = parser.parse_args() # args info print('[i] checkpoint: ', args.train_from_checkpoint) print('[i] path of class file: ', args.class_path) print('[i] path of anchors file: ', args.anchors_path) # read inputs h = args.input_height w = args.input_width ignore_thresh = args.ignore_threshold max_num_boxes_per_image = args.max_num_boxes_per_image anchors = get_anchors(args.anchors_path) lr = args.learning_rate num_anchors_per_detector = len(anchors) // 3 num_detectors_per_image = num_anchors_per_detector * ( ((h / 32) * (w / 32)) + ((h / 16) * (w / 16)) + ((h / 8) * (w / 8))) class_names = get_classes(args.class_path) num_classes = len(class_names) # tensorboard path tb_train_path = args.tensorboard_save_path + 'train/' tb_val_path = args.tensorboard_save_path + 'val/' training_data, validation_data, batch_size = prepare_data( args.data_path, args.train_val_data_split, args.batch_size, args.test_model_overfit) tf.reset_default_graph() # build graph with tf.variable_scope('y_true'): y_true_data = tf.placeholder(dtype=tf.float32, shape=[None, num_detectors_per_image, num_classes + 5]) with tf.variable_scope('y_true_boxes'): y_true_box_data = tf.placeholder(dtype=tf.float32, shape=[None, max_num_boxes_per_image * num_anchors_per_detector, 4]) with tf.variable_scope('x_input'): X = tf.placeholder(dtype=tf.float32, shape=[None, h, w, 3]) yolo_outputs = yolo_v3(inputs=X, num_classes=len(class_names), anchors=anchors, h=h, w=w, training=True) # output loss = yolo_v3_loss(yolo_outputs, y_true_data, y_true_box_data, ignore_threshold=ignore_thresh, anchors=anchors, num_classes=num_classes, h=h, w=w, batch_size=batch_size) tf.summary.scalar('loss', loss) global_step = tf.get_variable(name='global_step', trainable=False, initializer=0, dtype=tf.int32) # returns a varlist containing only the vars of the conv layers right before the yolo layers trainable_var_list = tf.trainable_variables() last_layer_var_list = [i for i in trainable_var_list if i.shape[-1] == (5 + num_classes) * num_anchors_per_detector] train_op_with_frozen_variables = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss, global_step=global_step, var_list=last_layer_var_list) train_op_with_all_variables = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss, global_step=global_step, var_list=trainable_var_list) summ = tf.summary.merge_all() # info print('-------info-------') print('[i] model weights will be saved with filename: ', args.model_name) print('[i] tensorboard event files located at path: ', args.tensorboard_save_path) # build training loop with tf.Session() as sess: train_writer = tf.summary.FileWriter(tb_train_path, sess.graph) val_writer = tf.summary.FileWriter(tb_val_path) # initialize model weights either randomly or from a saved checkpoint saver = tf.train.Saver() if args['train_from_checkpoint'] is None: print('[i] initializing variables...') sess.run(tf.global_variables_initializer()) else: print('[i] restoring weights from checkpoint: ', args.train_from_checkpoint) saver.restore(sess, args.train_from_checkpoint) num_iterations = args.num_epochs * len(training_data) print('[i] beginning to train the model...') for i in range(num_iterations): input_images, y_true, y_true_boxes = get_training_batch(training_data, anchors, num_classes, batch_size=batch_size, h=h, w=w, random=not args.test_model_overfit) # For the first epochs, train with the frozen layers. Then, unfreeze the entire graph. if i < num_iterations // 3: sess.run(train_op_with_frozen_variables, feed_dict={X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes}) else: sess.run(train_op_with_all_variables, feed_dict={X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes}) if i % args.log_iterations == 0: # write the training loss to tensorboard lt, st = sess.run([loss, summ], feed_dict={X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes}) train_writer.add_summary(st, i) # write the validation loss to tensorboard if we are not in overfit mode if not args.test_model_overfit: input_images, y_true, y_true_boxes = get_training_batch(validation_data, anchors, num_classes, batch_size=batch_size, h=h, w=w, random=not args.test_model_overfit) lv, sv = sess.run([loss, summ], feed_dict={X: input_images, y_true_data: y_true, y_true_box_data: y_true_boxes}) val_writer.add_summary(sv, i) print( "| iteration: " + str(i) + ", training loss: " + str(round(lt, 2)) + ", validation loss: " + str( round(lv, 2))) else: print("| iteration: " + str(i) + ", training loss: " + str(round(lt, 2))) if i % args.save_iterations == 0: print('[i] saving model weights at path: ', args.train_save_path) saver.save(sess, os.path.join(args.train_save_path, args.model_name), global_step) print('################################################################################') train_writer.close() val_writer.close()
def simple_test_Velonet(args, dataset): #---------------------Load the NN model-------------------------------------------- model = torch.load('./saved_NN/neural_network.p') windows_size = 11 device = torch.device('cpu') network = get_Relative_Kinematic().to(device) network.load_state_dict(model.get('model_state_dict')) fig_x = 4 fig_y = 4 fig, axes = plt.subplots(fig_x, ncols=fig_y) compute_mean = torch.zeros((windows_size, 1)) network.eval() for i in range(0, len(dataset)): dataset_name = dataset.dataset_name(i) print("Test filter on sequence: " + dataset_name) #---------------------Create a multi display-------------------------------------------- if (i % (fig_x * fig_y) == 0 and i != 0): plt.subplots_adjust(left=0.03, bottom=0.03, right=0.97, top=0.97, wspace=0.12, hspace=0.34) plt.show() fig, axes = plt.subplots(fig_x, ncols=fig_y) #---------------------Load the data-------------------------------------------- t, ang_gt, _, v_gt, u = prepare_data(args, dataset, dataset_name) dt = (t[1:] - t[:-1]).float() d_vgt = v_gt[1:] - v_gt[:-1] ang_gt = correct_ang_gt(ang_gt) #---------------------Smoothing ground truth quaternion-------------------------------------------- quat_tensor = torch.zeros(u.shape[0], 4) for j in range(0, quat_tensor.shape[0]): quat_tensor[j, :] = euler_to_quaternion(ang_gt[j, 0], ang_gt[j, 1], ang_gt[j, 2]) quat_tensor[2:-2, :] = (quat_tensor[0:-4, :] + quat_tensor[1:-3, :] + quat_tensor[2:-2, :] + quat_tensor[3:-1, :] + quat_tensor[4:, :]) / 5 quat_tensor = torch.div( quat_tensor, torch.norm(quat_tensor, dim=1).unsqueeze(1).repeat(1, 4)) relative_quaternion = relative_rotation(quat_tensor) relative_quaternion = torch.div( relative_quaternion, torch.norm(relative_quaternion, dim=1).unsqueeze(1).repeat(1, 4)) test_dataset = torch.zeros(u.shape[0], windows_size, 14) label_dataset = torch.zeros(u.shape[0], windows_size, 7) data_index = 0 #---------------------Prepare the testing data-------------------------------------------- for j in range(windows_size, u.shape[0]): index = j - windows_size test_dataset[data_index, :, 0:3] = u[index:j, 0:3] test_dataset[data_index, :, 3:6] = u[index:j, 3:6] test_dataset[data_index, :, 6:9] = v_gt[index:j, 0:3] test_dataset[data_index, :, 9:13] = quat_tensor[index:j, 0:4] test_dataset[data_index, :, 13] = dt[index:j] label_dataset[data_index, :, 0:3] = d_vgt[index:j] label_dataset[data_index, :, 3:7] = relative_quaternion[index:j, 0:4] data_index += 1 test_dataset = test_dataset[:data_index] label_dataset = label_dataset[:data_index] #test_dataset = quick_norm(test_dataset) test_dataset = quick_std(test_dataset) feat_gyr = (test_dataset[:, :, 0:3]).to(device) feat_acc = (test_dataset[:, :, 3:6]).to(device) feat_dt = (test_dataset[:, :, 13].unsqueeze(2)).to(device) feat_prev_quat = (test_dataset[:, :, 9:13]).to(device) #---------------------Use the NN to predict relative quaternion and relative velocity-------------------------------------------- with torch.no_grad(): measurements_vel, measurements_ori = network( feat_gyr, feat_acc, feat_dt, feat_prev_quat) measurements_vel = measurements_vel[:, 5, :] measurements_ori = measurements_ori[:, 5, :] label_dataset = label_dataset[:, 5, :] measurements_ori = torch.div( measurements_ori, torch.norm(measurements_ori, dim=1).unsqueeze(1).repeat(1, 4)) #---------------------Compute the prediction errors-------------------------------------------- conj_target = conjugate(label_dataset[:, 3:7]) qprod = hamilton_product(measurements_ori, conj_target) w, x, y, z = torch.chunk(qprod, 4, dim=-1) quat_error = 2 * torch.sum(torch.abs(torch.cat( (x, y, z), axis=-1)), dim=1) mse_vel = torch.mean(torch.pow( (label_dataset[:, 0:3] - measurements_vel), 2), dim=0) mse_ori = torch.mean(torch.pow( (label_dataset[:, 3:7] - measurements_ori), 2), dim=0) print('Error prediction') print(mse_vel, mse_ori, torch.mean(quat_error, dim=0)) measurements_vel = measurements_vel.detach().numpy() measurements_ori = measurements_ori.detach().numpy() #---------------------Multi display-------------------------------------------- x_ax = int((i % (fig_x * fig_y)) / fig_x) y_ax = i % fig_y axes[x_ax, y_ax].grid(axis='y') if dataset_name in dataset.datasets_train_filter: axes[x_ax, y_ax].set_title(dataset_name, color='red') else: axes[x_ax, y_ax].set_title(dataset_name) axes[x_ax, y_ax].plot(label_dataset[:, 0], color='orange') axes[x_ax, y_ax].plot(label_dataset[:, 1], color='blue') axes[x_ax, y_ax].plot(label_dataset[:, 2], color='green') axes[x_ax, y_ax].plot(measurements_vel[:, 0], color='red') axes[x_ax, y_ax].plot(measurements_vel[:, 1], color='c') axes[x_ax, y_ax].plot(measurements_vel[:, 2], color='purple') axes[x_ax, y_ax].set_xlabel('frame') axes[x_ax, y_ax].set_ylabel('Relative velocity') plt.subplots_adjust(left=0.03, bottom=0.03, right=0.97, top=0.97, wspace=0.12, hspace=0.34) plt.show()
def main(): args = get_args() prepare_data() word_vocab_config = { "<UNK>": 0, "<PAD>": 1, "<start>": 2, "<end>": 3, "insert_start": "<SOS>", "insert_end": "<EOS>", "tokenization": "nltk", "specials": ["<UNK>", "<PAD>", "<SOS>", "<EOS>"], "embedding_root": os.path.join(args.app_path, "data", "embedding", "word"), "embedding_type": "glove.840B", "embedding_dim": 300 } print("Reading Vocab", flush=True) char_vocab_config = word_vocab_config.copy() char_vocab_config["embedding_root"] = os.path.join(args.app_path, "data", "embedding", "char") char_vocab_config["embedding_type"] = "glove_char.840B" # TODO: build vocab out of dataset # build vocab itos, stoi, wv_vec = read_vocab(word_vocab_config) itoc, ctoi, cv_vec = read_vocab(char_vocab_config) char_embedding_config = { "embedding_weights": cv_vec, "padding_idx": word_vocab_config["<UNK>"], "update": args.update_char_embedding, "bidirectional": args.bidirectional, "cell_type": "gru", "output_dim": 300 } word_embedding_config = { "embedding_weights": wv_vec, "padding_idx": word_vocab_config["<UNK>"], "update": args.update_word_embedding } sentence_encoding_config = { "hidden_size": args.hidden_size, "num_layers": args.num_layers, "bidirectional": True, "dropout": args.dropout, } pair_encoding_config = { "hidden_size": args.hidden_size, "num_layers": args.num_layers, "bidirectional": args.bidirectional, "dropout": args.dropout, "gated": True, "mode": "GRU", "rnn_cell": torch.nn.GRUCell, "attn_size": args.attention_size, "residual": args.residual } self_matching_config = { "hidden_size": args.hidden_size, "num_layers": args.num_layers, "bidirectional": args.bidirectional, "dropout": args.dropout, "gated": True, "mode": "GRU", "rnn_cell": torch.nn.GRUCell, "attn_size": args.attention_size, "residual": args.residual } pointer_config = { "hidden_size": args.hidden_size, "num_layers": args.num_layers, "dropout": args.dropout, "residual": args.residual, "rnn_cell": torch.nn.GRUCell } print("DEBUG Mode is ", "On" if args.debug else "Off", flush=True) train_cache = "./data/cache/SQuAD%s.pkl" % ("_debug" if args.debug else "") dev_cache = "./data/cache/SQuAD_dev%s.pkl" % ("_debug" if args.debug else "") train_json = args.train_json dev_json = args.dev_json train = read_dataset(train_json, itos, stoi, itoc, ctoi, train_cache, args.debug) dev = read_dataset(dev_json, itos, stoi, itoc, ctoi, dev_cache, args.debug, split="dev") dev_dataloader = dev.get_dataloader(args.batch_size_dev) train_dataloader = train.get_dataloader(args.batch_size, shuffle=True, pin_memory=args.pin_memory) trainer = Trainer(args, train_dataloader, dev_dataloader, char_embedding_config, word_embedding_config, sentence_encoding_config, pair_encoding_config, self_matching_config, pointer_config) trainer.train(args.epoch_num)
def train(train_opt): train_data, captions, wordtoix, ixtoword, bias_init_vector = prepare_data( train_opt.video_data_path, train_opt.video_path, train_opt.video_feat_path, 'train', train_opt.dict_path) n_words = len(wordtoix) print('vocabulary size is %d' % n_words) sess = tf.InteractiveSession() caption_generator = Video_Caption_Generator( n_words=n_words, dim_embed=train_opt.dim_embed, dim_ctx=train_opt.dim_ctx, dim_hidden=train_opt.dim_hidden, n_caption_lstm_step=train_opt.n_caption_lstm_step, batch_size=train_opt.batch_size, ctx_shape=train_opt.ctx_shape, bias_init_vector=bias_init_vector) loss_m, context_m, sentence_m, mask_m = caption_generator.build_model_m() loss_n, context_n, sentence_n, mask_n = caption_generator.build_model_n() loss_cl, context_cl, sentence_cl_p, sentence_cl_n, mask_cl, h_seq_pos_cl, h_seq_neg_cl = caption_generator.build_model_cl( ) saver = tf.train.Saver(max_to_keep=50) train_op_m = tf.train.AdamOptimizer( train_opt.learning_rate).minimize(loss_m) train_op_n = tf.train.AdamOptimizer(train_opt.learning_rate).minimize( loss_n, var_list=[ caption_generator.lstm_W_n, caption_generator.lstm_U_n, caption_generator.lstm_b_n, caption_generator.decode_lstm_W_n, caption_generator.decode_lstm_W_n ]) train_op_cl = tf.train.AdamOptimizer(train_opt.learning_rate).minimize( loss_cl, var_list=[ caption_generator.lstm_W_m, caption_generator.lstm_U_m, caption_generator.lstm_b_m ]) tf.global_variables_initializer().run() #--------------------------------------------------------------------------- # first train p_m with pure positive samples for epochs_m pretrained_model_m = os.path.join(train_opt.model_path, train_opt.pretrained_graph_m) if os.path.isfile(pretrained_model_m): print("Target model has already been trained...") else: print("Start training target model...") for epoch in range(train_opt.epochs_m + 1): index = list(train_data.index) np.random.shuffle(index) train_data_m = train_data.ix[index] current_train_data = train_data_m.groupby('video_feat_path').apply( lambda x: x.iloc[0]) current_train_data = current_train_data.reset_index(drop=True) for start, end in zip( range(0, len(current_train_data), train_opt.batch_size), range(train_opt.batch_size, len(current_train_data), train_opt.batch_size)): start_time = time.time() current_caption_matrix, current_caption_masks, current_feats = fetch_batch_data( current_train_data, start, end, train_opt, wordtoix, 'target') _, loss_value = sess.run( [train_op_m, loss_m], feed_dict={ context_m: current_feats, sentence_m: current_caption_matrix, mask_m: current_caption_masks }) print('idx:', start, 'Epoch:', epoch, 'loss:', loss_value, 'Elapsed time:', str((time.time() - start_time))) if np.mod(epoch, 100) == 0 and epoch > 0: print("Epoch ", epoch, "is done. Saving the model...") saver.save(sess, os.path.join(train_opt.model_path, 'model'), global_step=epoch) #--------------------------------------------------------------------------- # then train p_n with pure negative samples for epochs_n, load the pre-trained p_m model pretrained_model_n = os.path.join(train_opt.model_path, train_opt.pretrained_graph_n) if os.path.isfile(pretrained_model_n): print("Reference model has already been trained...") else: print("Start training reference model...") saver = tf.train.import_meta_graph(pretrained_model_m) saver.restore(sess, os.path.join(train_opt.model_path, 'model-900')) for epoch in range(train_opt.epochs_m + 1, train_opt.epochs_m + train_opt.epochs_n + 1): current_train_data = generate_neg_samples(data=train_data, phase='reference') for start, end in zip( range(0, len(current_train_data), train_opt.batch_size), range(train_opt.batch_size, len(current_train_data), train_opt.batch_size)): start_time = time.time() current_caption_matrix, current_caption_masks, current_feats = fetch_batch_data( current_train_data, start, end, train_opt, wordtoix, 'reference') _, loss_value = sess.run( [train_op_n, loss_n], feed_dict={ context_n: current_feats, sentence_n: current_caption_matrix, mask_n: current_caption_masks }) print('idx:', start, 'Epoch:', epoch, 'loss:', loss_value, 'Elapsed time:', str((time.time() - start_time))) if np.mod(epoch, 100) == 0 and epoch > 0: print("Epoch ", epoch, "is done. Saving the model...") saver.save(sess, os.path.join(train_opt.model_path, 'model'), global_step=epoch) #--------------------------------------------------------------------------- # next finetune p_m with p_n model for epochs_cl, load pre-trained p_m and p_n model pretrained_model_cl = os.path.join(train_opt.model_path, train_opt.pretrained_graph_cl) if os.path.isfile(pretrained_model_cl): print("CL model has already been trained...") else: print("Start training CL model...") saver = tf.train.import_meta_graph( os.path.join(train_opt.model_path, 'model-1800.meta')) saver.restore(sess, os.path.join(train_opt.model_path, 'model-1800')) for epoch in range( train_opt.epochs_m + train_opt.epochs_n + 1, train_opt.epochs_m + train_opt.epochs_n + train_opt.epochs_cl + 1): current_train_data_p, current_train_data_n = generate_cl_samples( train_data, train_opt.neg_K) for start, end in zip( range(0, len(current_train_data_p), train_opt.batch_size * train_opt.neg_K), range(train_opt.batch_size * train_opt.neg_K, len(current_train_data_p), train_opt.batch_size * train_opt.neg_K)): start_time = time.time() current_caption_matrix_p, current_caption_masks_p, current_feats = fetch_batch_data( current_train_data_p, start, end, train_opt, wordtoix, 'CL') current_caption_matrix_n, current_caption_masks_n, _ = fetch_batch_data( current_train_data_n, start, end, train_opt, wordtoix, 'CL') _, loss_value = sess.run( [train_op_cl, loss_cl], feed_dict={ context_cl: current_feats, sentence_cl_p: current_caption_matrix_p, sentence_cl_n: current_caption_matrix_n, mask_cl: current_caption_masks_p }) print('idx:', start, 'Epoch:', epoch, 'loss:', loss_value, 'Elapsed time:', str((time.time() - start_time))) if np.mod(epoch, 10) == 0 and epoch > 0: print("Epoch ", epoch, "is done. Saving the model...") saver.save(sess, os.path.join(train_opt.model_path, 'model'), global_step=epoch)
network, _ = model_builder.build_model(args.model, net_input=net_input, num_classes=num_classes, crop_width=args.crop_width, crop_height=args.crop_height, is_training=False) sess.run(tf.global_variables_initializer()) print('Loading model checkpoint weights ...') saver=tf.train.Saver(max_to_keep=1000) saver.restore(sess, args.checkpoint_path) # Load the data print("Loading the data ...") train_input_names,train_output_names, val_input_names, val_output_names, test_input_names, test_output_names = utils.prepare_data(dataset_dir=os.path.join(args.dataset_path,args.dataset)) # Create directories if needed if args.output_path!='' and not os.path.isdir(args.output_path): os.mkdir(args.output_path) target=open(os.path.join(args.output_path, 'test.csv'),'w') target.write("test_name,test_accuracy,precision,recall,f1 score,mean iou,%s,%s" % (','.join(class_names_acc), ','.join(class_names_iou))) scores_list = [] class_scores_list = [] precision_list = [] recall_list = [] f1_list = [] iou_list = [] run_times_list = []