def main(): learning_rate = 5e-2 num_epoch = 20 save_dir = 'models' h5file = '/home/song/workspace/datasets/recog-alzheimer/train/train_pre_data.h5' csvfile = '/home/song/workspace/datasets/recog-alzheimer/train/train_pre_label.csv' dataset = TrainData(h5file, csvfile, img_size=48) train_loader = DataLoader(dataset, batch_size=5, shuffle=True) solver = Solver(learning_rate, num_epoch, save_dir) solver.train(train_loader)
def select_train_data(filename, methods=['POST']): global train_data try: with open("./datasets/" + filename['filename'], 'r') as file: train_data = TrainData() train_data.from_metadata(json.load(file)) socketio.emit('selected_train_data', 1) except Exception as ex: print(ex) socketio.emit('selected_train_data', 0)
def traintest(): if not os.path.isfile(train_data_pickle): # training data train_features, train_labels = get_features(['train_data'], "train") traindata = TrainData(train_features, train_labels) with open(train_data_pickle, mode='wb') as f: pickle.dump(traindata, f) else: print("loading: %s" % (train_data_pickle)) with open(train_data_pickle, mode='rb') as f: traindata = pickle.load(f) train_features = traindata.train_inputs train_labels = traindata.train_targets if not os.path.isfile(test_data_pickle): # testing data test_features, _ = get_features(['test_data'], "test") testdata = TestData(test_features) with open(test_data_pickle, mode='wb') as f: pickle.dump(testdata, f) else: print("loading: %s" % (test_data_pickle)) with open(test_data_pickle, mode='rb') as f: testdata = pickle.load(f) test_features = testdata.test_inputs train_labels = one_hot_encode(train_labels) n_dim = train_features.shape[1] print("input dim: %s" % (n_dim)) # random train and test sets. ''' train_test_split = np.random.rand(len(train_features)) < 0.80 Xtr = train_features[train_test_split] Ytr = train_labels[train_test_split] Xte = train_features[~train_test_split] Yte = train_labels[~train_test_split] ''' Xtr = train_features Ytr = train_labels Xte = test_features knn(n_dim, Xtr, Ytr, Xte)
def add_train(): status = 100 # = fail try: content = request.files['file'] # get the file name without its extension filename = (content.filename).replace(".json", "") content = content.read().decode('utf-8') content = json.loads(content) global train_data train_data = TrainData(filename) if not train_data.filter_json(content) : message = "incorrect data structure (1)" print(message) return make_response(jsonify({"message" : message}), status) if not train_data.is_correct() : message = "incorrect data structure (2)" print(message) return make_response(jsonify({"message" : message}), status) if not train_data.metadata() : message = "failed to create metadata" print(message) return make_response(jsonify({"message" : message}), status) # save metafile train_data.create_metafile() status = 200 return make_response(jsonify({"message" : "JSON received"}), status) #200 = success except Exception as ex: print(ex) return make_response(jsonify({"message" : str(ex)}), status)
def main(): input_dir = "/amit/kaggle/tgs" output_dir = "/artifacts" image_size_target = 128 batch_size = 32 epochs_to_train = 300 bce_loss_weight_gamma = 0.98 sgdr_min_lr = 0.0001 # 0.0001, 0.001 sgdr_max_lr = 0.001 # 0.001, 0.03 sgdr_cycle_epochs = 20 sgdr_cycle_epoch_prolongation = 3 sgdr_cycle_end_patience = 3 train_abort_epochs_without_improval = 30 ensemble_model_count = 3 swa_epoch_to_start = 30 model_dir = sys.argv[1] if len(sys.argv) > 1 else None train_data = TrainData(input_dir) train_set = TrainDataset(train_data.train_set_df, image_size_target, augment=True) train_set_data_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=8) val_set = TrainDataset(train_data.val_set_df, image_size_target, augment=False) val_set_data_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=2) if model_dir: model = create_model(pretrained=False).to(device) model.load_state_dict(torch.load("{}/model.pth".format(model_dir), map_location=device)) else: model = create_model(pretrained=True).to(device) torch.save(model.state_dict(), "{}/model.pth".format(output_dir)) swa_model = create_model(pretrained=False).to(device) print("train_set_samples: %d, val_set_samples: %d" % (len(train_set), len(val_set))) global_val_precision_best_avg = float("-inf") global_swa_val_precision_best_avg = float("-inf") sgdr_cycle_val_precision_best_avg = float("-inf") epoch_iterations = len(train_set) // batch_size # optimizer = optim.SGD(model.parameters(), lr=sgdr_max_lr, weight_decay=0, momentum=0.9, nesterov=True) optimizer = optim.Adam(model.parameters(), lr=sgdr_max_lr) lr_scheduler = CosineAnnealingLR(optimizer, T_max=sgdr_cycle_epochs, eta_min=sgdr_min_lr) optim_summary_writer = SummaryWriter(log_dir="{}/logs/optim".format(output_dir)) train_summary_writer = SummaryWriter(log_dir="{}/logs/train".format(output_dir)) val_summary_writer = SummaryWriter(log_dir="{}/logs/val".format(output_dir)) swa_val_summary_writer = SummaryWriter(log_dir="{}/logs/swa_val".format(output_dir)) sgdr_iterations = 0 sgdr_reset_count = 0 batch_count = 0 epoch_of_last_improval = 0 sgdr_next_cycle_end_epoch = sgdr_cycle_epochs + sgdr_cycle_epoch_prolongation swa_update_count = 0 ensemble_model_index = 0 for model_file_path in glob.glob("{}/model-*.pth".format(output_dir)): model_file_name = os.path.basename(model_file_path) model_index = int(model_file_name.replace("model-", "").replace(".pth", "")) ensemble_model_index = max(ensemble_model_index, model_index + 1) print('{"chart": "best_val_precision", "axis": "epoch"}') print('{"chart": "val_precision", "axis": "epoch"}') print('{"chart": "val_loss", "axis": "epoch"}') print('{"chart": "sgdr_reset", "axis": "epoch"}') print('{"chart": "precision", "axis": "epoch"}') print('{"chart": "loss", "axis": "epoch"}') print('{"chart": "swa_val_precision", "axis": "epoch"}') print('{"chart": "swa_val_loss", "axis": "epoch"}') train_start_time = time.time() criterion = nn.BCEWithLogitsLoss() for epoch in range(epochs_to_train): epoch_start_time = time.time() model.train() train_loss_sum = 0.0 train_precision_sum = 0.0 train_step_count = 0 for batch in train_set_data_loader: images, masks, mask_weights = \ batch[0].to(device, non_blocking=True), \ batch[1].to(device, non_blocking=True), \ batch[2].to(device, non_blocking=True) lr_scheduler.step(epoch=min(sgdr_cycle_epochs, sgdr_iterations / epoch_iterations)) optimizer.zero_grad() prediction_logits = model(images) predictions = torch.sigmoid(prediction_logits) criterion.weight = mask_weights loss = criterion(prediction_logits, masks) loss.backward() optimizer.step() train_loss_sum += loss.item() train_precision_sum += np.mean(precision_batch(predictions, masks)) sgdr_iterations += 1 train_step_count += 1 batch_count += 1 optim_summary_writer.add_scalar("lr", get_learning_rate(optimizer), batch_count + 1) train_loss_avg = train_loss_sum / train_step_count train_precision_avg = train_precision_sum / train_step_count val_loss_avg, val_precision_avg = evaluate(model, val_set_data_loader, criterion) model_improved_within_sgdr_cycle = val_precision_avg > sgdr_cycle_val_precision_best_avg if model_improved_within_sgdr_cycle: torch.save(model.state_dict(), "{}/model-{}.pth".format(output_dir, ensemble_model_index)) sgdr_cycle_val_precision_best_avg = val_precision_avg model_improved = val_precision_avg > global_val_precision_best_avg ckpt_saved = False if model_improved: torch.save(model.state_dict(), "{}/model.pth".format(output_dir)) global_val_precision_best_avg = val_precision_avg ckpt_saved = True swa_model_improved = False if epoch + 1 >= swa_epoch_to_start: if model_improved_within_sgdr_cycle: swa_update_count += 1 moving_average(swa_model, model, 1.0 / swa_update_count) bn_update(train_set_data_loader, swa_model) swa_model_improved = val_precision_avg > global_swa_val_precision_best_avg if swa_model_improved: torch.save(swa_model.state_dict(), "{}/swa_model.pth".format(output_dir)) global_swa_val_precision_best_avg = val_precision_avg if model_improved or swa_model_improved: epoch_of_last_improval = epoch sgdr_reset = False if (epoch + 1 >= sgdr_next_cycle_end_epoch) and (epoch - epoch_of_last_improval >= sgdr_cycle_end_patience): sgdr_iterations = 0 sgdr_next_cycle_end_epoch = epoch + 1 + sgdr_cycle_epochs + sgdr_cycle_epoch_prolongation ensemble_model_index += 1 sgdr_cycle_val_precision_best_avg = float("-inf") sgdr_reset_count += 1 sgdr_reset = True swa_val_loss_avg, swa_val_precision_avg = evaluate(swa_model, val_set_data_loader, criterion) optim_summary_writer.add_scalar("sgdr_reset", sgdr_reset_count, epoch + 1) train_summary_writer.add_scalar("loss", train_loss_avg, epoch + 1) train_summary_writer.add_scalar("precision", train_precision_avg, epoch + 1) val_summary_writer.add_scalar("loss", val_loss_avg, epoch + 1) val_summary_writer.add_scalar("precision", val_precision_avg, epoch + 1) swa_val_summary_writer.add_scalar("loss", swa_val_loss_avg, epoch + 1) swa_val_summary_writer.add_scalar("precision", swa_val_precision_avg, epoch + 1) epoch_end_time = time.time() epoch_duration_time = epoch_end_time - epoch_start_time print( "[%03d/%03d] %ds, lr: %.6f, loss: %.3f, val_loss: %.3f|%.3f, prec: %.3f, val_prec: %.3f|%.3f, ckpt: %d, rst: %d" % ( epoch + 1, epochs_to_train, epoch_duration_time, get_learning_rate(optimizer), train_loss_avg, val_loss_avg, swa_val_loss_avg, train_precision_avg, val_precision_avg, swa_val_precision_avg, int(ckpt_saved), int(sgdr_reset)), flush=True) print('{"chart": "best_val_precision", "x": %d, "y": %.3f}' % (epoch + 1, global_val_precision_best_avg)) print('{"chart": "val_precision", "x": %d, "y": %.3f}' % (epoch + 1, val_precision_avg)) print('{"chart": "val_loss", "x": %d, "y": %.3f}' % (epoch + 1, val_loss_avg)) print('{"chart": "sgdr_reset", "x": %d, "y": %.3f}' % (epoch + 1, sgdr_reset_count)) print('{"chart": "precision", "x": %d, "y": %.3f}' % (epoch + 1, train_precision_avg)) print('{"chart": "loss", "x": %d, "y": %.3f}' % (epoch + 1, train_loss_avg)) print('{"chart": "swa_val_precision", "x": %d, "y": %.3f}' % (epoch + 1, swa_val_precision_avg)) print('{"chart": "swa_val_loss", "x": %d, "y": %.3f}' % (epoch + 1, swa_val_loss_avg)) if sgdr_reset and sgdr_reset_count >= ensemble_model_count and epoch - epoch_of_last_improval >= train_abort_epochs_without_improval: print("early abort") break optim_summary_writer.close() train_summary_writer.close() val_summary_writer.close() train_end_time = time.time() print() print("Train time: %s" % str(datetime.timedelta(seconds=train_end_time - train_start_time))) eval_start_time = time.time() print() print("evaluation of the training model") model.load_state_dict(torch.load("{}/model.pth".format(output_dir), map_location=device)) analyze(Ensemble([model]), train_data.val_set_df, use_tta=False) analyze(Ensemble([model]), train_data.val_set_df, use_tta=True) score_to_model = {} ensemble_model_candidates = glob.glob("{}/model-*.pth".format(output_dir)) ensemble_model_candidates.append("{}/swa_model.pth".format(output_dir)) for model_file_path in ensemble_model_candidates: model_file_name = os.path.basename(model_file_path) m = create_model(pretrained=False).to(device) m.load_state_dict(torch.load(model_file_path, map_location=device)) val_loss_avg, val_precision_avg = evaluate(m, val_set_data_loader, criterion) print("ensemble '%s': val_loss=%.3f, val_precision=%.3f" % (model_file_name, val_loss_avg, val_precision_avg)) if len(score_to_model) < ensemble_model_count or min(score_to_model.keys()) < val_precision_avg: del score_to_model[min(score_to_model.keys())] score_to_model[val_precision_avg] = m ensemble_models = list(score_to_model.values()) for ensemble_model in ensemble_models: val_loss_avg, val_precision_avg = evaluate(ensemble_model, val_set_data_loader, criterion) print("ensemble: val_loss=%.3f, val_precision=%.3f" % (val_loss_avg, val_precision_avg)) model = Ensemble(ensemble_models) mask_threshold_global, mask_threshold_per_cc = analyze(model, train_data.val_set_df, use_tta=True) eval_end_time = time.time() print() print("Eval time: %s" % str(datetime.timedelta(seconds=eval_end_time - eval_start_time))) print() print("submission preparation") submission_start_time = time.time() test_data = TestData(input_dir) calculate_predictions(test_data.df, model, use_tta=True) calculate_prediction_masks(test_data.df, mask_threshold_global) print() print(test_data.df.groupby("predictions_cc").agg({"predictions_cc": "count"})) write_submission(test_data.df, "prediction_masks", "{}/{}".format(output_dir, "submission.csv")) write_submission(test_data.df, "prediction_masks_best", "{}/{}".format(output_dir, "submission_best.csv")) submission_end_time = time.time() print() print("Submission time: %s" % str(datetime.timedelta(seconds=submission_end_time - submission_start_time)))
def main(): args = argparser.parse_args() print("Arguments:") for arg in vars(args): print(" {}: {}".format(arg, getattr(args, arg))) print() input_dir = args.input_dir output_dir = args.output_dir base_model_dir = args.base_model_dir image_size = args.image_size augment = args.augment use_dummy_image = args.use_dummy_image use_progressive_image_sizes = args.use_progressive_image_sizes progressive_image_size_min = args.progressive_image_size_min progressive_image_size_step = args.progressive_image_size_step progressive_image_epoch_step = args.progressive_image_epoch_step batch_size = args.batch_size batch_iterations = args.batch_iterations test_size = args.test_size train_on_unrecognized = args.train_on_unrecognized num_category_shards = args.num_category_shards category_shard = args.category_shard eval_train_mapk = args.eval_train_mapk mapk_topk = args.mapk_topk num_shard_preload = args.num_shard_preload num_shard_loaders = args.num_shard_loaders num_workers = args.num_workers pin_memory = args.pin_memory epochs_to_train = args.epochs lr_scheduler_type = args.lr_scheduler lr_patience = args.lr_patience lr_min = args.lr_min lr_max = args.lr_max lr_min_decay = args.lr_min_decay lr_max_decay = args.lr_max_decay optimizer_type = args.optimizer loss_type = args.loss loss2_type = args.loss2 loss2_start_sgdr_cycle = args.loss2_start_sgdr_cycle model_type = args.model patience = args.patience sgdr_cycle_epochs = args.sgdr_cycle_epochs sgdr_cycle_epochs_mult = args.sgdr_cycle_epochs_mult sgdr_cycle_end_prolongation = args.sgdr_cycle_end_prolongation sgdr_cycle_end_patience = args.sgdr_cycle_end_patience max_sgdr_cycles = args.max_sgdr_cycles use_extended_stroke_channels = model_type in [ "cnn", "residual_cnn", "fc_cnn", "hc_fc_cnn" ] train_data = TrainData(data_dir=input_dir, shard=0, test_size=test_size, train_on_unrecognized=train_on_unrecognized, confusion_set=None, num_category_shards=num_category_shards, category_shard=category_shard) val_set = TrainDataset(train_data.val_set_df, image_size, use_extended_stroke_channels, False, use_dummy_image) val_set_data_loader = \ DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=pin_memory) categories = train_data.categories criterion = create_criterion(loss_type, len(categories)) model_dir = "/storage/models/quickdraw/seresnext50" model = load_ensemble_model(model_dir, 3, val_set_data_loader, criterion, model_type, image_size, len(categories)) cs_entry_categories = [ 'angel', 'arm', 'bat', 'bathtub', 'bottlecap', 'hospital', 'police car', 'spider', 'sun', 'tent', 'triangle', 'windmill' ] cs_categories = read_confusion_set( "/storage/models/quickdraw/seresnext50_confusion/confusion_set_{}.txt". format(0)) predicted_words = predict(model, val_set_data_loader, categories, tta=True) prediction_mask = [] cs_entry_match_count = 0 cs_match_count = 0 for i, p in enumerate(predicted_words): predicted_word = p.split(" ")[0].replace("_", " ") cond1 = predicted_word in cs_entry_categories prediction_mask.append(cond1) if cond1 and categories[train_data.val_set_df["category"] [i]] in cs_entry_categories: cs_entry_match_count += 1 if cond1 and categories[train_data.val_set_df["category"] [i]] in cs_categories: cs_match_count += 1 print("matched {} of {}".format(sum(prediction_mask), len(prediction_mask)), flush=True) print("cs_entry_match_count: {}".format(cs_entry_match_count), flush=True) print("cs_match_count: {}".format(cs_match_count), flush=True) df = { "category": train_data.val_set_df["category"][prediction_mask], "drawing": train_data.val_set_df["drawing"][prediction_mask] } val_set = TrainDataset(df, image_size, use_extended_stroke_channels, False, use_dummy_image) val_set_data_loader = \ DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=pin_memory) loss_avg, mapk_avg, accuracy_top1_avg, accuracy_top3_avg, accuracy_top5_avg, accuracy_top10_avg = \ evaluate(model, val_set_data_loader, criterion, mapk_topk) print( "loss: {:.3f}, map@3: {:.3f}, acc@1: {:.3f}, acc@3: {:.3f}, acc@5: {:.3f}, acc@10: {:.3f}" .format(loss_avg, mapk_avg, accuracy_top1_avg, accuracy_top3_avg, accuracy_top5_avg, accuracy_top10_avg), flush=True) predicted_words = predict(model, val_set_data_loader, categories, tta=True) match_count = 0 for i, p in enumerate(predicted_words): predicted_word = p.split(" ")[0].replace("_", " ") true_word = categories[df["category"][i]] if predicted_word == true_word: match_count += 1 if predicted_word not in cs_entry_categories: print("predicted unexpected word: '{}'".format(predicted_word), flush=True) print("acc@1: {}".format(match_count / len(predicted_words)), flush=True) criterion = create_criterion(loss_type, len(cs_categories)) model_dir = "/storage/models/quickdraw/seresnext50_cs_0" model = load_ensemble_model(model_dir, 3, val_set_data_loader, criterion, "seresnext50_cs", image_size, len(cs_categories)) predicted_words = predict(model, val_set_data_loader, cs_categories, tta=True) match_count = 0 for i, p in enumerate(predicted_words): predicted_word = p.split(" ")[0].replace("_", " ") true_word = categories[df["category"][i]] if predicted_word == true_word: match_count += 1 print("acc@1: {}".format(match_count / len(predicted_words)), flush=True)
def create_datasets(args): train_data = TrainData(args.train_path) dev_data = ValidData(args.validation_path) return dev_data, train_data
def train(): if not os.path.isfile(train_data_pickle): # trainig data train_features, train_labels = features(['fold0', 'fold1', 'fold2']) traindata = TrainData(train_features, train_labels) with open(train_data_pickle, mode='wb') as f: pickle.dump(traindata, f) else: print("loading: %s" % (train_data_pickle)) with open(train_data_pickle, mode='rb') as f: traindata = pickle.load(f) train_features = traindata.train_inputs train_labels = traindata.train_targets if not os.path.isfile(test_data_pickle): test_features, test_labels = features(['fold3']) testdata = TestData(test_features, test_labels) with open(test_data_pickle, mode='wb') as f: pickle.dump(testdata, f) else: print("loading: %s" % (test_data_pickle)) with open(test_data_pickle, mode='rb') as f: testdata = pickle.load(f) test_features = testdata.test_inputs test_labels = testdata.test_targets # TODO change to use train and test train_labels = one_hot_encode(train_labels) test_labels = one_hot_encode(test_labels) # random train and test sets. train_test_split = np.random.rand(len(train_features)) < 0.70 train_x = train_features[train_test_split] train_y = train_labels[train_test_split] test_x = train_features[~train_test_split] test_y = train_labels[~train_test_split] n_dim = train_features.shape[1] print("input dim: %s" % (n_dim)) # create placeholder X = tf.placeholder(tf.float32, [None, n_dim]) Y = tf.placeholder(tf.float32, [None, FLAGS.num_classes]) # build graph logits = model.inference(X, n_dim) weights = tf.all_variables() saver = tf.train.Saver(weights) # create loss loss = model.loss(logits, Y) tf.scalar_summary('loss', loss) accracy = model.accuracy(logits, Y) tf.scalar_summary('test accuracy', accracy) # train operation train_op = model.train_op(loss) # variable initializer init = tf.initialize_all_variables() # get Session sess = tf.Session() # sumary merge and writer merged = tf.merge_all_summaries() train_writer = tf.train.SummaryWriter(FLAGS.summaries_dir) # initialize sess.run(init) for step in xrange(MAX_STEPS): t_pred = sess.run(tf.argmax(logits, 1), feed_dict={X: train_features}) t_true = sess.run(tf.argmax(train_labels, 1)) print("train samples pred: %s" % t_pred[:30]) print("train samples target: %s" % t_true[:30]) print('Train accuracy: ', sess.run(accracy, feed_dict={ X: train_x, Y: train_y })) for epoch in xrange(training_epochs): summary, logits_val, _, loss_val = sess.run( [merged, logits, train_op, loss], feed_dict={ X: train_x, Y: train_y }) train_writer.add_summary(summary, step) print("step:%d, loss: %s" % (step, loss_val)) y_pred = sess.run(tf.argmax(logits, 1), feed_dict={X: test_x}) y_true = sess.run(tf.argmax(test_y, 1)) print("test samples pred: %s" % y_pred[:10]) print("test samples target: %s" % y_true[:10]) accracy_val = sess.run([accracy], feed_dict={X: test_x, Y: test_y}) # print('Test accuracy: ', accracy_val) # train_writer.add_summary(accracy_val, step) p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average='micro') print("F-score: %s" % f) if step % 1000 == 0: saver.save(sess, FLAGS.ckpt_dir, global_step=step)
def main(): args = argparser.parse_args() log_args(args) input_dir = args.input_dir output_dir = args.output_dir base_model_dir = args.base_model_dir image_size = args.image_size crop_images = args.crop_images augment = args.augment use_progressive_image_sizes = args.use_progressive_image_sizes progressive_image_size_min = args.progressive_image_size_min progressive_image_size_step = args.progressive_image_size_step progressive_image_epoch_step = args.progressive_image_epoch_step batch_size = args.batch_size batch_iterations = args.batch_iterations num_workers = args.num_workers pin_memory = args.pin_memory epochs_to_train = args.epochs lr_scheduler_type = args.lr_scheduler lr_patience = args.lr_patience lr_min = args.lr_min lr_max = args.lr_max lr_min_decay = args.lr_min_decay lr_max_decay = args.lr_max_decay optimizer_type = args.optimizer loss_type = args.loss focal_loss_gamma = args.focal_loss_gamma use_class_weights = args.use_class_weights use_weighted_sampling = args.use_weighted_sampling model_type = args.model patience = args.patience sgdr_cycle_epochs = args.sgdr_cycle_epochs sgdr_cycle_epochs_mult = args.sgdr_cycle_epochs_mult sgdr_cycle_end_prolongation = args.sgdr_cycle_end_prolongation sgdr_cycle_end_patience = args.sgdr_cycle_end_patience max_sgdr_cycles = args.max_sgdr_cycles if optimizer_type == "adam": lr_scheduler_type = "adam" progressive_image_sizes = list( range(progressive_image_size_min, image_size + 1, progressive_image_size_step)) train_data = TrainData(input_dir) train_set = TrainDataset(train_data.train_set_df, input_dir, 28, image_size, crop_images, augment) balance_weights, balance_class_weights = calculate_balance_weights( train_data.df, train_data.train_set_df, 28) train_set_sampler = WeightedRandomSampler(balance_weights, len(balance_weights)) train_set_data_loader = DataLoader( train_set, batch_size=batch_size, shuffle=False if use_weighted_sampling else True, sampler=train_set_sampler if use_weighted_sampling else None, num_workers=num_workers, pin_memory=pin_memory) val_set = TrainDataset(train_data.val_set_df, input_dir, 28, image_size, crop_images, False) val_set_data_loader = \ DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=pin_memory) if base_model_dir: for base_file_path in glob.glob("{}/*.pth".format(base_model_dir)): shutil.copyfile( base_file_path, "{}/{}".format(output_dir, os.path.basename(base_file_path))) model = create_model(type=model_type, num_classes=28).to(device) model.load_state_dict( torch.load("{}/model.pth".format(output_dir), map_location=device)) optimizer = create_optimizer(optimizer_type, model, lr_max) if os.path.isfile("{}/optimizer.pth".format(output_dir)): try: optimizer.load_state_dict( torch.load("{}/optimizer.pth".format(output_dir))) adjust_initial_learning_rate(optimizer, lr_max) adjust_learning_rate(optimizer, lr_max) except: log("Failed to load the optimizer weights") else: model = create_model(type=model_type, num_classes=28).to(device) optimizer = create_optimizer(optimizer_type, model, lr_max) torch.save(model.state_dict(), "{}/model.pth".format(output_dir)) ensemble_model_index = 0 for model_file_path in glob.glob("{}/model-*.pth".format(output_dir)): model_file_name = os.path.basename(model_file_path) model_index = int( model_file_name.replace("model-", "").replace(".pth", "")) ensemble_model_index = max(ensemble_model_index, model_index + 1) epoch_iterations = ceil(len(train_set) / batch_size) log("train_set_samples: {}, val_set_samples: {}".format( len(train_set), len(val_set))) log() global_val_score_best_avg = float("-inf") sgdr_cycle_val_score_best_avg = float("-inf") lr_scheduler = CosineAnnealingLR(optimizer, T_max=sgdr_cycle_epochs, eta_min=lr_min) optim_summary_writer = SummaryWriter( log_dir="{}/logs/optim".format(output_dir)) train_summary_writer = SummaryWriter( log_dir="{}/logs/train".format(output_dir)) val_summary_writer = SummaryWriter( log_dir="{}/logs/val".format(output_dir)) current_sgdr_cycle_epochs = sgdr_cycle_epochs sgdr_next_cycle_end_epoch = current_sgdr_cycle_epochs + sgdr_cycle_end_prolongation sgdr_iterations = 0 sgdr_cycle_count = 0 batch_count = 0 epoch_of_last_improval = 0 lr_scheduler_plateau = \ ReduceLROnPlateau(optimizer, mode="max", min_lr=lr_min, patience=lr_patience, factor=0.5, threshold=1e-4) lr_scheduler_step = StepLR(optimizer, step_size=10, gamma=0.1) log('{"chart": "best_val_score", "axis": "epoch"}') log('{"chart": "val_score", "axis": "epoch"}') log('{"chart": "val_loss", "axis": "epoch"}') log('{"chart": "sgdr_cycle", "axis": "epoch"}') log('{"chart": "score", "axis": "epoch"}') log('{"chart": "loss", "axis": "epoch"}') log('{"chart": "lr_scaled", "axis": "epoch"}') log('{"chart": "mem_used", "axis": "epoch"}') log('{"chart": "epoch_time", "axis": "epoch"}') train_start_time = time.time() loss_weight = CLASS_WEIGHTS_TENSOR if use_class_weights else None criterion = create_criterion(loss_type, loss_weight, focal_loss_gamma) for epoch in range(epochs_to_train): epoch_start_time = time.time() log("memory used: {:.2f} GB".format(psutil.virtual_memory().used / 2**30)) if use_progressive_image_sizes: next_image_size = \ progressive_image_sizes[min(epoch // progressive_image_epoch_step, len(progressive_image_sizes) - 1)] if train_set.image_size != next_image_size: log("changing image size to {}".format(next_image_size)) train_set.image_size = next_image_size val_set.image_size = next_image_size model.train() train_loss_sum_t = zero_item_tensor() epoch_batch_iter_count = 0 if lr_scheduler_type == "lr_finder": new_lr = lr_max * 0.5**(sgdr_cycle_epochs - min( sgdr_cycle_epochs, sgdr_iterations / epoch_iterations)) adjust_learning_rate(optimizer, new_lr) all_predictions = [] all_targets = [] for b, batch in enumerate(train_set_data_loader): images, categories = \ batch[0].to(device, non_blocking=True), \ batch[1].to(device, non_blocking=True) if lr_scheduler_type == "cosine_annealing": lr_scheduler.step( epoch=min(current_sgdr_cycle_epochs, sgdr_iterations / epoch_iterations)) if b % batch_iterations == 0: optimizer.zero_grad() prediction_logits = model(images) criterion.weight = CLASS_WEIGHTS_TENSOR loss = criterion(prediction_logits, categories) loss.backward() with torch.no_grad(): train_loss_sum_t += loss all_predictions.extend( torch.sigmoid(prediction_logits).cpu().data.numpy()) all_targets.extend(categories.cpu().data.numpy()) if (b + 1) % batch_iterations == 0 or ( b + 1) == len(train_set_data_loader): optimizer.step() sgdr_iterations += 1 batch_count += 1 epoch_batch_iter_count += 1 optim_summary_writer.add_scalar("lr", get_learning_rate(optimizer), batch_count + 1) train_loss_avg = train_loss_sum_t.item() / epoch_batch_iter_count train_score_avg = f1_score_from_probs(torch.tensor(all_predictions), torch.tensor(all_targets)) val_loss_avg, val_score_avg = evaluate(model, val_set_data_loader, criterion) if lr_scheduler_type == "reduce_on_plateau": lr_scheduler_plateau.step(val_score_avg) elif lr_scheduler_type == "step": lr_scheduler_step.step(epoch) model_improved_within_sgdr_cycle = check_model_improved( sgdr_cycle_val_score_best_avg, val_score_avg) if model_improved_within_sgdr_cycle: torch.save( model.state_dict(), "{}/model-{}.pth".format(output_dir, ensemble_model_index)) sgdr_cycle_val_score_best_avg = val_score_avg model_improved = check_model_improved(global_val_score_best_avg, val_score_avg) ckpt_saved = False if model_improved: torch.save(model.state_dict(), "{}/model.pth".format(output_dir)) torch.save(optimizer.state_dict(), "{}/optimizer.pth".format(output_dir)) np.save("{}/train_predictions.npy".format(output_dir), all_predictions) np.save("{}/train_targets.npy".format(output_dir), all_targets) global_val_score_best_avg = val_score_avg epoch_of_last_improval = epoch ckpt_saved = True sgdr_reset = False if (lr_scheduler_type == "cosine_annealing") \ and (epoch + 1 >= sgdr_next_cycle_end_epoch) \ and (epoch - epoch_of_last_improval >= sgdr_cycle_end_patience): sgdr_iterations = 0 current_sgdr_cycle_epochs = int(current_sgdr_cycle_epochs * sgdr_cycle_epochs_mult) sgdr_next_cycle_end_epoch = epoch + 1 + current_sgdr_cycle_epochs + sgdr_cycle_end_prolongation ensemble_model_index += 1 sgdr_cycle_val_score_best_avg = float("-inf") sgdr_cycle_count += 1 sgdr_reset = True new_lr_min = lr_min * (lr_min_decay**sgdr_cycle_count) new_lr_max = lr_max * (lr_max_decay**sgdr_cycle_count) new_lr_max = max(new_lr_max, new_lr_min) adjust_learning_rate(optimizer, new_lr_max) lr_scheduler = CosineAnnealingLR(optimizer, T_max=current_sgdr_cycle_epochs, eta_min=new_lr_min) optim_summary_writer.add_scalar("sgdr_cycle", sgdr_cycle_count, epoch + 1) train_summary_writer.add_scalar("loss", train_loss_avg, epoch + 1) train_summary_writer.add_scalar("score", train_score_avg, epoch + 1) val_summary_writer.add_scalar("loss", val_loss_avg, epoch + 1) val_summary_writer.add_scalar("score", val_score_avg, epoch + 1) epoch_end_time = time.time() epoch_duration_time = epoch_end_time - epoch_start_time log("[%03d/%03d] %ds, lr: %.6f, loss: %.4f, val_loss: %.4f, score: %.4f, val_score: %.4f, ckpt: %d, rst: %d" % (epoch + 1, epochs_to_train, epoch_duration_time, get_learning_rate(optimizer), train_loss_avg, val_loss_avg, train_score_avg, val_score_avg, int(ckpt_saved), int(sgdr_reset))) log('{"chart": "best_val_score", "x": %d, "y": %.4f}' % (epoch + 1, global_val_score_best_avg)) log('{"chart": "val_loss", "x": %d, "y": %.4f}' % (epoch + 1, val_loss_avg)) log('{"chart": "val_score", "x": %d, "y": %.4f}' % (epoch + 1, val_score_avg)) log('{"chart": "sgdr_cycle", "x": %d, "y": %d}' % (epoch + 1, sgdr_cycle_count)) log('{"chart": "loss", "x": %d, "y": %.4f}' % (epoch + 1, train_loss_avg)) log('{"chart": "score", "x": %d, "y": %.4f}' % (epoch + 1, train_score_avg)) log('{"chart": "lr_scaled", "x": %d, "y": %.4f}' % (epoch + 1, 1000 * get_learning_rate(optimizer))) log('{"chart": "mem_used", "x": %d, "y": %.2f}' % (epoch + 1, psutil.virtual_memory().used / 2**30)) log('{"chart": "epoch_time", "x": %d, "y": %d}' % (epoch + 1, epoch_duration_time)) if (sgdr_reset or lr_scheduler_type in ("reduce_on_plateau", "step")) \ and epoch - epoch_of_last_improval >= patience: log("early abort due to lack of improval") break if max_sgdr_cycles is not None and sgdr_cycle_count >= max_sgdr_cycles: log("early abort due to maximum number of sgdr cycles reached") break optim_summary_writer.close() train_summary_writer.close() val_summary_writer.close() train_end_time = time.time() log() log("Train time: %s" % str(datetime.timedelta(seconds=train_end_time - train_start_time))) model.load_state_dict( torch.load("{}/model.pth".format(output_dir), map_location=device)) val_predictions, val_targets = predict(model, val_set_data_loader) np.save("{}/val_predictions.npy".format(output_dir), val_predictions) np.save("{}/val_targets.npy".format(output_dir), val_targets) best_threshold, best_threshold_score, all_threshold_scores = calculate_best_threshold( val_predictions, val_targets) log("All threshold scores: {}".format(all_threshold_scores)) log("Best threshold / score: {} / {}".format(best_threshold, best_threshold_score)) test_data = TestData(input_dir) test_set = TestDataset(test_data.test_set_df, input_dir, image_size, crop_images) test_set_data_loader = \ DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=pin_memory) test_predictions, _ = predict(model, test_set_data_loader) np.save("{}/test_predictions.npy".format(output_dir), test_predictions) predicted_categories = calculate_categories_from_predictions( test_predictions, threshold=best_threshold) submission_df = test_data.test_set_df.copy() submission_df["Predicted"] = [ " ".join(map(str, pc)) for pc in predicted_categories ] submission_df.to_csv("{}/submission.csv".format(output_dir))
def main(args): """main function for training DHS net""" # print(args) # uncomment to test arg inputs bsize = args.batch_size train_dir = args.train_dir test_dir = args.test_dir model_dir = args.ckpt_dir tensorboard_dir = args.tensorboard_dir device = args.device if not os.path.exists(model_dir): os.mkdir(model_dir) train_loader = torch.utils.data.DataLoader(TrainData(train_dir, transform=True), batch_size=bsize, shuffle=True, num_workers=4, pin_memory=True) val_loader = torch.utils.data.DataLoader(TrainData(test_dir, transform=True), batch_size=bsize, shuffle=True, num_workers=4, pin_memory=True) model = SRM() if device == 'gpu': model.cuda() criterion = nn.BCELoss() optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) train_loss = [] evaluation = [] result = {'epoch': [], 'F_measure': [], 'MAE': []} progress = tqdm(range(0, args.epochs + 1), miniters=1, ncols=100, desc='Overall Progress', leave=True, position=0) offset = 1 best = 0 writer = SummaryWriter(tensorboard_dir) for epoch in progress: if epoch != 0: print("load parameters") model.load_state_dict(torch.load(model_dir + 'current_network.pth')) optimizer.load_state_dict( torch.load(model_dir + 'current_optimizer.pth')) title = 'Training Epoch {}'.format(epoch) progress_epoch = tqdm(train_loader, ncols=120, total=len(train_loader), smoothing=0.9, miniters=1, leave=True, position=offset, desc=title) for ib, (img, gt) in enumerate(progress_epoch): # inputs = Variable(img).cuda() # GPU version # gt = Variable(gt.unsqueeze(1)).cuda() # GPU version inputs = Variable(img) # CPU version gt = Variable(gt.unsqueeze(1)) # CPU version output1, output2 = model.forward(inputs) output1 = get_pred(output1) output2 = get_pred(output2) loss = criterion(output1, gt) + criterion(output2, gt) model.zero_grad() loss.backward() optimizer.step() train_loss.append(round(float(loss.data.cpu()), 3)) title = '{} Epoch {}/{}'.format('Training', epoch, args.epochs) progress_epoch.set_description(title + ' ' + 'loss:' + str(loss.data.cpu().numpy())) writer.add_scalar('Train/Loss', loss.data.cpu(), epoch) filename = model_dir + 'current_network.pth' filename_opti = model_dir + 'current_optimizer.pth' torch.save(model.state_dict(), filename) # save current model params torch.save(optimizer.state_dict(), filename_opti) # save current optimizer params if epoch % args.val_rate == 0: # start validation params = model_dir + 'current_network.pth' model.load_state_dict(torch.load(params)) pred_list = [] gt_list = [] for img, gt in val_loader: # inputs = Variable(img).cuda() # GPU version inputs = Variable(img) # CPU version _, output = model.forward(inputs) output = get_pred(output) out = output.data.cpu().numpy() pred_list.extend(out) gt = gt.numpy() gt_list.extend(gt) pred_list = np.array(pred_list) pred_list = np.squeeze(pred_list) gt_list = np.array(gt_list) F_measure = get_f_measure(pred_list, gt_list) mae = get_mae(pred_list, gt_list) evaluation.append([int(epoch), float(F_measure), float(mae)]) result['epoch'].append(int(epoch)) result['F_measure'].append(round(float(F_measure), 3)) result['MAE'].append(round(float(mae), 3)) df = pd.DataFrame(result).set_index('epoch') df.to_csv('./eval.csv') if epoch == 0: best = F_measure - mae elif F_measure - mae > best: # save model with best performance best = F_measure - mae filename = ('%s/best_network.pth' % model_dir) filename_opti = ('%s/best_optimizer.pth' % model_dir) torch.save(model.state_dict(), filename) torch.save(optimizer.state_dict(), filename_opti)
def main(): batch_size = 16 generator = Generator().cuda() discriminator = Discriminator(96, 96).cuda() optimizer_G = optim.Adam(generator.parameters(), lr=1e-4) optimizer_D = optim.Adam(discriminator.parameters(), lr=1e-4) # dataset = FaceData('train') dataset = TrainData() data_loader = DataLoader(dataset, batch_size, shuffle=True, num_workers=0, pin_memory=True, drop_last=True) MSE = nn.MSELoss() BCE = nn.BCELoss() # content loss, perceptual loss vgg / i,j == 5,4 vgg_net = vgg19(pretrained=True).features[:36].cuda() vgg_net.eval() for param in vgg_net.parameters(): param.requires_grad = False discriminator.train() generator.train() optimizer_G.zero_grad() optimizer_D.zero_grad() print("Start Training") current_epoch = 0 for epoch in range(current_epoch, 100): for step, (img_Input, img_GT) in tqdm(enumerate(data_loader)): img_GT = img_GT.cuda() img_Input = img_Input.cuda() # # Discriminator update # img_SR = generator(img_Input) # fake = discriminator(img_SR) # real = discriminator(img_GT) # loss_Dfake = 0.001 * BCE(fake, torch.zeros(batch_size, 1).cuda()) # loss_Dreal = 0.001 * BCE(real, torch.ones(batch_size, 1).cuda()) # loss_D = 0.001 * (loss_Dfake + loss_Dreal) # # if epoch > 0: # discriminator.zero_grad() # loss_D.backward(retain_graph=True) # optimizer_D.step() # # Generator update # img_SR = generator(img_Input) # loss_content = MSE(img_SR, img_GT) # loss_vgg = 0.006 * MSE(vgg_net(img_SR), vgg_net(img_GT)) # fake = discriminator(img_SR) # loss_Dfake = 0.001 * BCE(fake, torch.zeros(batch_size, 1).cuda()) # loss_G = loss_content + loss_vgg + loss_Dfake # generator.zero_grad() # loss_G.backward() # # loss_Dfake.backward() # optimizer_G.step() if epoch < 10: # SRResnet Initialize Generator update generator.zero_grad() img_SR = generator(img_Input) loss_content = MSE(img_SR, img_GT) loss_content.backward() optimizer_G.step() if step % 100 == 0: print() print("Loss_content : {}".format(loss_content.item())) continue # Discriminator update discriminator.zero_grad() D_real = discriminator(img_GT) loss_Dreal = 0.1 * BCE(D_real, torch.ones(batch_size, 1).cuda()) loss_Dreal.backward() D_x = D_real.mean().item() img_SR = generator(img_Input) D_fake = discriminator(img_SR.detach()) loss_Dfake = 0.1 * BCE(D_fake, torch.zeros(batch_size, 1).cuda()) loss_Dfake.backward() DG_z = D_fake.mean().item() loss_D = (loss_Dfake + loss_Dreal) optimizer_D.step() # Generator update generator.zero_grad() loss_content = MSE(img_SR, img_GT) loss_vgg = MSE(vgg_net(img_SR), vgg_net(img_GT)) # img_SR = generator(img_Input) G_fake = discriminator(img_SR) loss_Gfake = BCE(G_fake, torch.zeros(batch_size, 1).cuda()) loss_G = loss_content + 0.006 * loss_vgg + 0.001 * loss_Gfake loss_G.backward() # loss_Dfake.backward() optimizer_G.step() if step % 100 == 0: # :.10f print() print("fake out : {}".format(DG_z)) print("real out : {}".format(D_x)) print("Loss_Dfake : {}".format(loss_Dfake.item())) print("Loss_Dreal : {}".format(loss_Dreal.item())) print("Loss_D : {}".format(loss_D.item())) print("Loss_content : {}".format(loss_content.item())) print("Loss_vgg : {}".format(0.006 * loss_vgg.item())) print("Loss_Gfake : {}".format(0.001 * loss_Gfake.item())) print("Loss_G : {}".format(loss_G.item())) print("Loss_Total : {}".format((loss_G + loss_D).item())) # print("Loss_D : {:.4f}".format(loss_D.item())) # print("Loss : {:.4f}".format(loss_total.item())) with torch.no_grad(): generator.eval() save_image(denorm(img_SR[0].cpu()), "./Result/{0}_SR.png".format(epoch)) save_image(denorm(img_GT[0].cpu()), "./Result/{0}_GT.png".format(epoch)) generator.train()