def run(config,path): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train(**config["prep"]['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop(**config["prep"]['params']) else: prep_dict["test"] = prep.image_test(**config["prep"]['params']) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] train_bs = 34 test_bs = 34 dsets["source"] = ImageList(open(data_config["source"]["list_path"]).readlines(), \ transform=prep_dict["source"]) dset_loaders["source"] = DataLoader(dsets["source"], batch_size=train_bs, \ shuffle=True, num_workers=0, drop_last=True) dsets["target"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["target"]) dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, \ shuffle=True, num_workers=0, drop_last=True) dset_loaders["test1"] = DataLoader(dsets["target"], batch_size=train_bs, \ shuffle=False, num_workers=1, drop_last=False) if prep_config["test_10crop"]: for i in range(10): dsets["test"] = [ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"][i]) for i in range(10)] dset_loaders["test"] = [DataLoader(dset, batch_size=test_bs, \ shuffle=False, num_workers=0) for dset in dsets['test']] else: dsets["test"] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["test"] = DataLoader(dsets["test"], batch_size=test_bs, \ shuffle=False, num_workers=0) n_class = config["network"]["params"]["class_num"] # for p in [499,999,1499,1999,2499,2999]: # PATH = path + '/'+ str(p) + '_model.pth.tar' # # model = load_model(PATH) # base_network = model.cuda() # fun1(dset_loaders, base_network, n_class) PATH = path+'/2999_model.pth.tar' model = load_model(PATH) base_network = model.cuda() # homo_cl = train_homo_cl(dset_loaders, base_network) fun1(dset_loaders, base_network, n_class, dsets)
def run(config, model): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train(**config["prep"]['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop(**config["prep"]['params']) else: prep_dict["test"] = prep.image_test(**config["prep"]['params']) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] train_bs = 34 test_bs = 34 dsets["source"] = ImageList(open(data_config["source"]["list_path"]).readlines(), \ transform=prep_dict["source"]) dset_loaders["source"] = DataLoader(dsets["source"], batch_size=train_bs, \ shuffle=True, num_workers=0, drop_last=False) dsets["target"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["target"]) dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, \ shuffle=True, num_workers=0, drop_last=False) if prep_config["test_10crop"]: for i in range(10): dsets["test"] = [ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"][i]) for i in range(10)] dset_loaders["test"] = [DataLoader(dset, batch_size=test_bs, \ shuffle=False, num_workers=0) for dset in dsets['test']] else: dsets["test"] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["test"] = DataLoader(dsets["test"], batch_size=test_bs, \ shuffle=False, num_workers=0) n_class = config["network"]["params"]["class_num"] model.train(False) temp_acc = image_classification_test(dset_loaders, model, n_class=n_class) log_str = "precision: {:.5f}".format(temp_acc) print(log_str)
def get_label_list(args, target_list, feature_network_path, predict_network_path, num_layer, resize_size, crop_size, batch_size, use_gpu): """ Return the target list with pesudolabel :param target_list: list conatinging all target file path and a wrong label :param predict_network: network to perdict label for target image :param resize_size: :param crop_size: :param batch_size: :return: """ option = 'resnet' + args.resnet G = ResBase(option) F1 = ResClassifier(num_layer=num_layer) G.load_state_dict(torch.load(feature_network_path)) F1.load_state_dict(torch.load(predict_network_path)) if use_gpu: G.cuda() F1.cuda() G.eval() F1.eval() label_list = [] dsets_tar = ImageList(target_list, transform=prep.image_train(resize_size=resize_size, crop_size=crop_size)) dset_loaders_tar = util_data.DataLoader(dsets_tar, batch_size=batch_size, shuffle=False, num_workers=4) len_train_target = len(dset_loaders_tar) iter_target = iter(dset_loaders_tar) count = 0 for i in range(len_train_target): input_tar, label_tar = iter_target.next() if use_gpu: input_tar, label_tar = Variable(input_tar).cuda(), Variable( label_tar).cuda() else: input_tar, label_tar = Variable(input_tar), Variable(label_tar) tar_feature = G(input_tar) predict_score = F1(tar_feature) _, pre_lab = torch.max(predict_score, 1) predict_label = pre_lab.detach() for num in range(len(predict_label.cpu())): if target_list[count][-3] == ' ': ind = -2 else: ind = -3 label_list.append(target_list[count][:ind]) label_list[count] = label_list[count] + str( predict_label[num].cpu().numpy()) + "\n" count += 1 return label_list
def save(config, model, save_name): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train(**config["prep"]['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop(**config["prep"]['params']) else: prep_dict["test"] = prep.image_test(**config["prep"]['params']) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] train_bs = 34 test_bs = 34 dsets["source"] = ImageList(open(data_config["source"]["list_path"]).readlines(), \ transform=prep_dict["source"]) dset_loaders["source"] = DataLoader(dsets["source"], batch_size=train_bs, \ shuffle=True, num_workers=0, drop_last=False) dsets["target"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["target"]) dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, \ shuffle=True, num_workers=0, drop_last=False) if prep_config["test_10crop"]: for i in range(10): dsets["test"] = [ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"][i]) for i in range(10)] dset_loaders["test"] = [DataLoader(dset, batch_size=test_bs, \ shuffle=False, num_workers=0) for dset in dsets['test']] else: dsets["test"] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["test"] = DataLoader(dsets["test"], batch_size=test_bs, \ shuffle=False, num_workers=0) n_class = config["network"]["params"]["class_num"] model.train(False) save_feature(dset_loaders, model, './snapshot/model/', save_name)
def get_label_list(target_list, predict_network_name, resize_size, crop_size, batch_size, use_gpu): # done with debugging, works fine """ Return the target list with pesudolabel :param target_list: list conatinging all target file path and a wrong label :param predict_network: network to perdict label for target image :param resize_size: :param crop_size: :param batch_size: :return: """ label_list = [] net_config = predict_network_name predict_network = net_config["name"](**net_config["params"]) if use_gpu: predict_network = predict_network.cuda() dsets_tar = ImageList(target_list, transform=prep.image_train(resize_size=resize_size, crop_size=crop_size)) dset_loaders_tar = util_data.DataLoader(dsets_tar, batch_size=batch_size, shuffle=True, num_workers=4) len_train_target = len(dset_loaders_tar) iter_target = iter(dset_loaders_tar) count = 0 for i in range(len_train_target): input_tar, label_tar = iter_target.next() if use_gpu: input_tar, label_tar = Variable(input_tar).cuda(), Variable( label_tar).cuda() else: input_tar, label_tar = Variable(input_tar), Variable(label_tar) _, predict_score = predict_network(input_tar) _, predict_label = torch.max(predict_score, 1) for num in range(len(predict_label.cpu())): label_list.append(target_list[count][:-2]) label_list[count] = label_list[count] + str( predict_label[num].cpu().numpy()) + "\n" count += 1 return label_list
def get_label_list(target_list, predict_network, resize_size, crop_size, batch_size, use_gpu): """ Return the target list with pesudolabel :param target_list: list conatinging all target file path and a wrong label :param predict_network: network to perdict label for target image :param resize_size: :param crop_size: :param batch_size: :return: """ label_list = [] dsets_tar = ImageList(target_list, transform=prep.image_train(resize_size=resize_size, crop_size=crop_size)) dset_loaders_tar = util_data.DataLoader(dsets_tar, batch_size=batch_size, shuffle=False, num_workers=4) len_train_target = len(dset_loaders_tar) iter_target = iter(dset_loaders_tar) count = 0 for i in range(len_train_target): input_tar, label_tar = iter_target.next() if use_gpu: input_tar, label_tar = Variable(input_tar).cuda(), Variable( label_tar).cuda() else: input_tar, label_tar = Variable(input_tar), Variable(label_tar) _, predict_score = predict_network(input_tar) _, predict_label = torch.max(predict_score, 1) for num in range(len(predict_label.cpu())): if target_list[count][-3] == ' ': ind = -2 else: ind = -3 label_list.append(target_list[count][:ind]) label_list[count] = label_list[count] + str( predict_label[num].cpu().numpy()) + "\n" print(label_list[count]) count += 1 return label_list
def get_label_list(target_list, predict_network, resize_size, crop_size, batch_size): """ Return the target list with pesudolabel :param target_list: list conatinging all target file path and a wrong label :param predict_network: network to perdict label for target image :param resize_size: :param crop_size: :param batch_size: :return: """ label_list = [] dsets_tar = ImageList(target_list, transform=prep.image_train(resize_size=resize_size, crop_size=crop_size)) dset_loaders_tar = util_data.DataLoader(dsets_tar, batch_size=batch_size, shuffle=True, num_workers=4) len_train_target = len(dset_loaders_tar) iter_target = iter(dset_loaders_tar) for i in range(len_train_target): input_tar, label_tar = iter_target.next() predict_score = predict_network(input_tar)[1] label = np.argsort(-predict_score)[0] label_list.append(target_list[i][:-2]) label_list[i] = label_list[i] + str(label) + "\n" return label_list
def train(config): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train(**config["prep"]['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop(**config["prep"]['params']) else: prep_dict["test"] = prep.image_test(**config["prep"]['params']) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] train_bs = data_config["source"]["batch_size"] test_bs = data_config["test"]["batch_size"] dsets["source"] = ImageList(open(data_config["source"]["list_path"]).readlines(), \ transform=prep_dict["source"]) dset_loaders["source"] = DataLoader(dsets["source"], batch_size=train_bs, \ shuffle=True, num_workers=0, drop_last=True) dsets["target"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["target"]) dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, \ shuffle=True, num_workers=0, drop_last=True) if prep_config["test_10crop"]: for i in range(10): dsets["test"] = [ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"][i]) for i in range(10)] dset_loaders["test"] = [DataLoader(dset, batch_size=test_bs, \ shuffle=False, num_workers=0) for dset in dsets['test']] else: dsets["test"] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["test"] = DataLoader(dsets["test"], batch_size=test_bs, \ shuffle=False, num_workers=0) # class_num = config["network"]["params"]["class_num"] n_class = config["network"]["params"]["class_num"] ## set base network net_config = config["network"] base_network_stu = net_config["name"](**net_config["params"]).cuda() base_network_tea = net_config["name"](**net_config["params"]).cuda() ## add additional network for some methods if config["loss"]["random"]: random_layer = network.RandomLayer([base_network_stu.output_num(), n_class], config["loss"]["random_dim"]) ad_net = network.AdversarialNetwork(config["loss"]["random_dim"], 1024) else: random_layer = None if config['method'] == 'DANN': ad_net = network.AdversarialNetwork(base_network_stu.output_num(), 1024)#DANN else: ad_net = network.AdversarialNetwork(base_network_stu.output_num() * n_class, 1024) if config["loss"]["random"]: random_layer.cuda() ad_net = ad_net.cuda() ad_net2 = network.AdversarialNetwork(n_class, n_class*4) ad_net2.cuda() parameter_list = base_network_stu.get_parameters() + ad_net.get_parameters() teacher_params = list(base_network_tea.parameters()) for param in teacher_params: param.requires_grad = False ## set optimizer optimizer_config = config["optimizer"] optimizer = optimizer_config["type"](parameter_list, \ **(optimizer_config["optim_params"])) teacher_optimizer = EMAWeightOptimizer(base_network_tea, base_network_stu, alpha=0.99) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] ## train len_train_source = len(dset_loaders["source"]) len_train_target = len(dset_loaders["target"]) best_acc = 0.0 output1(log_name) loss1, loss2, loss3, loss4, loss5,loss6 = 0, 0, 0, 0, 0,0 output1(' ======= DA TRAINING ======= ') best1 = 0 f_t_result = [] max_iter = config["num_iterations"] for i in range(max_iter+1): if i % config["test_interval"] == config["test_interval"] - 1 and i > 1500: base_network_tea.train(False) base_network_stu.train(False) # print("test") if 'MT' in config['method']: temp_acc = image_classification_test(dset_loaders, base_network_tea, test_10crop=prep_config["test_10crop"]) if temp_acc > best_acc: best_acc = temp_acc log_str = "iter: {:05d}, tea_precision: {:.5f}".format(i, temp_acc) output1(log_str) # # if i > 20001 and best_acc < 0.69: # break # # if temp_acc < 0.67: # break # # if i > 30001 and best_acc < 0.71: # break # torch.save(base_network_tea, osp.join(path, "_model.pth.tar")) # temp_acc = image_classification_test(dset_loaders, # base_network_stu, test_10crop=prep_config["test_10crop"]) if temp_acc > best_acc: best_acc = temp_acc torch.save(base_network_stu, osp.join(path, "_model.pth.tar")) # log_str = "iter: {:05d}, stu_precision: {:.5f}".format(i, temp_acc) # output1(log_str) else: temp_acc = image_classification_test(dset_loaders, base_network_stu, test_10crop=prep_config["test_10crop"]) if temp_acc > best_acc: best_acc = temp_acc torch.save(base_network_stu, osp.join(path,"_model.pth.tar")) log_str = "iter: {:05d}, precision: {:.5f}".format(i, temp_acc) output1(log_str) loss_params = config["loss"] ## train one iter base_network_stu.train(True) base_network_tea.train(True) ad_net.train(True) optimizer = lr_scheduler(optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_target == 0: iter_target = iter(dset_loaders["target"]) if i % len_train_source == 0: iter_source = iter(dset_loaders["source"]) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() inputs_source, inputs_target, labels_source = inputs_source.cuda(), inputs_target.cuda(), labels_source.cuda() features_source, outputs_source = base_network_stu(inputs_source) features_target_stu, outputs_target_stu = base_network_stu(inputs_target) features_target_tea, outputs_target_tea = base_network_tea(inputs_target) softmax_out_source = nn.Softmax(dim=1)(outputs_source) softmax_out_target_stu = nn.Softmax(dim=1)(outputs_target_stu) softmax_out_target_tea = nn.Softmax(dim=1)(outputs_target_tea) features = torch.cat((features_source, features_target_stu), dim=0) if 'MT' in config['method']: softmax_out = torch.cat((softmax_out_source, softmax_out_target_tea), dim=0) else: softmax_out = torch.cat((softmax_out_source, softmax_out_target_stu), dim=0) vat_loss = VAT(base_network_stu).cuda() n, d = features_source.shape decay = cal_decay(start=1,end=0.6,i = i) # image number in each class s_labels = labels_source t_max, t_labels = torch.max(softmax_out_target_tea, 1) t_max, t_labels = t_max.cuda(), t_labels.cuda() if config['method'] == 'DANN+dis' or config['method'] == 'CDRL': pass elif config['method'] == 'RESNET': classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) total_loss = classifier_loss elif config['method'] == 'CDAN+E': entropy = losssntg.Entropy(softmax_out) ad_loss = losssntg.CDANori([features, softmax_out], ad_net, entropy, network.calc_coeff(i), random_layer) transfer_loss = loss_params["trade_off"] * ad_loss classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) total_loss = transfer_loss + classifier_loss elif config['method'] == 'CDAN': ad_loss = losssntg.CDANori([features, softmax_out], ad_net, None, None, random_layer) transfer_loss = loss_params["trade_off"] * ad_loss classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) total_loss = transfer_loss + classifier_loss elif config['method'] == 'DANN': ad_loss = losssntg.DANN(features, ad_net) transfer_loss = loss_params["trade_off"] * ad_loss classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) total_loss = transfer_loss + classifier_loss elif config['method'] == 'CDAN+MT': th = config['th'] ad_loss = losssntg.CDANori([features, softmax_out], ad_net, None, None, random_layer) unsup_loss = compute_aug_loss(softmax_out_target_stu, softmax_out_target_tea, n_class,confidence_thresh=th) unsup_loss = compute_aug_loss2(softmax_out_target_stu, softmax_out_target_tea, n_class,confidence_thresh=th) transfer_loss = loss_params["trade_off"] * ad_loss \ + 0.01 * unsup_loss classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) total_loss = transfer_loss + classifier_loss elif config['method'] == 'CDAN+MT+VAT': cent = ConditionalEntropyLoss().cuda() ad_loss = losssntg.CDANori([features, softmax_out], ad_net, None, None, random_layer) unsup_loss = compute_aug_loss(softmax_out_target_stu, softmax_out_target_tea, n_class) loss_trg_cent = 1e-2 * cent(outputs_target_stu) loss_trg_vat = 1e-2 * vat_loss(inputs_target, outputs_target_stu) transfer_loss = loss_params["trade_off"] * ad_loss \ + 0.001*(unsup_loss + loss_trg_cent + loss_trg_vat) classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) total_loss = transfer_loss + classifier_loss elif config['method'] == 'CDAN+MT+cent+VAT+temp': th = 0.7 ad_loss = losssntg.CDANori([features, softmax_out], ad_net, None, None, random_layer) unsup_loss = compute_aug_loss(softmax_out_target_stu, softmax_out_target_tea, n_class,confidence_thresh=th) cent = ConditionalEntropyLoss().cuda() # loss_src_vat = vat_loss(inputs_source, outputs_source) loss_trg_cent = 1e-2 * cent(outputs_target_stu) loss_trg_vat = 1e-2 * vat_loss(inputs_target, outputs_target_stu) transfer_loss = loss_params["trade_off"] * ad_loss \ + unsup_loss + loss_trg_cent + loss_trg_vat # temperature classifier_loss = nn.NLLLoss()(nn.LogSoftmax(1)(outputs_source / 1.05), labels_source) total_loss = transfer_loss + classifier_loss elif config['method'] == 'CDAN+MT+cent+VAT+weightCross+T': if i % len_train_target == 0: if i != 0: # print(cnt) cnt = torch.tensor(cnt).float() weight = cnt.sum() - cnt weight = weight.cuda() else: weight = torch.ones(n_class).cuda() cnt = [0] * n_class for j in t_labels: cnt[j.item()] += 1 a = config['a'] b = config['b'] th = config['th'] temp = config['temp'] ad_loss = losssntg.CDANori([features, softmax_out], ad_net, None, None, random_layer) unsup_loss = compute_aug_loss2(softmax_out_target_stu, softmax_out_target_tea, n_class,confidence_thresh=th) # cbloss = compute_cbloss(softmax_out_target_stu, n_class, cls_balance=0.05) # unsup_loss = compute_aug_loss(softmax_out_target_stu, softmax_out_target_tea, n_class,confidence_thresh=th) cent = ConditionalEntropyLoss().cuda() # loss_src_vat = vat_loss(inputs_source, outputs_source) loss_trg_cent = 1e-2 * cent(outputs_target_stu) loss_trg_vat = 1e-2 * vat_loss(inputs_target, outputs_target_stu) classifier_loss = nn.CrossEntropyLoss(weight=weight)(outputs_source/temp, labels_source) # classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) transfer_loss = loss_params["trade_off"] * ad_loss \ + a*unsup_loss + b*(loss_trg_vat+loss_trg_cent) total_loss = transfer_loss + classifier_loss elif config['method'] == 'CDAN+MT+E+VAT+weightCross+T': entropy = losssntg.Entropy(softmax_out) if i % len_train_target == 0: if i != 0: # print(cnt) cnt = torch.tensor(cnt).float() weight = cnt.sum() - cnt weight = weight.cuda() else: weight = torch.ones(n_class).cuda() cnt = [0] * n_class for j in t_labels: cnt[j.item()] += 1 a = config['a'] b = config['b'] th = config['th'] temp = config['temp'] ad_loss = losssntg.CDANori([features, softmax_out], ad_net, entropy, network.calc_coeff(i), random_layer) # unsup_loss = compute_aug_loss2(softmax_out_target_stu, softmax_out_target_tea, n_class,confidence_thresh=th) # cbloss = compute_cbloss(softmax_out_target_stu, n_class, cls_balance=0.05) unsup_loss = compute_aug_loss(softmax_out_target_stu, softmax_out_target_tea, n_class, confidence_thresh=th) cent = ConditionalEntropyLoss().cuda() # loss_src_vat = vat_loss(inputs_source, outputs_source) loss_trg_cent = 1e-2 * cent(outputs_target_stu) loss_trg_vat = 1e-2 * vat_loss(inputs_target, outputs_target_stu) classifier_loss = nn.CrossEntropyLoss(weight=weight)(outputs_source / temp, labels_source) # classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) transfer_loss = loss_params["trade_off"] * ad_loss \ + a * unsup_loss + b * (loss_trg_vat + loss_trg_cent) total_loss = transfer_loss + classifier_loss # adout # ad_out1 = ad_net(features_source) # w = 1-ad_out1 # c = w * nn.CrossEntropyLoss(reduction='none')(outputs_source, labels_source) # classifier_loss = c.mean() # total_loss = transfer_loss + classifier_loss total_loss.backward() optimizer.step() teacher_optimizer.step() loss1 += ad_loss.item() loss2 += classifier_loss.item() # loss3 += unsup_loss.item() # loss4 += loss_trg_cent.item() # loss5 += loss_trg_vat.item() # loss6 += cbloss.item() # dis_sloss_l += sloss_l.item() if i % 50 == 0 and i != 0: output1('iter:{:d}, ad_loss_D:{:.2f}, closs:{:.2f}, unsup_loss:{:.2f}, loss_trg_cent:{:.2f}, loss_trg_vatcd:{:.2f}, cbloss:{:.2f}' .format(i, loss1, loss2, loss3, loss4, loss5,loss6)) loss1, loss2, loss3, loss4, loss5, loss6 = 0, 0, 0, 0, 0, 0 # torch.save(best_model, osp.join(path, "best_model.pth.tar")) return best_acc
def cross_validation_loss(feature_network, predict_network, src_cls_list, target_path, val_cls_list, class_num, resize_size, crop_size, batch_size, use_gpu): """ Main function for computing the CV loss :param feature_network: :param predict_network: :param src_cls_list: :param target_path: :param val_cls_list: :param class_num: :param resize_size: :param crop_size: :param batch_size: :return: """ target_list_no_label = open(target_path).readlines() tar_cls_list = [] cross_val_loss = 0 # add pesudolabel for target data target_list = get_label_list(target_list_no_label, predict_network, resize_size, crop_size, batch_size, use_gpu) # seperate the class for i in range(class_num): tar_cls_list.append([ j for j in target_list if int(j.split(" ")[1].replace("\n", "")) == i ]) prep_dict = prep.image_train(resize_size=resize_size, crop_size=crop_size) # load different class's image for cls in range(class_num): dsets_src = ImageList(src_cls_list[cls], transform=prep_dict) dset_loaders_src = util_data.DataLoader(dsets_src, batch_size=batch_size, shuffle=True, num_workers=4) dsets_tar = ImageList(tar_cls_list[cls], transform=prep_dict) dset_loaders_tar = util_data.DataLoader(dsets_tar, batch_size=batch_size, shuffle=True, num_workers=4) dsets_val = ImageList(val_cls_list[cls], transform=prep_dict) dset_loaders_val = util_data.DataLoader(dsets_val, batch_size=batch_size, shuffle=True, num_workers=4) # prepare source feature iter_src = iter(dset_loaders_src) src_input = iter_src.next()[0] if use_gpu: src_input = Variable(src_input).cuda() else: src_input = Variable(src_input) src_feature, _ = feature_network(src_input) src_feature_de = src_feature.detach().cpu().numpy() for count_src in range(len(dset_loaders_src) - 1): src_input = iter_src.next()[0] if use_gpu: src_input = Variable(src_input).cuda() else: src_input = Variable(src_input) src_feature_new, _ = feature_network(src_input) src_feature_new_de = src_feature_new.detach().cpu().numpy() src_feature_de = np.append(src_feature_de, src_feature_new_de, axis=0) # prepare target feature iter_tar = iter(dset_loaders_tar) tar_input = iter_tar.next()[0] if use_gpu: tar_input = Variable(tar_input).cuda() else: tar_input = Variable(tar_input) tar_feature, _ = feature_network(tar_input) tar_feature_de = tar_feature.detach().cpu().numpy() for count_tar in range(len(dset_loaders_tar) - 1): tar_input = iter_tar.next()[0] if use_gpu: tar_input = Variable(tar_input).cuda() else: tar_input = Variable(tar_input) tar_feature_new, _ = feature_network(tar_input) tar_feature_new_de = tar_feature_new.detach().cpu().numpy() tar_feature_de = np.append(tar_feature_de, tar_feature_new_de, axis=0) # prepare validation feature and predicted label for validation iter_val = iter(dset_loaders_val) val_input, val_labels = iter_val.next() if use_gpu: val_input, val_labels = Variable(val_input).cuda(), Variable( val_labels).cuda() else: val_input, val_labels = Variable(val_input), Variable(val_labels) val_feature, _ = feature_network(val_input) _, pred_label = predict_network(val_input) val_feature_de = val_feature.detach().cpu().numpy() w = pred_label[0].shape[0] error = np.zeros(1) error[0] = predict_loss(cls, pred_label[0].reshape(1, w)).item() error = error.reshape(1, 1) for num_image in range(1, len(pred_label)): single_pred_label = pred_label[num_image] w = single_pred_label.shape[0] error = np.append( error, [[predict_loss(cls, single_pred_label.reshape(1, w)).item()]], axis=0) for count_val in range(len(dset_loaders_val) - 1): val_input, val_labels = iter_val.next() if use_gpu: val_input, val_labels = Variable(val_input).cuda(), Variable( val_labels).cuda() else: val_input, val_labels = Variable(val_input), Variable( val_labels) val_feature_new, _ = feature_network(val_input) val_feature_new_de = val_feature_new.detach().cpu().numpy() val_feature_de = np.append(val_feature_de, val_feature_new_de, axis=0) _, pred_label = predict_network(val_input) for num_image in range(len(pred_label)): single_pred_label = pred_label[num_image] w = single_pred_label.shape[0] # cls should be a value, new_labels should be a [[x]] tensor format, the input format required by predict_loss error = np.append(error, [[ predict_loss(cls, single_pred_label.reshape(1, w)).item() ]], axis=0) # error should be a (N, 1) numpy array, the input format required by get_dev_risk # print(cls) weight = get_weight(src_feature_de, tar_feature_de, val_feature_de) cross_val_loss = cross_val_loss + get_dev_risk(weight, error) / class_num return cross_val_loss
def train(config): # set pre-process prep_config = config["prep"] prep_dict = {} prep_dict["source"] = prep.image_train(**config["prep"]['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop(**config["prep"]['params']) else: prep_dict["test"] = prep.image_test(**config["prep"]['params']) # prepare data dsets = {} dset_loaders = {} data_config = config["data"] train_bs = data_config["source"]["batch_size"] test_bs = data_config["test"]["batch_size"] dsets["source"] = datasets.ImageFolder(data_config['source']['list_path'], transform=prep_dict["source"]) dset_loaders['source'] = getdataloader(dsets['source'], batchsize=train_bs, num_workers=4, drop_last=True, weightsampler=True) dsets["target"] = datasets.ImageFolder(data_config['target']['list_path'], transform=prep_dict["target"]) dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, shuffle=True, num_workers=4, drop_last=True) if prep_config["test_10crop"]: for i in range(10): dsets["test"] = [datasets.ImageFolder(data_config['test']['list_path'], transform=prep_dict["test"][i]) for i in range(10)] dset_loaders["test"] = [DataLoader(dset, batch_size=test_bs, shuffle=False, num_workers=4) for dset in dsets['test']] else: dsets["test"] = datasets.ImageFolder(data_config['test']['list_path'], transform=prep_dict["test"]) dset_loaders["test"] = DataLoader(dsets["test"], batch_size=test_bs, shuffle=False, num_workers=4) class_num = config["network"]["params"]["class_num"] # set base network net_config = config["network"] base_network = net_config["name"](**net_config["params"]) base_network = base_network.cuda() # set test_ad_net test_ad_net = network.AdversarialNetwork(base_network.output_num(), 1024, test_ad_net=True) test_ad_net = test_ad_net.cuda() # add additional network for some methods if config['method'] == 'DANN': random_layer = None ad_net = network.AdversarialNetwork(base_network.output_num(), 1024) elif config['method'] == 'MADA': random_layer = None ad_net = network.AdversarialNetworkClassGroup(base_network.output_num(), 1024, class_num) elif config['method'] == 'proposed': if config['loss']['random']: random_layer = network.RandomLayer([base_network.output_num(), class_num], config['loss']['random_dim']) ad_net = network.AdversarialNetwork(config['loss']['random_dim'], 1024) ad_net_group = network.AdversarialNetworkGroup(config['loss']['random_dim'], 256, class_num, config['center_threshold']) else: random_layer = None ad_net = network.AdversarialNetwork(base_network.output_num(), 1024) ad_net_group = network.AdversarialNetworkGroup(base_network.output_num(), 1024, class_num, config['center_threshold']) elif config['method'] == 'base': pass else: if config["loss"]["random"]: random_layer = network.RandomLayer([base_network.output_num(), class_num], config["loss"]["random_dim"]) ad_net = network.AdversarialNetwork(config["loss"]["random_dim"], 1024) else: random_layer = None ad_net = network.AdversarialNetwork(base_network.output_num() * class_num, 1024) if config["loss"]["random"] and config['method'] != 'base' and config['method'] != 'DANN' and config['method'] != 'MADA': random_layer.cuda() if config['method'] != 'base': ad_net = ad_net.cuda() if config['method'] == 'proposed': ad_net_group = ad_net_group.cuda() # set parameters if config['method'] == 'proposed': parameter_list = base_network.get_parameters() + test_ad_net.get_parameters() + ad_net.get_parameters() + ad_net_group.get_parameters() elif config['method'] == 'base': parameter_list = base_network.get_parameters() + test_ad_net.get_parameters() elif config['method'] == 'MADA': parameter_list = base_network.get_parameters() + test_ad_net.get_parameters() + ad_net.get_parameters() else: parameter_list = base_network.get_parameters() + test_ad_net.get_parameters() + ad_net.get_parameters() # set optimizer optimizer_config = config["optimizer"] optimizer = optimizer_config["type"](parameter_list, **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] # parallel gpus = config['gpu'].split(',') if len(gpus) > 1: base_network = nn.DataParallel(base_network) test_ad_net = nn.DataParallel(test_ad_net) if config['method'] == 'DANN': ad_net = nn.DataParallel(ad_net) elif config['method'] == 'proposed': if config['loss']['random']: random_layer = nn.DataParallel(random_layer) ad_net = nn.DataParallel(ad_net) #将ad_net_group设置成并行将会引发error,原因可能是由于ad_net_group的输出不是tensor类型,parallel还不能支持。 #ad_net_group = nn.DataParallel(ad_net_group) else: ad_net = nn.DataParallel(ad_net) #ad_net_group = nn.DataParallel(ad_net_group) elif config['method'] == 'base': pass else: # CDAN+E if config["loss"]["random"]: random_layer = nn.DataParallel(random_layer) ad_net = nn.DataParallel(ad_net) # CDAN else: ad_net = nn.DataParallel(ad_net) ## train len_train_source = len(dset_loaders["source"]) len_train_target = len(dset_loaders["target"]) transfer_loss_value = classifier_loss_value = total_loss_value = 0.0 best_acc = 0.0 for i in range(config["num_iterations"]): if i % config["test_interval"] == config["test_interval"] - 1: base_network.train(False) # eval() == train(False) is True temp_acc = image_classification_test(dset_loaders, base_network, test_10crop=prep_config["test_10crop"]) temp_model = nn.Sequential(base_network) if temp_acc > best_acc: best_acc = temp_acc best_model = temp_model log_str = "iter: {:05d}, precision: {:.5f}".format(i, temp_acc) config["out_file"].write(log_str + "\n") config["out_file"].flush() print(log_str) # if i % config["snapshot_interval"] == 0: # torch.save(nn.Sequential(base_network), osp.join(config["output_path"], # "iter_{:05d}_model.pth.tar".format(i))) loss_params = config["loss"] # train one iter base_network.train(True) if config['method'] != 'base': ad_net.train(True) if config['method'] == 'proposed': ad_net_group.train(True) # lr_scheduler optimizer = lr_scheduler(optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_source == 0: iter_source = iter(dset_loaders["source"]) if i % len_train_target == 0: iter_target = iter(dset_loaders["target"]) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() inputs_source, inputs_target, labels_source = inputs_source.cuda(), inputs_target.cuda(), labels_source.cuda() features_source, outputs_source = base_network(inputs_source) features_target, outputs_target = base_network(inputs_target) if config['tsne']: # feature visualization by using T-SNE if i == int(0.98*config['num_iterations']): features_source_total = features_source.cpu().detach().numpy() features_target_total = features_target.cpu().detach().numpy() elif i > int(0.98*config['num_iterations']) and i < int(0.98*config['num_iterations'])+10: features_source_total = np.concatenate((features_source_total, features_source.cpu().detach().numpy())) features_target_total = np.concatenate((features_target_total, features_target.cpu().detach().numpy())) elif i == int(0.98*config['num_iterations'])+10: for index in range(config['tsne_num']): features_embeded = TSNE(perplexity=10,n_iter=5000).fit_transform(np.concatenate((features_source_total, features_target_total))) fig = plt.figure() plt.scatter(features_embeded[:len(features_embeded)//2, 0], features_embeded[:len(features_embeded)//2, 1], c='r', s=1) plt.scatter(features_embeded[len(features_embeded)//2:, 0], features_embeded[len(features_embeded)//2:, 1], c='b', s=1) plt.savefig(osp.join(config["output_path"], config['method']+'-'+str(index)+'.png')) plt.close() else: pass assert features_source.size(0) == features_target.size(0), 'The batchsize must be same' assert outputs_source.size(0) == outputs_target.size(0), 'The batchsize must be same' # source first, target second features = torch.cat((features_source, features_target), dim=0) outputs = torch.cat((outputs_source, outputs_target), dim=0) # output the A_distance if i % config["test_interval"] == config["test_interval"] - 1: A_distance = cal_A_distance(test_ad_net, features) config['A_distance_file'].write(str(A_distance)+'\n') config['A_distance_file'].flush() softmax_out = nn.Softmax(dim=1)(outputs) if config['method'] == 'CDAN+E': entropy = loss.Entropy(softmax_out) transfer_loss = loss.CDAN([features, softmax_out], ad_net, entropy, network.calc_coeff(i), random_layer) elif config['method'] == 'CDAN': transfer_loss = loss.CDAN([features, softmax_out], ad_net, None, None, random_layer) elif config['method'] == 'DANN': transfer_loss = loss.DANN(features, ad_net) elif config['method'] == 'MADA': transfer_loss = loss.MADA(features, softmax_out, ad_net) elif config['method'] == 'proposed': entropy = loss.Entropy(softmax_out) transfer_loss = loss.proposed([features, outputs], labels_source, ad_net, ad_net_group, entropy, network.calc_coeff(i), i, random_layer, config['loss']['trade_off23']) elif config['method'] == 'base': pass else: raise ValueError('Method cannot be recognized.') test_domain_loss = loss.DANN(features.clone().detach(), test_ad_net) classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) if config['method'] == 'base': total_loss = classifier_loss + test_domain_loss else: total_loss = loss_params["trade_off"] * transfer_loss + classifier_loss + test_domain_loss total_loss.backward() optimizer.step() # torch.save(best_model, osp.join(config["output_path"], "best_model.pth.tar")) return best_acc
def train(config): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["train_set1"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) prep_dict["train_set2"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] dsets["train_set1"] = ImageList(open(data_config["train_set1"]["list_path"]).readlines(), \ transform=prep_dict["train_set1"]) dset_loaders["train_set1"] = util_data.DataLoader(dsets["train_set1"], \ batch_size=data_config["train_set1"]["batch_size"], \ shuffle=True, num_workers=4) dsets["train_set2"] = ImageList(open(data_config["train_set2"]["list_path"]).readlines(), \ transform=prep_dict["train_set2"]) dset_loaders["train_set2"] = util_data.DataLoader(dsets["train_set2"], \ batch_size=data_config["train_set2"]["batch_size"], \ shuffle=True, num_workers=4) hash_bit = config["hash_bit"] ## set base network net_config = config["network"] base_network = net_config["type"](**net_config["params"]) use_gpu = torch.cuda.is_available() if use_gpu: base_network = base_network.cuda() ## collect parameters parameter_list = [{"params":base_network.feature_layers.parameters(), "lr":1}, \ {"params":base_network.hash_layer.parameters(), "lr":10}] ## set optimizer optimizer_config = config["optimizer"] optimizer = optim_dict[optimizer_config["type"]](parameter_list, \ **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] ## train len_train1 = len(dset_loaders["train_set1"]) - 1 len_train2 = len(dset_loaders["train_set2"]) - 1 transfer_loss_value = classifier_loss_value = total_loss_value = 0.0 best_acc = 0.0 for i in range(config["num_iterations"]): if i % config["snapshot_interval"] == 0: torch.save(nn.Sequential(base_network), osp.join(config["output_path"], \ "iter_{:05d}_model.pth.tar".format(i))) ## train one iter base_network.train(True) optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train1 == 0: iter1 = iter(dset_loaders["train_set1"]) if i % len_train2 == 0: iter2 = iter(dset_loaders["train_set2"]) inputs1, labels1 = iter1.next() inputs2, labels2 = iter2.next() if use_gpu: inputs1, inputs2, labels1, labels2 = \ Variable(inputs1).cuda(), Variable(inputs2).cuda(), \ Variable(labels1).cuda(), Variable(labels2).cuda() else: inputs1, inputs2, labels1, labels2 = Variable(inputs1), \ Variable(inputs2), Variable(labels1), Variable(labels2) inputs = torch.cat((inputs1, inputs2), dim=0) outputs = base_network(inputs) similarity_loss = loss.pairwise_loss(outputs.narrow(0,0,inputs1.size(0)), \ outputs.narrow(0,inputs1.size(0),inputs2.size(0)), \ labels1, labels2, \ hashbit = hash_bit,\ gamma=config["loss"]["gamma"], \ normed=config["loss"]["normed"], \ q_lambda=config["loss"]["q_lambda"]) total_loss_value = total_loss_value + similarity_loss.float().data[0] similarity_loss.backward() if (i + 1) % len_train1 == 0: print("Epoch: {:05d}, loss: {:.3f}".format(i // len_train1, total_loss_value)) total_loss_value = 0.0 optimizer.step()
# "cub":"/vulcan-pvc1/ml_for_da_pan_base/dataset_list/cub200_2011_20.txt" } dataset_source = file_path[args.source] dataset_target = dataset_test = file_path[args.target] cate_all = [31] # dataset load batch_size = {"train": args.batch_size, "val": args.batch_size, "test": 4} for i in range(10): batch_size["val" + str(i)] = 4 dataset_loaders = {} dataset_list = ImageList(open(dataset_source).readlines(), transform=prep.image_train(resize_size=256, crop_size=224)) dataset_loaders["train"] = torch.utils.data.DataLoader( dataset_list, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) dataset_list = ImageList(open(dataset_target).readlines(), transform=prep.image_train(resize_size=256, crop_size=224)) dataset_loaders["val"] = torch.utils.data.DataLoader( dataset_list, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers)
def train(config): # set pre-process prep_dict = {'source': prep.image_train(**config['prep']['params']), 'target': prep.image_train(**config['prep']['params'])} if config['prep']['test_10crop']: prep_dict['test'] = prep.image_test_10crop(**config['prep']['params']) else: prep_dict['test'] = prep.image_test(**config['prep']['params']) # prepare data data_set = {} data_set_loader = {} data_config = config['data'] data_set['source'] = ImageList(open(data_config['source']['list_path']).readlines(), transform=prep_dict['source']) data_set_loader['source'] = torch.utils.data.DataLoader(data_set['source'], batch_size=data_config['source']['batch_size'], shuffle=True, num_workers=4, drop_last=True) data_set['target'] = ImageList(open(data_config['target']['list_path']).readlines(), transform=prep_dict['target']) data_set_loader['target'] = torch.utils.data.DataLoader(data_set['target'], batch_size=data_config['target']['batch_size'], shuffle=True, num_workers=4, drop_last=True) if config['prep']['test_10crop']: data_set['test'] = [ImageList(open(data_config['test']['list_path']).readlines(), transform=prep_dict['test'][i]) for i in range(10)] data_set_loader['test'] = [torch.utils.data.DataLoader(dset, batch_size=data_config['test']['batch_size'], shuffle=False, num_workers=4) for dset in data_set['test']] else: data_set['test'] = ImageList(open(data_config['test']['list_path']).readlines(), transform=prep_dict['test']) data_set_loader['test'] = torch.utils.data.DataLoader(data_set['test'], batch_size=data_config['test']['batch_size'], shuffle=False, num_workers=4) # set base network net_config = config['network'] base_net = net_config['name']() # res50 base_net = base_net.cuda() # add domain, classifier network class_num = config['network']['params']['class_num'] ad_net = network.Domain(in_features=base_net.output_num(), hidden_size=1024) f1_net = network.Classifier(in_features=base_net.output_num(), hidden_size=128, class_num=class_num) f2_net = network.Classifier(in_features=base_net.output_num(), hidden_size=128, class_num=class_num) ad_net = ad_net.cuda() f1_net = f1_net.cuda() f2_net = f2_net.cuda() parameter_list = base_net.get_parameters() + ad_net.get_parameters() \ + f1_net.get_parameters() + f2_net.get_parameters() # multi gpus gpus = config['gpu'].split(',') if len(gpus) > 1: base_net = nn.DataParallel(base_net, device_ids=[int(i) for i in gpus]) f1_net = nn.DataParallel(f1_net, device_ids=[int(i) for i in gpus]) f2_net = nn.DataParallel(f2_net, device_ids=[int(i) for i in gpus]) ad_net = nn.DataParallel(ad_net, device_ids=[int(i) for i in gpus]) # set optimizer optimizer_config = config['optimizer'] optimizer = optimizer_config['type'](parameter_list, **(optimizer_config['optim_params'])) schedule_param = optimizer_config['lr_param'] lr_scheduler = lr_schedule.schedule_dict[optimizer_config['lr_type']] # set loss class_criterion = nn.CrossEntropyLoss() transfer_criterion = nn.BCELoss() loss_params = config['loss'] # train len_train_source = len(data_set_loader['source']) len_train_taget = len(data_set_loader['target']) best_acc = 0.0 since = time.time() for num_iter in tqdm(range(config['max_iter'])): if num_iter % config['val_iter'] == 0: base_net.train(False) f1_net.train(False) f2_net.train(False) temp_acc = val(data_set_loader, base_net, f1_net, f2_net, test_10crop=config['prep']['test_10crop'], config=config, num_iter=num_iter) if temp_acc > best_acc: best_acc = temp_acc log_str = 'iter: {:d}, accu: {:.4f}\ntime: {:.4f}'.format(num_iter, temp_acc, time.time() - since) config['logger'].logger.debug(log_str) config['results'][num_iter].append(temp_acc) if num_iter % config['snapshot_iter'] == 0: # TODO pass base_net.train(True) f1_net.train(True) f2_net.train(True) ad_net.train(True) optimizer = lr_scheduler(optimizer, num_iter, **schedule_param) optimizer.zero_grad() if num_iter % len_train_source == 0: iter_source = iter(data_set_loader['source']) if num_iter % len_train_taget == 0: iter_target = iter(data_set_loader['target']) input_source, label_source = iter_source.next() input_target, _ = iter_target.next() input_source, label_source, input_target = input_source.cuda(), label_source.cuda(), input_target.cuda() inputs = torch.cat((input_source, input_target), 0) batch_size = len(label_source) # class-wise adaptation features = base_net(inputs) alpha = 2. / (1. + np.exp(-10 * float(num_iter / config['max_iter']))) - 1 output1 = f1_net(features, alpha) output2 = f2_net(features, alpha) output_s1 = output1[:batch_size, :] output_s2 = output2[:batch_size, :] output_t1 = output1[batch_size:, :] output_t2 = output2[batch_size:, :] output_t1 = F.softmax(output_t1) output_t2 = F.softmax(output_t2) inconsistency_loss = torch.mean(torch.abs(output_t1 - output_t2)) classifier_loss1 = class_criterion(output_s1, label_source) classifier_loss2 = class_criterion(output_s2, label_source) classifier_loss = classifier_loss1 + classifier_loss2 # domain-wise adaptation domain_labels = torch.from_numpy(np.array([[1]] * batch_size + [[0]] * batch_size)).float().cuda() domain_output = ad_net(features, alpha) transfer_loss = transfer_criterion(domain_output, domain_labels) total_loss = classifier_loss + loss_params['domain_off'] * transfer_loss \ - loss_params['dis_off'] * inconsistency_loss total_loss.backward() optimizer.step() if num_iter % config['val_iter'] == 0: print('class:', classifier_loss.item(), 'domain:', transfer_loss.item() * loss_params['domain_off'], 'inconsistency:', inconsistency_loss.item() * loss_params['dis_off'], 'total:', total_loss.item()) if config['is_writer']: config['writer'].add_scalars('train', {'total': total_loss.item(), 'class': classifier_loss.item(), 'domain': transfer_loss.item() * loss_params['domain_off'], 'inconsistency': inconsistency_loss.item() * loss_params[ 'dis_off']}, num_iter) if config['is_writer']: config['writer'].close() return best_acc
def train(config): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["train_set1"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) prep_dict["train_set2"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] dsets["train_set1"] = ImageList(open(data_config["train_set1"]["list_path"]).readlines(), \ transform=prep_dict["train_set1"]) dset_loaders["train_set1"] = util_data.DataLoader(dsets["train_set1"], \ batch_size=data_config["train_set1"]["batch_size"], \ shuffle=True, num_workers=4) dsets["train_set2"] = ImageList(open(data_config["train_set2"]["list_path"]).readlines(), \ transform=prep_dict["train_set2"]) dset_loaders["train_set2"] = util_data.DataLoader(dsets["train_set2"], \ batch_size=data_config["train_set2"]["batch_size"], \ shuffle=True, num_workers=4) hash_bit = config["hash_bit"] ## set base network net_config = config["network"] base_network = net_config["type"](**net_config["params"]) use_gpu = torch.cuda.is_available() if use_gpu: base_network = base_network.cuda() ## collect parameters parameter_list = [{"params":base_network.feature_layers.parameters(), "lr":1}, \ {"params":base_network.hash_layer.parameters(), "lr":10}] ## set optimizer optimizer_config = config["optimizer"] optimizer = optim_dict[optimizer_config["type"]](parameter_list, \ **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] ## train len_train1 = len(dset_loaders["train_set1"]) - 1 len_train2 = len(dset_loaders["train_set2"]) - 1 transfer_loss_value = classifier_loss_value = total_loss_value = 0.0 best_acc = 0.0 for i in range(config["num_iterations"]): if i % config["snapshot_interval"] == 0: torch.save(nn.Sequential(base_network), osp.join(config["output_path"], \ "iter_{:05d}_model.pth.tar".format(i))) ## train one iter base_network.train(True) optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train1 == 0: iter1 = iter(dset_loaders["train_set1"]) if i % len_train2 == 0: iter2 = iter(dset_loaders["train_set2"]) inputs1, labels1 = iter1.next() # 取出train_set1的 image和对应的labels inputs2, labels2 = iter2.next() # 随机显示一个batch_size的图像,batch=36 # print(inputs1.shape) # torch.Size([36, 3, 224, 224]) batch_size 36 图片224*224*3通道 # cv2.imshow(torchvision.utils.make_grid(inputs1)) if use_gpu: inputs1, inputs2, labels1, labels2 = \ Variable(inputs1).cuda(), Variable(inputs2).cuda(), \ Variable(labels1).cuda(), Variable(labels2).cuda() else: inputs1, inputs2, labels1, labels2 = Variable(inputs1), \ Variable(inputs2), Variable(labels1), Variable(labels2) inputs = torch.cat((inputs1, inputs2), dim=0) # print(inputs.shape) # torch.Size([72, 3, 224, 224]) outputs = base_network(inputs) # print(outputs.shape) # torch.Size([72, 48]) 48是Hash code的bits # narrow: https://blog.csdn.net/u011961856/article/details/78696146 # 返回的是估计的labels? # 计算一次前向传播的loss similarity_loss = loss.pairwise_loss(outputs.narrow(0,0,inputs1.size(0)), \ outputs.narrow(0,inputs1.size(0),inputs2.size(0)), \ labels1, labels2, \ sigmoid_param=config["loss"]["sigmoid_param"], \ l_threshold=config["loss"]["l_threshold"], \ class_num=config["loss"]["class_num"]) similarity_loss.backward() # 反向传播 print("Iter: {:05d}, loss: {:.3f}".format( i, similarity_loss.float().item())) config["out_file"].write("Iter: {:05d}, loss: {:.3f} \n".format(i, \ similarity_loss.float().item())) # print("Iter: {:05d}, loss: {:.3f}".format(i, similarity_loss.float().data[0])) # config["out_file"].write("Iter: {:05d}, loss: {:.3f}".format(i, \ # similarity_loss.float().data[0])) optimizer.step()
def train(config): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train(**config["prep"]['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop(**config["prep"]['params']) else: prep_dict["test"] = prep.image_test(**config["prep"]['params']) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] train_bs = data_config["source"]["batch_size"] test_bs = data_config["test"]["batch_size"] dsets["source"] = ImageList(open(data_config["source"]["list_path"]).readlines(), \ transform=prep_dict["source"]) dset_loaders["source"] = DataLoader(dsets["source"], batch_size=train_bs, \ shuffle=True, num_workers=4, drop_last=True) dsets["target"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["target"]) dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, \ shuffle=True, num_workers=4, drop_last=True) # if prep_config["test_10crop"]: # for i in range(10): # dsets["test"] = [ImageList(open(data_config["test"]["list_path"]).readlines(), \ # transform=prep_dict["test"][i]) for i in range(10)] # dset_loaders["test"] = [DataLoader(dset, batch_size=test_bs, \ # shuffle=False, num_workers=4) for dset in dsets['test']] # else: # dsets["test"] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ # transform=prep_dict["test"]) # dset_loaders["test"] = DataLoader(dsets["test"], batch_size=test_bs, \ # shuffle=False, num_workers=4) class_num = config["network"]["params"]["class_num"] ## set base network net_config = config["network"] base_network = net_config["name"](**net_config["params"]) base_network = base_network.cuda() ## add additional network for some methods if config["loss"]["random"]: random_layer = network.RandomLayer( [base_network.output_num(), class_num], config["loss"]["random_dim"]) ad_net = network.AdversarialNetwork(config["loss"]["random_dim"], 1024) else: random_layer = None ad_net = network.AdversarialNetwork( base_network.output_num() * class_num, 1024) if config["loss"]["random"]: random_layer.cuda() ad_net = ad_net.cuda() parameter_list = base_network.get_parameters() + ad_net.get_parameters() ## set optimizer optimizer_config = config["optimizer"] optimizer = optimizer_config["type"](parameter_list, \ **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] gpus = config['gpu'].split(',') if len(gpus) > 1: ad_net = nn.DataParallel(ad_net, device_ids=[int(i) for i in gpus]) base_network = nn.DataParallel(base_network, device_ids=[int(i) for i in gpus]) ## train len_train_source = len(dset_loaders["source"]) len_train_target = len(dset_loaders["target"]) transfer_loss_value = classifier_loss_value = total_loss_value = 0.0 best_acc = 0.0 for i in range(config["num_iterations"]): # if i % config["test_interval"] == config["test_interval"] - 1: # base_network.train(False) # temp_acc = image_classification_test(dset_loaders, \ # base_network, test_10crop=prep_config["test_10crop"]) # temp_model = nn.Sequential(base_network) # if temp_acc > best_acc: # best_acc = temp_acc # best_model = temp_model # log_str = "iter: {:05d}, precision: {:.5f}".format(i, temp_acc) # config["out_file"].write(log_str+"\n") # config["out_file"].flush() # print(log_str) if i % config["snapshot_interval"] == 0: torch.save(nn.Sequential(base_network), osp.join(config["output_path"], \ "iter_{:05d}_model.pth.tar".format(i))) loss_params = config["loss"] ## train one iter base_network.train(True) ad_net.train(True) optimizer = lr_scheduler(optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_source == 0: iter_source = iter(dset_loaders["source"]) if i % len_train_target == 0: iter_target = iter(dset_loaders["target"]) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() inputs_source, inputs_target, labels_source = inputs_source.cuda( ), inputs_target.cuda(), labels_source.cuda() features_source, outputs_source = base_network(inputs_source) features_target, outputs_target = base_network(inputs_target) features = torch.cat((features_source, features_target), dim=0) outputs = torch.cat((outputs_source, outputs_target), dim=0) softmax_out = nn.Softmax(dim=1)(outputs) if config['method'] == 'CDAN+E': entropy = loss.Entropy(softmax_out) transfer_loss = loss.CDAN([features, softmax_out], ad_net, entropy, network.calc_coeff(i), random_layer) elif config['method'] == 'CDAN': transfer_loss = loss.CDAN([features, softmax_out], ad_net, None, None, random_layer) elif config['method'] == 'DANN': transfer_loss = loss.DANN(features, ad_net) else: raise ValueError('Method cannot be recognized.') classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) if i % 10 == 0: print('iter: ', i, 'classifier_loss: ', classifier_loss.data, 'transfer_loss: ', transfer_loss.data) total_loss = loss_params["trade_off"] * transfer_loss + classifier_loss total_loss.backward() optimizer.step() torch.save(best_model, osp.join(config["output_path"], "best_model.pth.tar")) return best_acc
def transfer_classification(config): # ---set log writer --- writer = SummaryWriter(log_dir=config["log_dir"]) # set pre-process prep_dict = {} for prep_config in config["prep"]: name = prep_config["name"] prep_dict[name] = {} if prep_config["type"] == "image": prep_dict[name]["test_10crop"] = prep_config["test_10crop"] prep_dict[name]["train"] = prep.image_train(resize_size=prep_config["resize_size"], crop_size=prep_config["crop_size"]) if prep_config["test_10crop"]: prep_dict[name]["test"] = prep.image_test_10crop(resize_size=prep_config["resize_size"], crop_size=prep_config["crop_size"]) else: prep_dict[name]["test"] = prep.image_test(resize_size=prep_config["resize_size"], crop_size=prep_config["crop_size"]) # print(prep_dict) ## set loss class_criterion = nn.CrossEntropyLoss() loss_config = config["loss"] # transfer_criterion = loss.loss_dict[loss_config["name"]] if "params" not in loss_config: loss_config["params"] = {} # print(transfer_criterion) transfer_criterion = DANLoss(**loss_config["params"]) ## prepare data dsets = {} dset_loaders = {} for data_config in config["data"]: name = data_config['name'] dsets[name] = {} dset_loaders[name] = {} ## image data if data_config["type"] == "image": dsets[name]["train"] = ImageList(open(data_config["list_path"]["train"]).readlines(), transform=prep_dict[name]["train"]) dset_loaders[name]["train"] = util_data.DataLoader(dsets[name]["train"], batch_size=data_config["batch_size"]["train"], shuffle=True, num_workers=4) if "test" in data_config["list_path"]: if prep_dict[name]["test_10crop"]: for i in range(10): dsets[name]["test" + str(i)] = ImageList( open(data_config["list_path"]["test"]).readlines(), transform=prep_dict[name]["test"]["val" + str(i)]) dset_loaders[name]["test" + str(i)] = util_data.DataLoader( dsets[name]["test" + str(i)], batch_size=data_config["batch_size"]["test"], shuffle=False, num_workers=4) else: dsets[name]["test"] = ImageList( open(data_config["list_path"]["test"]).readlines(), transform=prep_dict[name]["test"]) dset_loaders[name]["test"] = util_data.DataLoader( dsets[name]["test"], batch_size = data_config["batch_size"]["test"], shuffle=False, num_workers=4) else: if prep_dict[name]["test_10crop"]: for i in range(10): dsets[name]["test" + str(i)] = ImageList( open(data_config["list_path"]["train"]).readlines(), transform=prep_dict[name]["test"]["val" + str(i)]) dset_loaders[name]["test" + str(i)] = util_data.DataLoader( dsets[name]["test" + str(i)], batch_size=data_config["batch_size"]["test"], shuffle=False, num_workers=4) else: dsets[name]["test"] = ImageList(open(data_config["list_path"]["train"]).readlines(), transform=prep_dict[name]["test"]) dset_loaders[data_config["name"]]["test"] = util_data.DataLoader( dsets[data_config["name"]]["test"], batch_size=data_config["batch_size"]["test"], shuffle=False, num_workers=4) class_num = 31 ## set base network net_config = config["network"] base_network = network.network_dict[net_config["name"]]() if net_config["use_bottleneck"]: bottleneck_layer = nn.Linear(base_network.output_num(), net_config["bottleneck_dim"]) classifier_layer = nn.Linear(bottleneck_layer.out_features, class_num) else: classifier_layer = nn.Linear(base_network.output_num(), class_num) ## initialization if net_config["use_bottleneck"]: bottleneck_layer.weight.data.normal_(0, 0.005) bottleneck_layer.bias.data.fill_(0.1) bottleneck_layer = nn.Sequential(bottleneck_layer, nn.ReLU(), nn.Dropout(0.5)) classifier_layer.weight.data.normal_(0, 0.01) classifier_layer.bias.data.fill_(0.0) # print(base_network.state_dict()) # print(classifier_layer.state_dict()) use_gpu = torch.cuda.is_available() if use_gpu: if net_config["use_bottleneck"]: bottleneck_layer = bottleneck_layer.to(device) classifier_layer = classifier_layer.to(device) base_network = base_network.to(device) ## collect parameters if net_config["use_bottleneck"]: parameter_list = [{"params":base_network.parameters(), "lr":1}, {"params":bottleneck_layer.parameters(), "lr":10}, {"params":classifier_layer.parameters(), "lr":10}] else: parameter_list = [{"params":base_network.parameters(), "lr":1}, {"params":classifier_layer.parameters(), "lr":10}] ## add additional network for some methods if loss_config["name"] == "JAN": softmax_layer = nn.Softmax() if use_gpu: softmax_layer = softmax_layer.to(device) ## set optimizer optimizer_config = config["optimizer"] optimizer = optim_dict[optimizer_config["type"]](parameter_list, **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] ## train len_train_source = len(dset_loaders["source"]["train"]) - 1 len_train_target = len(dset_loaders["target"]["train"]) - 1 # transfer_loss_value = classifier_loss_value = total_loss_value = 0.0 for i in range(config["num_iterations"]): # test in the train if i % config["test_interval"] == 0: base_network.train(False) classifier_layer.train(False) if net_config["use_bottleneck"]: bottleneck_layer.train(False) test_model = nn.Sequential(base_network, bottleneck_layer, classifier_layer) test_acc = image_classification_test(dset_loaders["target"], test_model, test_10crop=prep_dict["target"]["test_10crop"], gpu=use_gpu) print("iteration: %d, test accuracy: %f" % (i, test_acc)) else: test_model = nn.Sequential(base_network, classifier_layer) test_acc = image_classification_test(dset_loaders["target"], test_model, test_10crop=prep_dict["target"]["test_10crop"], gpu=use_gpu) print("iteration: %d, test accuracy: %f" % (i, test_acc)) # save model parameters if i % config["checkpoint_iterval"] == 0: torch.save(test_model.state_dict(), config["checkpoint_dir"]+"checkpoint_"+str(i).zfill(5)+".pth") # loss_test = nn.BCELoss() # train one iter base_network.train(True) if net_config["use_bottleneck"]: bottleneck_layer.train(True) classifier_layer.train(True) optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_source == 0: iter_source = iter(dset_loaders["source"]["train"]) if i % len_train_target == 0: iter_target = iter(dset_loaders["target"]["train"]) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() inputs_source = inputs_source.to(device) inputs_target = inputs_target.to(device) labels_source = labels_source.to(device) inputs = torch.cat((inputs_source, inputs_target), dim=0) ### cat the source and target features = base_network(inputs) if net_config["use_bottleneck"]: features = bottleneck_layer(features) outputs = classifier_layer(features) # split the output for different loss num = inputs.size(0)//2 # the premise is the batch size of source and target is the same classifier_loss = class_criterion(outputs.narrow(0, 0, num), labels_source) ## switch between different transfer loss if loss_config["name"] == "DAN": # transfer_loss = transfer_criterion(features.narrow(0, 0, num), # features.narrow(0, num, num), # **loss_config["params"]) transfer_loss = transfer_criterion(features.narrow(0, 0, num), features.narrow(0, num, num)) elif loss_config["name"] == "RTN": ## RTN is still under developing transfer_loss = 0 elif loss_config["name"] == "JAN": softmax_out = softmax_layer(outputs) transfer_loss = transfer_criterion([features.narrow(0, 0, num), softmax_out.narrow(0, 0, num)], [features.narrow(0, num, num), softmax_out.narrow(0, num, num)], **loss_config["params"]) total_loss = loss_config["trade_off"] * transfer_loss + classifier_loss if i % 100 == 0: print("iteration: ", str(i), "\t transfer loss: ", str(transfer_loss.item()), "\t classification loss: ", str(classifier_loss.item()), "\t total loss: ", str(total_loss.item())) for param_group in optimizer.param_groups: print("lr: ", param_group["lr"]) # log for losses writer.add_scalar('Train/loss_classification', classifier_loss.item(), i//100) writer.add_scalar('Train/loss_transfer', transfer_loss.item(), i//100) writer.add_scalar('Train/loss_total', total_loss.item(), i//100) total_loss.backward() optimizer.step()
file_path = { "cub_train":"./dataset_list/cub200_2011_multi_train.txt", "cub_test":"./dataset_list/cub200_2011_multi_test.txt" } dataset_source = file_path["cub_train"] dataset_test = file_path["cub_test"] # dataset load batch_size = {"train": 36, "val": 36, "test": 4} for i in range(10): batch_size["val" + str(i)] = 4 dataset_loaders = {} dataset_list = ImageList(open(dataset_source).readlines(), transform=prep.image_train(resize_size=256, crop_size=224)) dataset_loaders["train"] = torch.utils.data.DataLoader(dataset_list, batch_size=36, shuffle=True, num_workers=4) # dataset_list = ImageList(open(dataset_test).readlines(), transform=prep.image_train(resize_size=256, crop_size=224)) # dataset_loaders["test"] = torch.utils.data.DataLoader(dataset_list, batch_size=36, shuffle=True, num_workers=4) prep_dict_test = prep.image_test_10crop(resize_size=256, crop_size=224) for i in range(10): dataset_list = ImageList(open(dataset_test).readlines(), transform=prep_dict_test["val" + str(i)]) dataset_loaders["val" + str(i)] = torch.utils.data.DataLoader(dataset_list, batch_size=4, shuffle=False, num_workers=4) # fine-grained categories and coarse-grained categories cate_all = [200, 122, 38, 14, 1, 1, 1] num_coarse_cate_sel = 3
def cross_validation_loss(args, feature_network_path, predict_network_path, num_layer, src_list, target_path, val_list, class_num, resize_size, crop_size, batch_size, use_gpu): """ Main function for computing the CV loss :param feature_network: :param predict_network: :param src_cls_list: :param target_path: :param val_cls_list: :param class_num: :param resize_size: :param crop_size: :param batch_size: :return: """ option = 'resnet' + args.resnet G = ResBase(option) F1 = ResClassifier(num_layer=num_layer) G.load_state_dict(torch.load(feature_network_path)) F1.load_state_dict(torch.load(predict_network_path)) if use_gpu: G.cuda() F1.cuda() G.eval() F1.eval() val_list = seperate_data.dimension_rd(val_list) tar_list = open(target_path).readlines() cross_val_loss = 0 prep_dict = prep.image_train(resize_size=resize_size, crop_size=crop_size) # load different class's image dsets_src = ImageList(src_list, transform=prep_dict) dset_loaders_src = util_data.DataLoader(dsets_src, batch_size=batch_size, shuffle=True, num_workers=4) # dsets_val = ImageList(val_list, transform=prep_dict) # dset_loaders_val = util_data.DataLoader(dsets_val, batch_size=batch_size, shuffle=True, num_workers=4) # dsets_tar = ImageList(tar_list, transform=prep_dict) # dset_loaders_tar = util_data.DataLoader(dsets_tar, batch_size=batch_size, shuffle=True, num_workers=4) # prepare source feature iter_src = iter(dset_loaders_src) src_input, src_labels = iter_src.next() if use_gpu: src_input, src_labels = Variable(src_input).cuda(), Variable( src_labels).cuda() else: src_input, src_labels = Variable(src_input), Variable(src_labels) # src_feature, _ = feature_network(src_input) feature_val = G(src_input) src_feature_de = feature_val.detach().detach().cpu().numpy() # src_feature_de = src_feature.detach().cpu().numpy() for _ in range(len(dset_loaders_src) - 1): src_input, src_labels = iter_src.next() if use_gpu: src_input, src_labels = Variable(src_input).cuda(), Variable( src_labels).cuda() else: src_input, src_labels = Variable(src_input), Variable(src_labels) # src_feature_new, _ = feature_network(src_input) # print("Src_input: {}".format(src_input)) # print("Src_shape: {}".format(src_input.shape)) src_feature_new = G(src_input) src_feature_new_de = src_feature_new.detach().cpu().numpy() # src_feature_new_de = src_feature_new.detach().cpu().numpy() src_feature_de = np.append(src_feature_de, src_feature_new_de, axis=0) # src_feature = torch.cat((src_feature, src_feature_new), 0) print("Pass Source") # prepare target feature dsets_tar = ImageList(tar_list, transform=prep_dict) dset_loaders_tar = util_data.DataLoader(dsets_tar, batch_size=batch_size, shuffle=True, num_workers=4) iter_tar = iter(dset_loaders_tar) tar_input, _ = iter_tar.next() if use_gpu: tar_input, _ = Variable(tar_input).cuda(), Variable(_).cuda() else: src_input, _ = Variable(tar_input), Variable(_) # tar_feature, _ = feature_network(tar_input) tar_feature = G(tar_input) tar_feature_de = tar_feature.detach().cpu().numpy() # tar_feature_de = tar_feature.detach().cpu().numpy() for _ in range(len(dset_loaders_tar) - 1): tar_input, _ = iter_tar.next() if use_gpu: tar_input, _ = Variable(tar_input).cuda(), Variable(_).cuda() else: src_input, _ = Variable(tar_input), Variable(_) # tar_feature_new, _ = feature_network(tar_input) tar_feature_new = G(tar_input) tar_feature_new_de = tar_feature_new.detach().cpu().numpy() # tar_feature_new_de = tar_feature_new.detach().cpu().numpy() tar_feature_de = np.append(tar_feature_de, tar_feature_new_de, axis=0) # tar_feature = torch.cat((tar_feature, tar_feature_new), 0) print("Pass Target") # prepare validation feature and predicted label for validation dsets_val = ImageList(val_list, transform=prep_dict) dset_loaders_val = util_data.DataLoader(dsets_val, batch_size=batch_size, shuffle=True, num_workers=4) iter_val = iter(dset_loaders_val) val_input, val_labels = iter_val.next() if use_gpu: val_input, val_labels = Variable(val_input).cuda(), Variable( val_labels).cuda() else: val_input, val_labels = Variable(val_input), Variable(val_labels) # val_feature, _ = feature_network(val_input) # _, pred_label = predict_network(val_input) val_feature = G(val_input) pred_label = F1(val_feature) val_feature_de = val_feature.detach().cpu().numpy() # val_feature_de = val_feature.detach().cpu().numpy() w = pred_label[0].shape[0] error = np.zeros(1) error[0] = predict_loss(val_labels[0].item(), pred_label[0].reshape(1, w)).item() error = error.reshape(1, 1) print("Before the final") print(pred_label.shape) print(len(val_feature_de)) for num_image in range(1, len(pred_label)): new_error = np.zeros(1) single_pred_label = pred_label[num_image] w = single_pred_label.shape[0] single_val_label = val_labels[num_image] new_error[0] = predict_loss(single_val_label.item(), single_pred_label.reshape(1, w)).item() new_error = new_error.reshape(1, 1) error = np.append(error, new_error, axis=0) for _ in range(len(dset_loaders_val) - 1): val_input, val_labels = iter_val.next() if use_gpu: val_input, val_labels = Variable(val_input).cuda(), Variable( val_labels).cuda() else: val_input, val_labels = Variable(val_input), Variable(val_labels) # val_feature_new, _ = feature_network(val_input) val_feature_new = G(val_input) val_feature_new_de = val_feature_new.detach().cpu().numpy() # val_feature_new_de = val_feature_new.detach().cpu().numpy() val_feature_de = np.append(val_feature_de, val_feature_new_de, axis=0) # val_feature = torch.cat((val_feature, val_feature_new), 0) # _, pred_label = predict_network(val_input) pred_label = F1(val_feature_new) for num_image in range(len(pred_label)): new_error = np.zeros(1) single_pred_label = pred_label[num_image] w = single_pred_label.shape[0] single_val_label = val_labels[num_image] new_error[0] = predict_loss(single_val_label.item(), single_pred_label.reshape(1, w)).item() new_error = new_error.reshape(1, 1) error = np.append(error, new_error, axis=0) print("Pass validation") # print("Insides the for loop") # print(len(error)) # print(len(val_feature_de)) # # print("Input for scrore calculation: ") # print(len(error)) # print(len(val_feature_de)) print(src_feature_de) print(tar_feature_de) print(val_feature_de) weight = get_weight(src_feature_de, tar_feature_de, val_feature_de) print(weight) print(error) cross_val_loss = cross_val_loss + get_dev_risk(weight, error) print(cross_val_loss) return cross_val_loss
def train(config): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train(**config["prep"]['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop(**config["prep"]['params']) else: prep_dict["test"] = prep.image_test(**config["prep"]['params']) tensor_writer = SummaryWriter(config["tensorboard_path"]) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] train_bs = data_config["source"]["batch_size"] test_bs = data_config["test"]["batch_size"] dsets["source"] = ImageList(open(data_config["source"]["list_path"]).readlines(), \ transform=prep_dict["source"]) dset_loaders["source"] = DataLoader(dsets["source"], batch_size=train_bs, \ shuffle=True, num_workers=4, drop_last=True) dsets["target"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["target"]) dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, \ shuffle=True, num_workers=4, drop_last=True) if prep_config["test_10crop"]: for i in range(10): dsets["test"] = [ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"][i]) for i in range(10)] dset_loaders["test"] = [DataLoader(dset, batch_size=test_bs, \ shuffle=False, num_workers=4) for dset in dsets['test']] else: dsets["test"] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["test"] = DataLoader(dsets["test"], batch_size=test_bs, \ shuffle=False, num_workers=4) class_num = config["network"]["params"]["class_num"] ## set base network net_config = config["network"] base_network = net_config["name"](**net_config["params"]) base_network = base_network.cuda() parameter_list = base_network.get_parameters() ## set optimizer optimizer_config = config["optimizer"] optimizer = optimizer_config["type"](parameter_list, \ **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] gpus = config['gpu'].split(',') if len(gpus) > 1: base_network = nn.DataParallel(base_network, device_ids=[int(i) for i in gpus]) ## train len_train_source = len(dset_loaders["source"]) len_train_target = len(dset_loaders["target"]) best_acc = 0.0 for i in range(config["num_iterations"]): if i % config["test_interval"] == config["test_interval"] - 1 or i==0: base_network.train(False) temp_acc, output, prediction, label, feature = image_classification_test(dset_loaders, \ base_network, test_10crop=prep_config["test_10crop"]) temp_model = nn.Sequential(base_network) if temp_acc > best_acc: best_acc = temp_acc best_model = temp_model log_str = "iter: {:05d}, precision: {:.5f}".format(i, temp_acc) config["out_file"].write(log_str+"\n") config["out_file"].flush() print(log_str) if i % config["snapshot_interval"] == 0: torch.save(nn.Sequential(base_network), osp.join(config["output_path"], \ "iter_{:05d}_model.pth.tar".format(i))) loss_params = config["loss"] ## train one iter base_network.train(True) optimizer = lr_scheduler(optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_source == 0: iter_source = iter(dset_loaders["source"]) if i % len_train_target == 0: iter_target = iter(dset_loaders["target"]) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() inputs_source, inputs_target, labels_source = inputs_source.cuda(), inputs_target.cuda(), labels_source.cuda() features_source, outputs_source = base_network(inputs_source) features_target, outputs_target = base_network(inputs_target) outputs_target_temp = outputs_target / config['temperature'] target_softmax_out_temp = nn.Softmax(dim=1)(outputs_target_temp) target_entropy_weight = loss.Entropy(target_softmax_out_temp).detach() target_entropy_weight = 1 + torch.exp(-target_entropy_weight) target_entropy_weight = train_bs * target_entropy_weight / torch.sum(target_entropy_weight) cov_matrix_t = target_softmax_out_temp.mul(target_entropy_weight.view(-1,1)).transpose(1,0).mm(target_softmax_out_temp) cov_matrix_t = cov_matrix_t / torch.sum(cov_matrix_t, dim=1) mcc_loss = (torch.sum(cov_matrix_t) - torch.trace(cov_matrix_t)) / class_num classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) total_loss = classifier_loss + mcc_loss total_loss.backward() optimizer.step() tensor_writer.add_scalar('total_loss', total_loss, i) tensor_writer.add_scalar('classifier_loss', classifier_loss, i) tensor_writer.add_scalar('cov_matrix_penalty', mcc_loss, i) torch.save(best_model, osp.join(config["output_path"], "best_model.pth.tar")) return best_acc
def train(config): #################################################### # Tensorboard setting #################################################### #tensor_writer = SummaryWriter(config["tensorboard_path"]) #################################################### # Data setting #################################################### prep_dict = {} # 데이터 전처리 transforms 부분 prep_dict["source"] = prep.image_train(**config['prep']['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) prep_dict["test"] = prep.image_test(**config['prep']['params']) dsets = {} dsets["source"] = datasets.ImageFolder(config['s_dset_path'], transform=prep_dict["source"]) dsets["target"] = datasets.ImageFolder(config['t_dset_path'], transform=prep_dict['target']) dsets['test'] = datasets.ImageFolder(config['t_dset_path'], transform=prep_dict['test']) data_config = config["data"] train_source_bs = data_config["source"][ "batch_size"] #원본은 source와 target 모두 source train bs로 설정되었는데 이를 수정함 train_target_bs = data_config['target']['batch_size'] test_bs = data_config["test"]["batch_size"] dset_loaders = {} dset_loaders["source"] = DataLoader( dsets["source"], batch_size=train_source_bs, shuffle=True, num_workers=4, drop_last=True ) # 원본은 drop_last=True, 이렇게 해야 마지막까지 source, target에서 동일한 수로 배치 생성가능 dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_target_bs, shuffle=True, num_workers=4, drop_last=True) dset_loaders['test'] = DataLoader(dsets['test'], batch_size=test_bs, shuffle=False, num_workers=4, drop_last=False) #################################################### # Network Setting #################################################### class_num = config["network"]['params']['class_num'] net_config = config["network"] """ config['network'] = {'name': network.ResNetFc, 'params': {'resnet_name': args.net, 'use_bottleneck': True, 'bottleneck_dim': 256, 'new_cls': True, 'class_num': args.class_num, 'type' : args.type} } """ base_network = net_config["name"](**net_config["params"]) #network.py에 정의된 ResNetFc() 클래스 호출 base_network = base_network.cuda() # ResNetFc(Resnet, True, 256, True, 12) if config["loss"]["random"]: random_layer = network.RandomLayer( [base_network.output_num(), class_num], config["loss"]["random_dim"]) random_layer.cuda() ad_net = network.AdversarialNetwork(config["loss"]["random_dim"], 1024) else: random_layer = None ad_net = network.AdversarialNetwork( base_network.output_num() * class_num, 1024) # 왜 class 수 만큼 곱하지? ad_net = ad_net.cuda() parameter_list = base_network.get_parameters() + ad_net.get_parameters() #################################################### # Env Setting #################################################### #gpus = config['gpu'].split(',') #if len(gpus) > 1 : #ad_net = nn.DataParallel(ad_net, device_ids=[int(i) for i in gpus]) #base_network = nn.DataParallel(base_network, device_ids=[int(i) for i in gpus]) #################################################### # Optimizer Setting #################################################### optimizer_config = config['optimizer'] optimizer = optimizer_config["type"](parameter_list, **(optimizer_config["optim_params"])) # optim.SGD #config['optimizer'] = {'type': optim.SGD, #'optim_params': {'lr': args.lr, #'momentum': 0.9, #'weight_decay': 0.0005, #'nestrov': True}, #'lr_type': "inv", #'lr_param': {"lr": args.lr, #'gamma': 0.001, # 이거 0.01이여야 하지 않나? #'power': 0.75 #} param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group['lr']) schedule_param = optimizer_config['lr_param'] lr_scheduler = lr_schedule.schedule_dict[ optimizer_config["lr_type"]] # return optimizer #################################################### # Train #################################################### len_train_source = len(dset_loaders["source"]) len_train_target = len(dset_loaders["target"]) transfer_loss_value = 0.0 classifier_loss_value = 0.0 total_loss_value = 0.0 best_acc = 0.0 batch_size = config["data"]["source"]["batch_size"] for i in range( config["num_iterations"]): # num_iterations수의 batch가 학습에 사용됨 sys.stdout.write("Iteration : {} \r".format(i)) sys.stdout.flush() loss_params = config["loss"] base_network.train(True) ad_net.train(True) optimizer = lr_scheduler(optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_source == 0: iter_source = iter(dset_loaders["source"]) if i % len_train_target == 0: iter_target = iter(dset_loaders["target"]) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() inputs_source, labels_source = inputs_source.cuda( ), labels_source.cuda() inputs_target = inputs_target.cuda() inputs = torch.cat((inputs_source, inputs_target), dim=0) features, outputs, tau, cur_mean_source, cur_mean_target, output_mean_source, output_mean_target = base_network( inputs) softmax_out = nn.Softmax(dim=1)(outputs) outputs_source = outputs[:batch_size] outputs_target = outputs[batch_size:] if config['method'] == 'CDAN+E' or config['method'] == 'CDAN_TransNorm': entropy = loss.Entropy(softmax_out) transfer_loss = loss.CDAN([features, softmax_out], ad_net, entropy, network.calc_coeff(i), random_layer) elif config['method'] == 'CDAN': transfer_loss = loss.CDAN([features, softmax_out], ad_net, None, None, random_layer) elif config['method'] == 'DANN': pass # 나중에 정리하기 else: raise ValueError('Method cannot be recognized') classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) total_loss = loss_params["trade_off"] * transfer_loss + classifier_loss total_loss.backward() optimizer.step() #tensor_writer.add_scalar('total_loss', total_loss.i ) #tensor_writer.add_scalar('classifier_loss', classifier_loss, i) #tensor_writer.add_scalar('transfer_loss', transfer_loss, i) #################################################### # Test #################################################### if i % config["test_interval"] == config["test_interval"] - 1: # test interval 마다 base_network.train(False) temp_acc = image_classification_test(dset_loaders, base_network) temp_model = nn.Sequential(base_network) if temp_acc > best_acc: best_acc = temp_acc best_model = temp_model ACC = round(best_acc, 2) * 100 torch.save( best_model, os.path.join(config["output_path"], "iter_{}_model.pth.tar".format(ACC))) log_str = "iter: {:05d}, precision: {:.5f}".format(i, temp_acc) config["out_file"].write(log_str + "\n") config["out_file"].flush() print(log_str)
def train(config): base_network = network.ResNetFc('ResNet50', use_bottleneck=True, bottleneck_dim=config["bottleneck_dim"], new_cls=True, class_num=config["class_num"]) ad_net = network.AdversarialNetwork(config["bottleneck_dim"], config["hidden_dim"]) base_network = base_network.cuda() ad_net = ad_net.cuda() parameter_list = base_network.get_parameters() + ad_net.get_parameters() source_path = ImageList(open(config["s_path"]).readlines(), transform=preprocess.image_train(resize_size=256, crop_size=224)) target_path = ImageList(open(config["t_path"]).readlines(), transform=preprocess.image_train(resize_size=256, crop_size=224)) test_path = ImageList(open(config["t_path"]).readlines(), transform=preprocess.image_test(resize_size=256, crop_size=224)) source_loader = DataLoader(source_path, batch_size=config["train_bs"], shuffle=True, num_workers=0, drop_last=True) target_loader = DataLoader(target_path, batch_size=config["train_bs"], shuffle=True, num_workers=0, drop_last=True) test_loader = DataLoader(test_path, batch_size=config["test_bs"], shuffle=True, num_workers=0, drop_last=True) optimizer_config = config["optimizer"] optimizer = optimizer_config["type"](parameter_list, \ **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] gpus = config["gpus"].split(',') if len(gpus) > 1: ad_net = nn.DataParallel(ad_net, device_ids=[int(i) for i in gpus]) base_network = nn.DataParallel(base_network, device_ids=[int(i) for i in gpus]) len_train_source = len(source_loader) len_train_target = len(target_loader) transfer_loss_value = classifier_loss_value = total_loss_value = 0.0 best_acc = 0.0 best_model_path = None for i in trange(config["iterations"], leave=False): if i % config["test_interval"] == config["test_interval"] - 1: base_network.train(False) temp_acc = image_classification_test(test_loader, base_network) temp_model = nn.Sequential(base_network) if temp_acc > best_acc: best_acc = temp_acc best_model = copy.deepcopy(temp_model) best_iter = i if best_model_path and osp.exists(best_model_path): try: os.remove(best_model_path) except: pass best_model_path = osp.join(config["output_path"], "iter_{:05d}.pth.tar".format(best_iter)) torch.save(best_model, best_model_path) log_str = "iter: {:05d}, precision: {:.5f}".format(i, temp_acc) config["out_file"].write(log_str+"\n") config["out_file"].flush() # print("cut_loss: ", cut_loss.item()) print("mix_loss: ", mix_loss.item()) print(log_str) base_network.train(True) ad_net.train(True) optimizer = lr_scheduler(optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_source == 0: iter_source = iter(source_loader) if i % len_train_target == 0: iter_target = iter(target_loader) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() inputs_source, inputs_target, labels_source = inputs_source.cuda(), inputs_target.cuda(), labels_source.cuda() labels_src_one_hot = torch.nn.functional.one_hot(labels_source, config["class_num"]).float() # inputs_cut, labels_cut = cutmix(base_network, inputs_source, labels_src_one_hot, inputs_target, config["alpha"], config["class_num"]) inputs_mix, labels_mix = mixup(base_network, inputs_source, labels_src_one_hot, inputs_target, config["alpha"], config["class_num"], config["temperature"]) features_source, outputs_source = base_network(inputs_source) features_target, outputs_target = base_network(inputs_target) # features_cut, outputs_cut = base_network(inputs_cut) features_mix, outputs_mix = base_network(inputs_mix) features = torch.cat((features_source, features_target), dim=0) outputs = torch.cat((outputs_source, outputs_target), dim=0) softmax_out = nn.Softmax(dim=1)(outputs) if config["method"] == 'DANN': transfer_loss = loss.DANN(features, ad_net) # cut_loss = utils.kl_loss(outputs_cut, labels_cut.detach()) mix_loss = utils.kl_loss(outputs_mix, labels_mix.detach()) else: raise ValueError('Method cannot be recognized.') classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) total_loss = transfer_loss + classifier_loss + (5*mix_loss) total_loss.backward() optimizer.step() torch.save(best_model, osp.join(config["output_path"], "best_model.pth.tar")) print("Training Finished! Best Accuracy: ", best_acc) return best_acc
def train(config): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train(**config["prep"]['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop(**config["prep"]['params']) else: prep_dict["test"] = prep.image_test(**config["prep"]['params']) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] train_bs = data_config["source"]["batch_size"] test_bs = data_config["test"]["batch_size"] dsets["source"] = ImageList(open(data_config["source"]["list_path"]).readlines(), \ transform=prep_dict["source"]) dset_loaders["source"] = DataLoader(dsets["source"], batch_size=train_bs, \ shuffle=True, num_workers=0, drop_last=True) dsets["target"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["target"]) dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, \ shuffle=True, num_workers=0, drop_last=True) if prep_config["test_10crop"]: for i in range(10): dsets["test"] = [ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"][i]) for i in range(10)] dset_loaders["test"] = [DataLoader(dset, batch_size=test_bs, \ shuffle=False, num_workers=0) for dset in dsets['test']] else: dsets["test"] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["test"] = DataLoader(dsets["test"], batch_size=test_bs, \ shuffle=False, num_workers=0) class_num = config["network"]["params"]["class_num"] ## set base network net_config = config["network"] base_network = net_config["name"](**net_config["params"]) # base_network = base_network.cuda() ## 添加判别器D_s,D_t,生成器G_s2t,G_t2s z_dimension = 256 D_s = network.models["Discriminator"]() # D_s = D_s.cuda() G_s2t = network.models["Generator"](z_dimension, 1024) # G_s2t = G_s2t.cuda() D_t = network.models["Discriminator"]() # D_t = D_t.cuda() G_t2s = network.models["Generator"](z_dimension, 1024) # G_t2s = G_t2s.cuda() criterion_GAN = torch.nn.MSELoss() criterion_cycle = torch.nn.L1Loss() criterion_identity = torch.nn.L1Loss() criterion_Sem = torch.nn.L1Loss() optimizer_G = torch.optim.Adam(itertools.chain(G_s2t.parameters(), G_t2s.parameters()), lr=0.0003) optimizer_D_s = torch.optim.Adam(D_s.parameters(), lr=0.0003) optimizer_D_t = torch.optim.Adam(D_t.parameters(), lr=0.0003) fake_S_buffer = ReplayBuffer() fake_T_buffer = ReplayBuffer() classifier_optimizer = torch.optim.Adam(base_network.parameters(), lr=0.0003) ## 添加分类器 classifier1 = net.Net(256,class_num) # classifier1 = classifier1.cuda() classifier1_optim = optim.Adam(classifier1.parameters(), lr=0.0003) ## add additional network for some methods if config["loss"]["random"]: random_layer = network.RandomLayer([base_network.output_num(), class_num], config["loss"]["random_dim"]) ad_net = network.AdversarialNetwork(config["loss"]["random_dim"], 1024) else: random_layer = None ad_net = network.AdversarialNetwork(base_network.output_num() * class_num, 1024) if config["loss"]["random"]: random_layer.cuda() # ad_net = ad_net.cuda() parameter_list = base_network.get_parameters() + ad_net.get_parameters() ## set optimizer optimizer_config = config["optimizer"] optimizer = optimizer_config["type"](parameter_list, \ **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] gpus = config['gpu'].split(',') if len(gpus) > 1: ad_net = nn.DataParallel(ad_net, device_ids=[int(i) for i in gpus]) base_network = nn.DataParallel(base_network, device_ids=[int(i) for i in gpus]) ## train len_train_source = len(dset_loaders["source"]) len_train_target = len(dset_loaders["target"]) transfer_loss_value = classifier_loss_value = total_loss_value = 0.0 best_acc = 0.0 for i in range(config["num_iterations"]): if i % config["test_interval"] == config["test_interval"] - 1: base_network.train(False) temp_acc = image_classification_test(dset_loaders, \ base_network, test_10crop=prep_config["test_10crop"]) temp_model = nn.Sequential(base_network) if temp_acc > best_acc: best_acc = temp_acc best_model = temp_model now = datetime.datetime.now() d = str(now.month) + '-' + str(now.day) + ' ' + str(now.hour) + ':' + str(now.minute) + ":" + str( now.second) torch.save(best_model, osp.join(config["output_path"], "{}_to_{}_best_model_acc-{}_{}.pth.tar".format(args.source, args.target, best_acc, d))) log_str = "iter: {:05d}, precision: {:.5f}".format(i, temp_acc) config["out_file"].write(log_str + "\n") config["out_file"].flush() print(log_str) if i % config["snapshot_interval"] == 0: torch.save(nn.Sequential(base_network), osp.join(config["output_path"], \ "{}_to_{}_iter_{:05d}_model_{}.pth.tar".format(args.source, args.target, i, str( datetime.datetime.utcnow())))) print("it_train: {:05d} / {:05d} start".format(i, config["num_iterations"])) loss_params = config["loss"] ## train one iter classifier1.train(True) base_network.train(True) ad_net.train(True) optimizer = lr_scheduler(optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_source == 0: iter_source = iter(dset_loaders["source"]) if i % len_train_target == 0: iter_target = iter(dset_loaders["target"]) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() # inputs_source, inputs_target, labels_source = inputs_source.cuda(), inputs_target.cuda(), labels_source.cuda() # 提取特征 features_source, outputs_source = base_network(inputs_source) features_target, outputs_target = base_network(inputs_target) features = torch.cat((features_source, features_target), dim=0) outputs = torch.cat((outputs_source, outputs_target), dim=0) softmax_out = nn.Softmax(dim=1)(outputs) outputs_source1 = classifier1(features_source.detach()) outputs_target1 = classifier1(features_target.detach()) outputs1 = torch.cat((outputs_source1,outputs_target1),dim=0) softmax_out1 = nn.Softmax(dim=1)(outputs1) softmax_out = (1-args.cla_plus_weight)*softmax_out + args.cla_plus_weight*softmax_out1 if config['method'] == 'CDAN+E': entropy = loss.Entropy(softmax_out) transfer_loss = loss.CDAN([features, softmax_out], ad_net, entropy, network.calc_coeff(i), random_layer) elif config['method'] == 'CDAN': transfer_loss = loss.CDAN([features, softmax_out], ad_net, None, None, random_layer) elif config['method'] == 'DANN': transfer_loss = loss.DANN(features, ad_net) else: raise ValueError('Method cannot be recognized.') classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) # Cycle num_feature = features_source.size(0) # =================train discriminator T real_label = Variable(torch.ones(num_feature)) # real_label = Variable(torch.ones(num_feature)).cuda() fake_label = Variable(torch.zeros(num_feature)) # fake_label = Variable(torch.zeros(num_feature)).cuda() # 训练生成器 optimizer_G.zero_grad() # Identity loss same_t = G_s2t(features_target.detach()) loss_identity_t = criterion_identity(same_t, features_target) same_s = G_t2s(features_source.detach()) loss_identity_s = criterion_identity(same_s, features_source) # Gan loss fake_t = G_s2t(features_source.detach()) pred_fake = D_t(fake_t) loss_G_s2t = criterion_GAN(pred_fake, labels_source.float()) fake_s = G_t2s(features_target.detach()) pred_fake = D_s(fake_s) loss_G_t2s = criterion_GAN(pred_fake, labels_source.float()) # cycle loss recovered_s = G_t2s(fake_t) loss_cycle_sts = criterion_cycle(recovered_s, features_source) recovered_t = G_s2t(fake_s) loss_cycle_tst = criterion_cycle(recovered_t, features_target) # sem loss pred_recovered_s = base_network.fc(recovered_s) pred_fake_t = base_network.fc(fake_t) loss_sem_t2s = criterion_Sem(pred_recovered_s, pred_fake_t) pred_recovered_t = base_network.fc(recovered_t) pred_fake_s = base_network.fc(fake_s) loss_sem_s2t = criterion_Sem(pred_recovered_t, pred_fake_s) loss_cycle = loss_cycle_tst + loss_cycle_sts weights = args.weight_in_lossG.split(',') loss_G = float(weights[0]) * (loss_identity_s + loss_identity_t) + \ float(weights[1]) * (loss_G_s2t + loss_G_t2s) + \ float(weights[2]) * loss_cycle + \ float(weights[3]) * (loss_sem_s2t + loss_sem_t2s) # 训练softmax分类器 outputs_fake = classifier1(fake_t.detach()) # 分类器优化 classifier_loss1 = nn.CrossEntropyLoss()(outputs_fake, labels_source) classifier1_optim.zero_grad() classifier_loss1.backward() classifier1_optim.step() total_loss = loss_params["trade_off"] * transfer_loss + classifier_loss + args.cyc_loss_weight*loss_G total_loss.backward() optimizer.step() optimizer_G.step() ###### Discriminator S ###### optimizer_D_s.zero_grad() # Real loss pred_real = D_s(features_source.detach()) loss_D_real = criterion_GAN(pred_real, real_label) # Fake loss fake_s = fake_S_buffer.push_and_pop(fake_s) pred_fake = D_s(fake_s.detach()) loss_D_fake = criterion_GAN(pred_fake, fake_label) # Total loss loss_D_s = loss_D_real + loss_D_fake loss_D_s.backward() optimizer_D_s.step() ################################### ###### Discriminator t ###### optimizer_D_t.zero_grad() # Real loss pred_real = D_t(features_target.detach()) loss_D_real = criterion_GAN(pred_real, real_label) # Fake loss fake_t = fake_T_buffer.push_and_pop(fake_t) pred_fake = D_t(fake_t.detach()) loss_D_fake = criterion_GAN(pred_fake, fake_label) # Total loss loss_D_t = loss_D_real + loss_D_fake loss_D_t.backward() optimizer_D_t.step() print("it_train: {:05d} / {:05d} over".format(i, config["num_iterations"])) now = datetime.datetime.now() d = str(now.month)+'-'+str(now.day)+' '+str(now.hour)+':'+str(now.minute)+":"+str(now.second) torch.save(best_model, osp.join(config["output_path"], "{}_to_{}_best_model_acc-{}_{}.pth.tar".format(args.source, args.target, best_acc,d))) return best_acc
def test(config): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) prep_dict["target"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) else: prep_dict["test"] = prep.image_test( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] dsets["source"] = ImageList(open(data_config["source"]["list_path"]).readlines(), \ transform=prep_dict["source"]) dset_loaders["source"] = util_data.DataLoader(dsets["source"], \ batch_size=data_config["source"]["batch_size"], \ shuffle=True, num_workers=2) dsets["target"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["target"]) dset_loaders["target"] = util_data.DataLoader(dsets["target"], \ batch_size=data_config["target"]["batch_size"], \ shuffle=True, num_workers=2) if prep_config["test_10crop"]: for i in range(10): dsets["test"+str(i)] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]["val"+str(i)]) dset_loaders["test"+str(i)] = util_data.DataLoader(dsets["test"+str(i)], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=2) dsets["target"+str(i)] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["test"]["val"+str(i)]) dset_loaders["target"+str(i)] = util_data.DataLoader(dsets["target"+str(i)], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=2) else: dsets["test"] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["test"] = util_data.DataLoader(dsets["test"], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=2) dsets["target_test"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["target_test"] = util_data.DataLoader(dsets["target_test"], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=2) class_num = config["network"]["params"]["class_num"] # load checkpoint print('load model from {}'.format(config['ckpt_path'])) # load in an old way # base_network = torch.load(config["ckpt_path"])[0] # recommended practice ckpt = torch.load(config['ckpt_path']) print('recorded best precision: {:0.4f} at step {}'.format( ckpt["precision"], ckpt["step"])) train_accuracy = ckpt["precision"] ## set base network net_config = config["network"] base_network = net_config["name"](**net_config["params"]) base_network.load_state_dict(ckpt['base_network']) centroids = None if 'center_criterion' in ckpt.keys(): centroids = ckpt['center_criterion']['centers'].cpu() target_centroids = None if 'target_center_criterion' in ckpt.keys(): target_centroids = ckpt['target_center_criterion']['centers'].cpu() use_gpu = torch.cuda.is_available() if use_gpu: base_network = base_network.cuda() ## test print("start test: ") base_network.train(False) if config["ly_type"] == 'cosine': test_acc, test_confusion_matrix = image_classification_test(dset_loaders, \ base_network, test_10crop=prep_config["test_10crop"], \ gpu=use_gpu) elif config["ly_type"] == "euclidean": eval_centroids = None if centroids is not None: eval_centroids = centroids if target_centroids is not None: eval_centroids = target_centroids test_acc, test_confusion_matrix = utils.distance_classification_test(dset_loaders, \ base_network, eval_centroids, test_10crop=prep_config["test_10crop"], \ gpu=use_gpu) # save train/test accuracy as pkl file with open(os.path.join(config["output_path"], 'accuracy.pkl'), 'wb') as pkl_file: pkl.dump({'train': train_accuracy, 'test': test_acc}, pkl_file) ## top k confusion classes for each class #base_data_path = os.path.split(data_config["target"]["list_path"])[0] #with open(os.path.join(base_data_path, 'label2class.json'), "r") as f: # obj = json.load(f)[0] # label_to_class = {int(k):v for k, v in obj.items()} #convert key to int np.set_printoptions(precision=2) log_str = "train precision: {:.5f}\ttest precision: {:.5f}\nconfusion matrix:\n{}\n".format( train_accuracy, test_acc, test_confusion_matrix) config["out_file"].write(log_str) config["out_file"].flush() print(log_str) return test_acc
def train(config): ## set up summary writer writer = SummaryWriter(config['output_path']) # set up early stop early_stop_engine = EarlyStopping(config["early_stop_patience"]) ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) prep_dict["target"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) else: prep_dict["test"] = prep.image_test( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) ## set loss class_num = config["network"]["params"]["class_num"] loss_params = config["loss"] class_criterion = nn.CrossEntropyLoss() transfer_criterion = loss.PADA center_criterion = loss_params["loss_type"](num_classes=class_num, feat_dim=config["network"]["params"]["bottleneck_dim"]) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] dsets["source"] = ImageList(stratify_sampling(open(data_config["source"]["list_path"]).readlines(), prep_config["source_size"]), \ transform=prep_dict["source"]) dset_loaders["source"] = util_data.DataLoader(dsets["source"], \ batch_size=data_config["source"]["batch_size"], \ shuffle=True, num_workers=1) dsets["target"] = ImageList(stratify_sampling(open(data_config["target"]["list_path"]).readlines(), prep_config["target_size"]), \ transform=prep_dict["target"]) dset_loaders["target"] = util_data.DataLoader(dsets["target"], \ batch_size=data_config["target"]["batch_size"], \ shuffle=True, num_workers=1) if prep_config["test_10crop"]: for i in range(10): dsets["test"+str(i)] = ImageList(stratify_sampling(open(data_config["test"]["list_path"]).readlines(), ratio=prep_config['target_size']), \ transform=prep_dict["test"]["val"+str(i)]) dset_loaders["test"+str(i)] = util_data.DataLoader(dsets["test"+str(i)], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=1) dsets["target"+str(i)] = ImageList(stratify_sampling(open(data_config["target"]["list_path"]).readlines(), ratio=prep_config['target_size']), \ transform=prep_dict["test"]["val"+str(i)]) dset_loaders["target"+str(i)] = util_data.DataLoader(dsets["target"+str(i)], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=1) else: dsets["test"] = ImageList(stratify_sampling(open(data_config["test"]["list_path"]).readlines(), ratio=prep_config['target_size']), \ transform=prep_dict["test"]) dset_loaders["test"] = util_data.DataLoader(dsets["test"], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=1) dsets["target_test"] = ImageList(stratify_sampling(open(data_config["target"]["list_path"]).readlines(), ratio=prep_config['target_size']), \ transform=prep_dict["test"]) dset_loaders["target_test"] = MyDataLoader(dsets["target_test"], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=1) config['out_file'].write("dataset sizes: source={}, target={}\n".format( len(dsets["source"]), len(dsets["target"]))) ## set base network net_config = config["network"] base_network = net_config["name"](**net_config["params"]) use_gpu = torch.cuda.is_available() if use_gpu: base_network = base_network.cuda() ## collect parameters if net_config["params"]["new_cls"]: if net_config["params"]["use_bottleneck"]: parameter_list = [{"params":base_network.feature_layers.parameters(), "lr_mult":1, 'decay_mult':2}, \ {"params":base_network.bottleneck.parameters(), "lr_mult":10, 'decay_mult':2}, \ {"params":base_network.fc.parameters(), "lr_mult":10, 'decay_mult':2}] else: parameter_list = [{"params":base_network.feature_layers.parameters(), "lr_mult":1, 'decay_mult':2}, \ {"params":base_network.fc.parameters(), "lr_mult":10, 'decay_mult':2}] else: parameter_list = [{"params":base_network.parameters(), "lr_mult":1, 'decay_mult':2}] ## add additional network for some methods ad_net = network.AdversarialNetwork(base_network.output_num()) gradient_reverse_layer = network.AdversarialLayer(high_value=config["high"]) #, #max_iter_value=config["num_iterations"]) if use_gpu: ad_net = ad_net.cuda() parameter_list.append({"params":ad_net.parameters(), "lr_mult":10, 'decay_mult':2}) parameter_list.append({"params":center_criterion.parameters(), "lr_mult": 10, 'decay_mult':1}) ## set optimizer optimizer_config = config["optimizer"] optimizer = optim_dict[optimizer_config["type"]](parameter_list, \ **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] ## train len_train_source = len(dset_loaders["source"]) - 1 len_train_target = len(dset_loaders["target"]) - 1 transfer_loss_value = classifier_loss_value = total_loss_value = 0.0 best_acc = 0.0 for i in range(config["num_iterations"]): if i % config["test_interval"] == 0: base_network.train(False) if config['loss']['ly_type'] == "cosine": temp_acc = image_classification_test(dset_loaders, \ base_network, test_10crop=prep_config["test_10crop"], \ gpu=use_gpu) elif config['loss']['ly_type'] == "euclidean": temp_acc, _ = distance_classification_test(dset_loaders, \ base_network, center_criterion.centers.detach(), test_10crop=prep_config["test_10crop"], \ gpu=use_gpu) else: raise ValueError("no test method for cls loss: {}".format(config['loss']['ly_type'])) snapshot_obj = {'step': i, "base_network": base_network.state_dict(), 'precision': temp_acc, } if config["loss"]["loss_name"] != "laplacian" and config["loss"]["ly_type"] == "euclidean": snapshot_obj['center_criterion'] = center_criterion.state_dict() if temp_acc > best_acc: best_acc = temp_acc # save best model torch.save(snapshot_obj, osp.join(config["output_path"], "best_model.pth.tar")) log_str = "iter: {:05d}, {} precision: {:.5f}\n".format(i, config['loss']['ly_type'], temp_acc) config["out_file"].write(log_str) config["out_file"].flush() writer.add_scalar("precision", temp_acc, i) if early_stop_engine.is_stop_training(temp_acc): config["out_file"].write("no improvement after {}, stop training at step {}\n".format( config["early_stop_patience"], i)) # config["out_file"].write("finish training! \n") break if (i+1) % config["snapshot_interval"] == 0: torch.save(snapshot_obj, osp.join(config["output_path"], "iter_{:05d}_model.pth.tar".format(i))) ## train one iter base_network.train(True) optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_source == 0: iter_source = iter(dset_loaders["source"]) if i % len_train_target == 0: iter_target = iter(dset_loaders["target"]) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() if use_gpu: inputs_source, inputs_target, labels_source = \ Variable(inputs_source).cuda(), Variable(inputs_target).cuda(), \ Variable(labels_source).cuda() else: inputs_source, inputs_target, labels_source = Variable(inputs_source), \ Variable(inputs_target), Variable(labels_source) inputs = torch.cat((inputs_source, inputs_target), dim=0) source_batch_size = inputs_source.size(0) if config['loss']['ly_type'] == 'cosine': features, logits = base_network(inputs) source_logits = logits.narrow(0, 0, source_batch_size) elif config['loss']['ly_type'] == 'euclidean': features, _ = base_network(inputs) logits = -1.0 * loss.distance_to_centroids(features, center_criterion.centers.detach()) source_logits = logits.narrow(0, 0, source_batch_size) ad_net.train(True) weight_ad = torch.ones(inputs.size(0)) transfer_loss = transfer_criterion(features, ad_net, gradient_reverse_layer, \ weight_ad, use_gpu) ad_out, _ = ad_net(features.detach()) ad_acc, source_acc_ad, target_acc_ad = domain_cls_accuracy(ad_out) # source domain classification task loss classifier_loss = class_criterion(source_logits, labels_source) # fisher loss on labeled source domain fisher_loss, fisher_intra_loss, fisher_inter_loss, center_grad = center_criterion(features.narrow(0, 0, int(inputs.size(0)/2)), labels_source, inter_class=config["loss"]["inter_type"], intra_loss_weight=loss_params["intra_loss_coef"], inter_loss_weight=loss_params["inter_loss_coef"]) # entropy minimization loss em_loss = loss.EntropyLoss(nn.Softmax(dim=1)(logits)) # final loss total_loss = loss_params["trade_off"] * transfer_loss \ + fisher_loss \ + loss_params["em_loss_coef"] * em_loss \ + classifier_loss total_loss.backward() if center_grad is not None: # clear mmc_loss center_criterion.centers.grad.zero_() # Manually assign centers gradients other than using autograd center_criterion.centers.backward(center_grad) optimizer.step() if i % config["log_iter"] == 0: config['out_file'].write('iter {}: total loss={:0.4f}, transfer loss={:0.4f}, cls loss={:0.4f}, ' 'em loss={:0.4f}, ' 'mmc loss={:0.4f}, intra loss={:0.4f}, inter loss={:0.4f}, ' 'ad acc={:0.4f}, source_acc={:0.4f}, target_acc={:0.4f}\n'.format( i, total_loss.data.cpu().float().item(), transfer_loss.data.cpu().float().item(), classifier_loss.data.cpu().float().item(), em_loss.data.cpu().float().item(), fisher_loss.cpu().float().item(), fisher_intra_loss.cpu().float().item(), fisher_inter_loss.cpu().float().item(), ad_acc, source_acc_ad, target_acc_ad, )) config['out_file'].flush() writer.add_scalar("total_loss", total_loss.data.cpu().float().item(), i) writer.add_scalar("cls_loss", classifier_loss.data.cpu().float().item(), i) writer.add_scalar("transfer_loss", transfer_loss.data.cpu().float().item(), i) writer.add_scalar("ad_acc", ad_acc, i) writer.add_scalar("d_loss/total", fisher_loss.data.cpu().float().item(), i) writer.add_scalar("d_loss/intra", fisher_intra_loss.data.cpu().float().item(), i) writer.add_scalar("d_loss/inter", fisher_inter_loss.data.cpu().float().item(), i) return best_acc
def train(config): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train(**config["prep"]['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop(**config["prep"]['params']) else: prep_dict["test"] = prep.image_test(**config["prep"]['params']) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] train_bs = data_config["source"]["batch_size"] test_bs = data_config["test"]["batch_size"] dsets["source"] = ImageList(open(data_config["source"]["list_path"]).readlines(), \ transform=prep_dict["source"]) dset_loaders["source"] = DataLoader(dsets["source"], batch_size=train_bs, \ shuffle=True, num_workers=4, drop_last=True) dsets["target"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["target"]) dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, \ shuffle=True, num_workers=4, drop_last=True) if prep_config["test_10crop"]: for i in range(10): dsets["test"] = [ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"][i]) for i in range(10)] dset_loaders["test"] = [DataLoader(dset, batch_size=test_bs, \ shuffle=False, num_workers=4) for dset in dsets['test']] else: dsets["test"] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["test"] = DataLoader(dsets["test"], batch_size=test_bs, \ shuffle=False, num_workers=4) class_num = config["network"]["params"]["class_num"] ## set base network net_config = config["network"] base_network = net_config["name"](**net_config["params"]) base_network = base_network.cuda() #加载基础网络结构 parameter_list = base_network.get_parameters() ## set optimizer optimizer_config = config["optimizer"] optimizer = optimizer_config["type"](parameter_list, \ **(optimizer_config["optim_params"])) crit = LabelSmoothingLoss(smoothing=0.1, classes=31) #标签平滑操作 param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] gpus = config['gpu'].split(',') if len(gpus) > 1: base_network = nn.DataParallel(base_network, device_ids=[int(i) for i in gpus]) ## train len_train_source = len(dset_loaders["source"]) len_train_target = len(dset_loaders["target"]) best_acc = 0.0 start_time = time.time() for i in range(config["num_iterations"]): if i % config["test_interval"] == config["test_interval"] - 1: # 在这里进行测试的工作 base_network.train(False) temp_acc = image_classification_test(dset_loaders, base_network, test_10crop=False) temp_model = nn.Sequential(base_network) if temp_acc > best_acc: best_acc = temp_acc best_model = temp_model log_str = "iter: {:05d}, precision: {:.5f}".format(i, temp_acc) config["out_file"].write(log_str + "\n") config["out_file"].flush() print(log_str) end_time = time.time() print('iter {} cost time {:.4f} sec.'.format( i, end_time - start_time)) # 打印时间间隔 start_time = time.time() if i % config["snapshot_interval"] == 0: torch.save(nn.Sequential(base_network), osp.join(config["output_path"], \ "iter_{:05d}_model.pth.tar".format(i))) ## train one iter base_network.train(True) # 训练模式 optimizer = lr_scheduler(optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_source == 0: iter_source = iter(dset_loaders["source"]) if i % len_train_target == 0: iter_target = iter(dset_loaders["target"]) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() inputs_source, inputs_target, labels_source = inputs_source.cuda( ), inputs_target.cuda(), labels_source.cuda() features_source, outputs_source = base_network(inputs_source) features_target, outputs_target = base_network(inputs_target) #目标域的熵正则化操作 t_logit = outputs_target t_prob = F.softmax(t_logit) t_entropy_loss = get_entropy_loss(t_prob) # 计算目标域的熵的损失 entropy_loss = config['ENT_w'] * (t_entropy_loss) # classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) # 源域的分类损失 classifier_loss = crit(outputs_source, labels_source) total_loss = classifier_loss + entropy_loss if i % config["test_interval"] == config["test_interval"] - 1: print('total loss: {:.4f}, classifier loss: {:.4f}'.format( total_loss.item(), classifier_loss.item())) total_loss.backward() optimizer.step() if (i + 1) % 1000 == 0: model_path = os.path.join( os.path.dirname(config['save_model_name']), 'temp_model.pth') save_model(base_network, model_path) model_path = config['save_model_name'] best_model_path = os.path.join(os.path.dirname(model_path), 'best_model.pth') save_model(base_network, model_path) save_model(best_model, best_model_path) torch.save(best_model, osp.join(config["output_path"], "best_model.pth.tar")) return best_acc
def train(config): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train(**config["prep"]['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop(**config["prep"]['params']) else: prep_dict["test"] = prep.image_test(**config["prep"]['params']) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] train_bs = data_config["source"]["batch_size"] test_bs = data_config["test"]["batch_size"] source_list = [ '.' + i for i in open(data_config["source"]["list_path"]).readlines() ] target_list = [ '.' + i for i in open(data_config["target"]["list_path"]).readlines() ] dsets["source"] = ImageList(source_list, \ transform=prep_dict["source"]) dset_loaders["source"] = DataLoader(dsets["source"], batch_size=train_bs, \ shuffle=True, num_workers=config['args'].num_worker, drop_last=True) dsets["target"] = ImageList(target_list, \ transform=prep_dict["target"]) dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, \ shuffle=True, num_workers=config['args'].num_worker, drop_last=True) print("source dataset len:", len(dsets["source"])) print("target dataset len:", len(dsets["target"])) if prep_config["test_10crop"]: for i in range(10): test_list = [ '.' + i for i in open(data_config["test"]["list_path"]).readlines() ] dsets["test"] = [ImageList(test_list, \ transform=prep_dict["test"][i]) for i in range(10)] dset_loaders["test"] = [DataLoader(dset, batch_size=test_bs, \ shuffle=False, num_workers=config['args'].num_worker) for dset in dsets['test']] else: test_list = [ '.' + i for i in open(data_config["test"]["list_path"]).readlines() ] dsets["test"] = ImageList(test_list, \ transform=prep_dict["test"]) dset_loaders["test"] = DataLoader(dsets["test"], batch_size=test_bs, \ shuffle=False, num_workers=config['args'].num_worker) dsets["target_label"] = ImageList_label(target_list, \ transform=prep_dict["target"]) dset_loaders["target_label"] = DataLoader(dsets["target_label"], batch_size=test_bs, \ shuffle=False, num_workers=config['args'].num_worker, drop_last=False) class_num = config["network"]["params"]["class_num"] ## set base network net_config = config["network"] base_network = net_config["name"](**net_config["params"]) base_network = base_network.to(network.dev) if config["restore_path"]: checkpoint = torch.load( osp.join(config["restore_path"], "best_model.pth"))["base_network"] ckp = {} for k, v in checkpoint.items(): if "module" in k: ckp[k.split("module.")[-1]] = v else: ckp[k] = v base_network.load_state_dict(ckp) log_str = "successfully restore from {}".format( osp.join(config["restore_path"], "best_model.pth")) config["out_file"].write(log_str + "\n") config["out_file"].flush() print(log_str) ## add additional network for some methods if "ALDA" in args.method: ad_net = network.Multi_AdversarialNetwork(base_network.output_num(), 1024, class_num) else: ad_net = network.AdversarialNetwork(base_network.output_num(), 1024) ad_net = ad_net.to(network.dev) parameter_list = base_network.get_parameters() + ad_net.get_parameters() ## set optimizer optimizer_config = config["optimizer"] optimizer = optimizer_config["type"](parameter_list, \ **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] gpus = config['gpu'].split(',') if len(gpus) > 1: ad_net = nn.DataParallel(ad_net, device_ids=[int(i) for i in range(len(gpus))]) base_network = nn.DataParallel( base_network, device_ids=[int(i) for i in range(len(gpus))]) loss_params = config["loss"] high = loss_params["trade_off"] begin_label = False writer = SummaryWriter(config["output_path"]) ## train len_train_source = len(dset_loaders["source"]) len_train_target = len(dset_loaders["target"]) transfer_loss_value = classifier_loss_value = total_loss_value = 0.0 best_acc = 0.0 loss_value = 0 loss_adv_value = 0 loss_correct_value = 0 for i in tqdm(range(config["num_iterations"]), total=config["num_iterations"]): if i % config["test_interval"] == config["test_interval"] - 1: base_network.train(False) temp_acc = image_classification_test(dset_loaders, \ base_network, test_10crop=prep_config["test_10crop"]) temp_model = base_network #nn.Sequential(base_network) if temp_acc > best_acc: best_step = i best_acc = temp_acc best_model = temp_model checkpoint = { "base_network": best_model.state_dict(), "ad_net": ad_net.state_dict() } torch.save(checkpoint, osp.join(config["output_path"], "best_model.pth")) print( "\n########## save the best model. #############\n") log_str = "iter: {:05d}, precision: {:.5f}".format(i, temp_acc) config["out_file"].write(log_str + "\n") config["out_file"].flush() writer.add_scalar('precision', temp_acc, i) print(log_str) print("adv_loss: {:.3f} correct_loss: {:.3f} class_loss: {:.3f}". format(loss_adv_value, loss_correct_value, loss_value)) loss_value = 0 loss_adv_value = 0 loss_correct_value = 0 #show val result on tensorboard images_inv = prep.inv_preprocess(inputs_source.clone().cpu(), 3) for index, img in enumerate(images_inv): writer.add_image(str(index) + '/Images', img, i) # save the pseudo_label if 'PseudoLabel' in config['method'] and ( i % config["label_interval"] == config["label_interval"] - 1): base_network.train(False) pseudo_label_list = image_label(dset_loaders, base_network, threshold=config['threshold'], \ out_dir=config["output_path"]) dsets["target"] = ImageList(open(pseudo_label_list).readlines(), \ transform=prep_dict["target"]) dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, \ shuffle=True, num_workers=config['args'].num_worker, drop_last=True) iter_target = iter( dset_loaders["target"] ) # replace the target dataloader with Pseudo_Label dataloader begin_label = True if i > config["stop_step"]: log_str = "method {}, iter: {:05d}, precision: {:.5f}".format( config["output_path"], best_step, best_acc) config["final_log"].write(log_str + "\n") config["final_log"].flush() break ## train one iter base_network.train(True) ad_net.train(True) optimizer = lr_scheduler(optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_source == 0: iter_source = iter(dset_loaders["source"]) if i % len_train_target == 0: iter_target = iter(dset_loaders["target"]) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() inputs_source, inputs_target, labels_source = Variable( inputs_source).to(network.dev), Variable(inputs_target).to( network.dev), Variable(labels_source).to(network.dev) features_source, outputs_source = base_network(inputs_source) if args.source_detach: features_source = features_source.detach() features_target, outputs_target = base_network(inputs_target) features = torch.cat((features_source, features_target), dim=0) outputs = torch.cat((outputs_source, outputs_target), dim=0) softmax_out = nn.Softmax(dim=1)(outputs) loss_params["trade_off"] = network.calc_coeff( i, high=high) #if i > 500 else 0.0 transfer_loss = 0.0 if 'DANN' in config['method']: transfer_loss = loss.DANN(features, ad_net) elif "ALDA" in config['method']: ad_out = ad_net(features) adv_loss, reg_loss, correct_loss = loss.ALDA_loss( ad_out, labels_source, softmax_out, weight_type=config['args'].weight_type, threshold=config['threshold']) # whether add the corrected self-training loss if "nocorrect" in config['args'].loss_type: transfer_loss = adv_loss else: transfer_loss = config['args'].adv_weight * adv_loss + config[ 'args'].adv_weight * loss_params["trade_off"] * correct_loss # reg_loss is only backward to the discriminator if "noreg" not in config['args'].loss_type: for param in base_network.parameters(): param.requires_grad = False reg_loss.backward(retain_graph=True) for param in base_network.parameters(): param.requires_grad = True # on-line self-training elif 'SelfTraining' in config['method']: transfer_loss += loss_params["trade_off"] * loss.SelfTraining_loss( outputs, softmax_out, config['threshold']) # off-line self-training elif 'PseudoLabel' in config['method']: labels_target = labels_target.to(network.dev) if begin_label: transfer_loss += loss_params["trade_off"] * nn.CrossEntropyLoss( ignore_index=-1)(outputs_target, labels_target) else: transfer_loss += 0.0 * nn.CrossEntropyLoss(ignore_index=-1)( outputs_target, labels_target) classifier_loss = nn.CrossEntropyLoss()(outputs_source, labels_source) loss_value += classifier_loss.item() / config["test_interval"] loss_adv_value += adv_loss.item() / config["test_interval"] loss_correct_value += correct_loss.item() / config["test_interval"] total_loss = classifier_loss + transfer_loss total_loss.backward() optimizer.step() checkpoint = { "base_network": temp_model.state_dict(), "ad_net": ad_net.state_dict() } torch.save(checkpoint, osp.join(config["output_path"], "final_model.pth")) return best_acc
def train_classification(config): ## set pre-process prep_train = prep.image_train(resize_size=256, crop_size=224) prep_test = prep.image_test(resize_size=256, crop_size=224) ## set loss class_criterion = nn.CrossEntropyLoss() ## prepare data TRAIN_LIST = 'data/I2AWA2.txt' #'AWA_SS.txt#'data/new_AwA2_common.txt' TEST_LIST = 'data/new_AwA2.txt' BSZ = args.batch_size dsets_train = ImageList(open(TRAIN_LIST).readlines(), shape=(args.img_size, args.img_size), transform=prep_train) loaders_train = util_data.DataLoader(dsets_train, batch_size=BSZ, shuffle=True, num_workers=8, pin_memory=True) dsets_test = ImageList(open(TEST_LIST).readlines(), shape=(args.img_size, args.img_size), transform=prep_test, train=False) loaders_test = util_data.DataLoader(dsets_test, batch_size=BSZ, shuffle=True, num_workers=4, pin_memory=True) begin_num = 127 class_num = 40 all_num = 50 ## set base network net_config = config["network"] base_network = network.network_dict[net_config["name"]]() base_network.load_state_dict( torch.load( 'GCN/materials/AWA2/base_net_pretrained_on_I2AwA2_source_only.pkl') ) classifier_layer1 = nn.Linear(base_network.output_num(), class_num) classifier_layer2 = nn.Linear(base_network.output_num(), all_num - class_num) for param in base_network.parameters(): param.requires_grad = False for param in base_network.layer4.parameters(): param.requires_grad = True for param in base_network.layer3.parameters(): param.requires_grad = True ## initialization weight_bias = torch.load('GCN/awa_50_cls_basic')['fc50'] weight_bias_127 = torch.load('GCN/materials/AWA2/151_cls_from_1K')['fc151'] classifier_layer1.weight.data = weight_bias[:class_num, :2048] classifier_layer2.weight.data = weight_bias[class_num:, :2048] classifier_layer1.bias.data = weight_bias[:class_num, -1] classifier_layer2.bias.data = weight_bias[class_num:, -1] ad_net = AdversarialNetwork(2048, 1024) graph = json.load(open('GCN/materials/AWA2/animals_graph_all.json', 'r')) word_vectors = torch.tensor(graph['vectors']) wnids = graph['wnids'] n = len(wnids) use_att = False if use_att: edges_set = graph['edges_set'] print('edges_set', [len(l) for l in edges_set]) lim = 4 for i in range(lim + 1, len(edges_set)): edges_set[lim].extend(edges_set[i]) edges_set = edges_set[:lim + 1] print('edges_set', [len(l) for l in edges_set]) edges = edges_set else: edges = graph['edges'] edges = edges + [(v, u) for (u, v) in edges] edges = edges + [(u, u) for u in range(n)] word_vectors = F.normalize(word_vectors).cuda() hidden_layers = 'd2048,d' gcn = GCN(n, edges, word_vectors.shape[1], 2049, hidden_layers) gcn.load_state_dict( torch.load('GCN/RESULTS_MODELS/awa-basic/epoch-3000.pth')) gcn.train() use_gpu = torch.cuda.is_available() if use_gpu: classifier_layer1 = classifier_layer1.cuda() classifier_layer2 = classifier_layer2.cuda() base_network = base_network.cuda() gcn = gcn.cuda() ad_net = ad_net.cuda() ## collect parameters parameter_list = [{ "params": classifier_layer2.parameters(), "lr": 2 }, { "params": classifier_layer1.parameters(), "lr": 5 }, { "params": ad_net.parameters(), "lr": 5 }, { "params": base_network.layer3.parameters(), "lr": 1 }, { "params": base_network.layer4.parameters(), "lr": 2 }, { "params": gcn.parameters(), "lr": 5 }] ## set optimizer optimizer_config = config["optimizer"] optimizer = optim_dict[optimizer_config["type"]]( parameter_list, **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] len_train_source = len(loaders_train) - 1 len_test_source = len(loaders_test) - 1 optimizer.zero_grad() for i in range(config["num_iterations"]): if ((i + 0) % config["test_interval"] == 0 and i > 100) or i == config["num_iterations"] - 1: base_network.layer3.train(False) base_network.layer4.train(False) classifier_layer1.train(False) classifier_layer2.train(False) print(str(i) + ' ACC:') iter_target = iter(loaders_test) print( image_classification_test(iter_target, len_test_source, base_network, classifier_layer1, classifier_layer2, gpu=use_gpu)) iter_target = iter(loaders_test) classifier_layer1.train(True) classifier_layer2.train(True) base_network.layer3.train(True) base_network.layer4.train(True) ad_net.train(True) optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) if i % len_train_source == 0: iter_source = iter(loaders_train) if i % (len_test_source) == 0: iter_target = iter(loaders_test) inputs_source, labels_source, labels_source_father, inputs_target = iter_source.next( ) if use_gpu: inputs_source, labels_source, inputs_target = Variable( inputs_source).cuda(), Variable( labels_source).cuda(), Variable(inputs_target).cuda() else: inputs_source, labels_source, inputs_target = Variable( inputs_source), Variable(labels_source), Variable( inputs_target) features_source = base_network(inputs_source) features_target = base_network(inputs_target) outputs_source1 = classifier_layer1(features_source) outputs_source2 = classifier_layer2(features_source) outputs_target1 = classifier_layer1(features_target) outputs_target2 = classifier_layer2(features_target) outputs_source = torch.cat((outputs_source1, outputs_source2), dim=1) outputs_target = torch.cat((outputs_target1, outputs_target2), dim=1) output_vectors = gcn(word_vectors) cls_loss = class_criterion(outputs_source, labels_source) outputs_softmax = F.softmax(outputs_target, dim=1) WEIGHT = torch.sum( torch.softmax(outputs_source, dim=1)[:, :40] * outputs_softmax[:, :40], 1) # - 0.2 max_s = torch.max(outputs_softmax[:, :40], 1)[0] # - 0.2 coeff = calc_coeff(i) entropy = Entropy(outputs_softmax) entropy.register_hook(grl_hook(coeff)) entropy = 1.0 + torch.exp(-entropy) entropy_s = Entropy(F.softmax(outputs_source, dim=1)) entropy_s.register_hook(grl_hook(coeff)) entropy_s = 1.0 + torch.exp(-entropy_s) Mask_ = (max_s.gt(0.4).float() * WEIGHT.gt(0.5).float()).byte() Mask = (max_s.gt(0.4).float() * WEIGHT.gt(0.5).float()).view( -1, 1).repeat(1, 2048).byte() source_matched = torch.masked_select(features_source, Mask).view(-1, 2048) target_matched = torch.masked_select(features_target, Mask).view(-1, 2048) transfer_loss = DANN( torch.cat((features_source, target_matched), dim=0), ad_net, BSZ, target_matched.size()[0], torch.cat((entropy_s, torch.masked_select(entropy, Mask_)), dim=0)) entropy_loss = torch.sum( torch.sum(outputs_softmax[:, class_num:], 1) * (1.0 - max_s.gt(0.5).float() * WEIGHT.gt(0.6).float()) ) / torch.sum(1.0 - max_s.gt(0.5).float() * WEIGHT.gt(0.6).float()) class1_weightbias = torch.cat( (classifier_layer1.weight, classifier_layer1.bias.view(-1, 1)), dim=1) class2_weightbias = torch.cat( (classifier_layer2.weight, classifier_layer2.bias.view(-1, 1)), dim=1) classifier_weight_bias = torch.cat( (class1_weightbias, class2_weightbias), dim=0) gcn_loss_1k = my_l2_loss( output_vectors[begin_num:(begin_num + class_num)], class1_weightbias) gcn_loss_4k = my_l2_loss( output_vectors[(begin_num + class_num):(begin_num + all_num)], class2_weightbias) gcn_loss_127 = my_l2_loss(output_vectors[:begin_num], weight_bias_127[:127].cuda()) total_loss = cls_loss + args.w_gcn * ( gcn_loss_1k + gcn_loss_4k + gcn_loss_127) + args.w_entropy * ( entropy_loss + args.w_data * args.w_data / entropy_loss) + transfer_loss * args.w_align print("Step " + str(i) + ": cls_loss: " + str(cls_loss.cpu().data.numpy()) + " entropy_loss: " + str(entropy_loss.cpu().data.numpy()) + " transfer_loss: " + str(transfer_loss.cpu().data.numpy()) + " gcn_1k_loss: " + str(gcn_loss_1k.cpu().data.numpy()) + " gcn_4k_loss: " + str(gcn_loss_4k.cpu().data.numpy()) + " gcn_127_loss: " + str(gcn_loss_4k.cpu().data.numpy())) if (i + 0) % config["save_num"] == 0: if not osp.exists(osp.join('save', args.save_name)): os.mkdir(osp.join('save', args.save_name)) torch.save( base_network.state_dict(), osp.join('save', args.save_name, 'base_net%d.pkl' % (i + 1))) torch.save( classifier_weight_bias, osp.join('save', args.save_name, 'class%d.pkl' % (i + 1))) torch.save( gcn.state_dict(), osp.join('save', args.save_name, 'gcn_net%d.pkl' % (i + 1))) total_loss.backward(retain_graph=True) if (i + 1) % config["opt_num"] == 0: optimizer.step() optimizer.zero_grad()
def train(config): ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train(**config["prep"]['params']) prep_dict["target"] = prep.image_train(**config["prep"]['params']) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop(**config["prep"]['params']) else: prep_dict["test"] = prep.image_test(**config["prep"]['params']) ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] train_bs = data_config["source"]["batch_size"] test_bs = data_config["test"]["batch_size"] print(config, data_config) # dsets["source"] = ImageList(open(data_config["source"]["list_path"]).readlines(), \ # transform=prep_dict["source"]) # dset_loaders["source"] = DataLoader(dsets["source"], batch_size=train_bs, \ # shuffle=True, num_workers=4, drop_last=True) # dsets["target"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ # transform=prep_dict["target"]) # dset_loaders["target"] = DataLoader(dsets["target"], batch_size=train_bs, \ # shuffle=True, num_workers=4, drop_last=True) # if prep_config["test_10crop"]: # for i in range(10): # dsets["test"] = [ImageList(open(data_config["test"]["list_path"]).readlines(), \ # transform=prep_dict["test"][i]) for i in range(10)] # dset_loaders["test"] = [DataLoader(dset, batch_size=test_bs, \ # shuffle=False, num_workers=4) for dset in dsets['test']] # else: dsets["test"] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["test"] = DataLoader(dsets["test"], batch_size=test_bs, \ shuffle=False, num_workers=4) print('load data finished') class_num = config["network"]["params"]["class_num"] ## set base network net_config = config["network"] #base_network = net_config["name"](**net_config["params"]) #base_network = base_network.cuda() #print(base_network) base_network = torch.load('snapshot/iter_90000_model.pth.tar') base_network = list(base_network.children())[0] base_network = base_network.module # for key, value in base_network.state_dict().items(): # print(key) base_network = list(base_network.children())[9].cuda() # base_network= torch.nn.Sequential(*list(base_network.children())[:-1]).cuda() base_network = nn.DataParallel(base_network) print(base_network) #base_network.load_state_dict(checkpoint, strict=False) # ## add additional network for some methods # if config["loss"]["random"]: # random_layer = network.RandomLayer([base_network.output_num(), class_num], config["loss"]["random_dim"]) # ad_net = network.AdversarialNetwork(config["loss"]["random_dim"], 1024) # else: # random_layer = None # ad_net = network.AdversarialNetwork(base_network.output_num() * class_num, 1024) # if config["loss"]["random"]: # random_layer.cuda() # ad_net = ad_net.cuda() # parameter_list = base_network.get_parameters()# + ad_net.get_parameters() # ## set optimizer # optimizer_config = config["optimizer"] # optimizer = optimizer_config["type"](parameter_list, \ # **(optimizer_config["optim_params"])) # param_lr = [] # for param_group in optimizer.param_groups: # param_lr.append(param_group["lr"]) # schedule_param = optimizer_config["lr_param"] # lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] # gpus = config['gpu'].split(',') # if len(gpus) > 1: # # ad_net = nn.DataParallel(ad_net, device_ids=[int(i) for i in gpus]) # base_network = nn.DataParallel(base_network, device_ids=[int(i) for i in gpus]) ## train # len_train_source = len(dset_loaders["source"]) # len_train_target = len(dset_loaders["target"]) # transfer_loss_value = classifier_loss_value = total_loss_value = 0.0 # best_acc = 0.0 base_network.train(False) image_classification_test(dset_loaders, \ base_network, test_10crop=prep_config["test_10crop"]) return
def train(config): tie = 1.0 ## set pre-process prep_dict = {} prep_config = config["prep"] prep_dict["source"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) prep_dict["target"] = prep.image_train( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) if prep_config["test_10crop"]: prep_dict["test"] = prep.image_test_10crop( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) else: prep_dict["test"] = prep.image_test( \ resize_size=prep_config["resize_size"], \ crop_size=prep_config["crop_size"]) ## set loss class_criterion = nn.CrossEntropyLoss() # 分类损失 transfer_criterion1 = loss.PADA # 迁移学习损失(带权重),BCEloss balance_criterion = loss.balance_loss() loss_params = config["loss"] ## prepare data dsets = {} dset_loaders = {} data_config = config["data"] dsets["source"] = ImageList(open(data_config["source"]["list_path"]).readlines(), \ transform=prep_dict["source"]) dset_loaders["source"] = util_data.DataLoader(dsets["source"], \ batch_size=data_config["source"]["batch_size"], \ shuffle=True, num_workers=4) dsets["target"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["target"]) dset_loaders["target"] = util_data.DataLoader(dsets["target"], \ batch_size=data_config["target"]["batch_size"], \ shuffle=True, num_workers=4) if prep_config["test_10crop"]: for i in range(10): dsets["test" + str(i)] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]["val" + str(i)]) dset_loaders["test" + str(i)] = util_data.DataLoader(dsets["test" + str(i)], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=4) dsets["target" + str(i)] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["test"]["val" + str(i)]) dset_loaders["target" + str(i)] = util_data.DataLoader(dsets["target" + str(i)], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=4) else: dsets["test"] = ImageList(open(data_config["test"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["test"] = util_data.DataLoader(dsets["test"], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=4) dsets["target_test"] = ImageList(open(data_config["target"]["list_path"]).readlines(), \ transform=prep_dict["test"]) dset_loaders["target_test"] = MyDataLoader(dsets["target_test"], \ batch_size=data_config["test"]["batch_size"], \ shuffle=False, num_workers=4) class_num = config["network"]["params"]["class_num"] ## set base network net_config = config["network"] base_network = net_config["name"](**net_config["params"]) use_gpu = torch.cuda.is_available() if use_gpu: base_network = base_network.cuda() ## collect parameters if net_config["params"]["new_cls"]: if net_config["params"]["use_bottleneck"]: parameter_list = [{"params": base_network.feature_layers.parameters(), "lr": 1}, \ {"params": base_network.bottleneck.parameters(), "lr": 10}, \ {"params": base_network.fc.parameters(), "lr": 10}] else: parameter_list = [{"params": base_network.feature_layers.parameters(), "lr": 1}, \ {"params": base_network.fc.parameters(), "lr": 10}] else: parameter_list = [{"params": base_network.parameters(), "lr": 1}] ## add additional network for some methods class_weight = torch.from_numpy(np.array( [1.0] * class_num)) # 权重初始化为class_num维向量,初始值为1 if use_gpu: class_weight = class_weight.cuda() ad_net = network.AdversarialNetwork( base_network.output_num()) # 鉴别器设置,输入为特征提取器的维数,输出为属于共域的可能性 nad_net = network.NAdversarialNetwork(base_network.output_num()) gradient_reverse_layer = network.AdversarialLayer( high_value=config["high"]) if use_gpu: ad_net = ad_net.cuda() nad_net = nad_net.cuda() parameter_list.append({"params": ad_net.parameters(), "lr": 10}) ## set optimizer optimizer_config = config["optimizer"] optimizer = optim_dict[optimizer_config["type"]](parameter_list, \ **(optimizer_config["optim_params"])) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group["lr"]) schedule_param = optimizer_config["lr_param"] lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]] ## train len_train_source = len(dset_loaders["source"]) - 1 len_train_target = len(dset_loaders["target"]) - 1 transfer_loss_value = classifier_loss_value = total_loss_value = attention_loss_value = 0.0 best_acc = 0.0 for i in range(config["num_iterations"]): if i % config["test_interval"] == 0: # 测试 base_network.train(False) # 先对特征提取器进行训练 temp_acc = image_classification_test(dset_loaders, \ base_network, test_10crop=prep_config["test_10crop"], \ gpu=use_gpu) # 对第一个有监督训练器进行分类训练,提取特征 temp_model = nn.Sequential(base_network) if temp_acc > best_acc: best_acc = temp_acc best_model = temp_model log_str = "iter: {:05d}, alpha: {:03f}, tradeoff: {:03f} ,precision: {:.5f}".format( i, loss_params["para_alpha"], loss_params["trade_off"], temp_acc) config["out_file"].write(log_str + '\n') config["out_file"].flush() print(log_str) if i % config["snapshot_interval"] == 0: # 输出 torch.save(nn.Sequential(base_network), osp.join(config["output_path"], \ "iter_{:05d}_model.pth.tar".format(i))) if i % loss_params["update_iter"] == loss_params["update_iter"] - 1: base_network.train(False) target_fc8_out = image_classification_predict( dset_loaders, base_network, softmax_param=config["softmax_param"] ) # 在对有监督学习进行了训练后,喂进去目标域数据,判断权重 class_weight = torch.mean( target_fc8_out, 0) # predict模型输出的预测向量取均值,为每一个体权重,将个体权重转化为类别权重 class_weight = (class_weight / torch.mean(class_weight)).cuda().view(-1) # 归一化 class_criterion = nn.CrossEntropyLoss( weight=class_weight) # 这里的交叉熵损失函数就是带权重的 ## train one iter base_network.train(True) optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param) optimizer.zero_grad() if i % len_train_source == 0: iter_source = iter(dset_loaders["source"]) if i % len_train_target == 0: iter_target = iter(dset_loaders["target"]) inputs_source, labels_source = iter_source.next() inputs_target, labels_target = iter_target.next() if use_gpu: inputs_source, inputs_target, labels_source = \ Variable(inputs_source).cuda(), Variable(inputs_target).cuda(), \ Variable(labels_source).cuda() else: inputs_source, inputs_target, labels_source = Variable(inputs_source), \ Variable(inputs_target), Variable(labels_source) inputs = torch.cat((inputs_source, inputs_target), dim=0) # 这里输入后一半目标域,前一半源域 features, outputs = base_network(inputs) # 从有分类网络获得特征与输出,到这里是特征提取器 softmax_out = nn.Softmax(dim=1)(outputs).detach() ad_net.train(True) nad_net.train(True) weight_ad = torch.zeros(inputs.size(0)) label_numpy = labels_source.data.cpu().numpy() for j in range(inputs.size(0) / 2): weight_ad[j] = class_weight[int(label_numpy[j])] # 计算实际样例权重 # print(label_numpy) weight_ad = weight_ad / torch.max( weight_ad[0:inputs.size(0) / 2]) # 权重归一化 for j in range(inputs.size(0) / 2, inputs.size(0)): # 前一半源域,所以权重是计算的,后一半目标域,权重全为1 weight_ad[j] = 1.0 classifier_loss = class_criterion( outputs.narrow(0, 0, inputs.size(0) / 2), labels_source) # 分类损失 total_loss = classifier_loss total_loss.backward() optimizer.step() torch.save(best_model, osp.join(config["output_path"], "best_model.pth.tar")) return best_acc
def main(config): ## set loss criterion use_gpu = torch.cuda.is_available() au_weight = torch.from_numpy(np.loadtxt(config.train_path_prefix + '_weight.txt')) if use_gpu: au_weight = au_weight.float().cuda() else: au_weight = au_weight.float() ## prepare data dsets = {} dset_loaders = {} dsets['train'] = ImageList(crop_size=config.crop_size, path=config.train_path_prefix, transform=prep.image_train(crop_size=config.crop_size), target_transform=prep.land_transform(img_size=config.crop_size, flip_reflect=np.loadtxt( config.flip_reflect))) dset_loaders['train'] = util_data.DataLoader(dsets['train'], batch_size=config.train_batch_size, shuffle=True, num_workers=config.num_workers) dsets['test'] = ImageList(crop_size=config.crop_size, path=config.test_path_prefix, phase='test', transform=prep.image_test(crop_size=config.crop_size), target_transform=prep.land_transform(img_size=config.crop_size, flip_reflect=np.loadtxt( config.flip_reflect)) ) dset_loaders['test'] = util_data.DataLoader(dsets['test'], batch_size=config.eval_batch_size, shuffle=False, num_workers=config.num_workers) ## set network modules region_learning = network.network_dict[config.region_learning](input_dim=3, unit_dim = config.unit_dim) align_net = network.network_dict[config.align_net](crop_size=config.crop_size, map_size=config.map_size, au_num=config.au_num, land_num=config.land_num, input_dim=config.unit_dim*8, fill_coeff=config.fill_coeff) local_attention_refine = network.network_dict[config.local_attention_refine](au_num=config.au_num, unit_dim=config.unit_dim) local_au_net = network.network_dict[config.local_au_net](au_num=config.au_num, input_dim=config.unit_dim*8, unit_dim=config.unit_dim) global_au_feat = network.network_dict[config.global_au_feat](input_dim=config.unit_dim*8, unit_dim=config.unit_dim) au_net = network.network_dict[config.au_net](au_num=config.au_num, input_dim = 12000, unit_dim = config.unit_dim) if config.start_epoch > 0: print('resuming model from epoch %d' %(config.start_epoch)) region_learning.load_state_dict(torch.load( config.write_path_prefix + config.run_name + '/region_learning_' + str(config.start_epoch) + '.pth')) align_net.load_state_dict(torch.load( config.write_path_prefix + config.run_name + '/align_net_' + str(config.start_epoch) + '.pth')) local_attention_refine.load_state_dict(torch.load( config.write_path_prefix + config.run_name + '/local_attention_refine_' + str(config.start_epoch) + '.pth')) local_au_net.load_state_dict(torch.load( config.write_path_prefix + config.run_name + '/local_au_net_' + str(config.start_epoch) + '.pth')) global_au_feat.load_state_dict(torch.load( config.write_path_prefix + config.run_name + '/global_au_feat_' + str(config.start_epoch) + '.pth')) au_net.load_state_dict(torch.load( config.write_path_prefix + config.run_name + '/au_net_' + str(config.start_epoch) + '.pth')) if use_gpu: region_learning = region_learning.cuda() align_net = align_net.cuda() local_attention_refine = local_attention_refine.cuda() local_au_net = local_au_net.cuda() global_au_feat = global_au_feat.cuda() au_net = au_net.cuda() print(region_learning) print(align_net) print(local_attention_refine) print(local_au_net) print(global_au_feat) print(au_net) ## collect parameters region_learning_parameter_list = [{'params': filter(lambda p: p.requires_grad, region_learning.parameters()), 'lr': 1}] align_net_parameter_list = [ {'params': filter(lambda p: p.requires_grad, align_net.parameters()), 'lr': 1}] local_attention_refine_parameter_list = [ {'params': filter(lambda p: p.requires_grad, local_attention_refine.parameters()), 'lr': 1}] local_au_net_parameter_list = [ {'params': filter(lambda p: p.requires_grad, local_au_net.parameters()), 'lr': 1}] global_au_feat_parameter_list = [ {'params': filter(lambda p: p.requires_grad, global_au_feat.parameters()), 'lr': 1}] au_net_parameter_list = [ {'params': filter(lambda p: p.requires_grad, au_net.parameters()), 'lr': 1}] ## set optimizer optimizer = optim_dict[config.optimizer_type](itertools.chain(region_learning_parameter_list, align_net_parameter_list, local_attention_refine_parameter_list, local_au_net_parameter_list, global_au_feat_parameter_list, au_net_parameter_list), lr=1.0, momentum=config.momentum, weight_decay=config.weight_decay, nesterov=config.use_nesterov) param_lr = [] for param_group in optimizer.param_groups: param_lr.append(param_group['lr']) lr_scheduler = lr_schedule.schedule_dict[config.lr_type] if not os.path.exists(config.write_path_prefix + config.run_name): os.makedirs(config.write_path_prefix + config.run_name) if not os.path.exists(config.write_res_prefix + config.run_name): os.makedirs(config.write_res_prefix + config.run_name) res_file = open( config.write_res_prefix + config.run_name + '/AU_pred_' + str(config.start_epoch) + '.txt', 'w') ## train count = 0 for epoch in range(config.start_epoch, config.n_epochs + 1): if epoch > config.start_epoch: print('taking snapshot ...') torch.save(region_learning.state_dict(), config.write_path_prefix + config.run_name + '/region_learning_' + str(epoch) + '.pth') torch.save(align_net.state_dict(), config.write_path_prefix + config.run_name + '/align_net_' + str(epoch) + '.pth') torch.save(local_attention_refine.state_dict(), config.write_path_prefix + config.run_name + '/local_attention_refine_' + str(epoch) + '.pth') torch.save(local_au_net.state_dict(), config.write_path_prefix + config.run_name + '/local_au_net_' + str(epoch) + '.pth') torch.save(global_au_feat.state_dict(), config.write_path_prefix + config.run_name + '/global_au_feat_' + str(epoch) + '.pth') torch.save(au_net.state_dict(), config.write_path_prefix + config.run_name + '/au_net_' + str(epoch) + '.pth') # eval in the train if epoch > config.start_epoch: print('testing ...') region_learning.train(False) align_net.train(False) local_attention_refine.train(False) local_au_net.train(False) global_au_feat.train(False) au_net.train(False) local_f1score_arr, local_acc_arr, f1score_arr, acc_arr, mean_error, failure_rate = AU_detection_evalv2( dset_loaders['test'], region_learning, align_net, local_attention_refine, local_au_net, global_au_feat, au_net, use_gpu=use_gpu) print('epoch =%d, local f1 score mean=%f, local accuracy mean=%f, ' 'f1 score mean=%f, accuracy mean=%f, mean error=%f, failure rate=%f' % (epoch, local_f1score_arr.mean(), local_acc_arr.mean(), f1score_arr.mean(), acc_arr.mean(), mean_error, failure_rate)) print('%d\t%f\t%f\t%f\t%f\t%f\t%f' % (epoch, local_f1score_arr.mean(), local_acc_arr.mean(), f1score_arr.mean(), acc_arr.mean(), mean_error, failure_rate), file=res_file) region_learning.train(True) align_net.train(True) local_attention_refine.train(True) local_au_net.train(True) global_au_feat.train(True) au_net.train(True) if epoch >= config.n_epochs: break for i, batch in enumerate(dset_loaders['train']): if i % config.display == 0 and count > 0: print('[epoch = %d][iter = %d][total_loss = %f][loss_au_softmax = %f][loss_au_dice = %f]' '[loss_local_au_softmax = %f][loss_local_au_dice = %f]' '[loss_land = %f]' % (epoch, i, total_loss.data.cpu().numpy(), loss_au_softmax.data.cpu().numpy(), loss_au_dice.data.cpu().numpy(), loss_local_au_softmax.data.cpu().numpy(), loss_local_au_dice.data.cpu().numpy(), loss_land.data.cpu().numpy())) print('learning rate = %f %f %f %f %f %f' % (optimizer.param_groups[0]['lr'], optimizer.param_groups[1]['lr'], optimizer.param_groups[2]['lr'], optimizer.param_groups[3]['lr'], optimizer.param_groups[4]['lr'], optimizer.param_groups[5]['lr'])) print('the number of training iterations is %d' % (count)) input, land, biocular, au = batch if use_gpu: input, land, biocular, au = input.cuda(), land.float().cuda(), \ biocular.float().cuda(), au.long().cuda() else: au = au.long() optimizer = lr_scheduler(param_lr, optimizer, epoch, config.gamma, config.stepsize, config.init_lr) optimizer.zero_grad() region_feat = region_learning(input) align_feat, align_output, aus_map = align_net(region_feat) if use_gpu: aus_map = aus_map.cuda() output_aus_map = local_attention_refine(aus_map.detach()) local_au_out_feat, local_aus_output = local_au_net(region_feat, output_aus_map) global_au_out_feat = global_au_feat(region_feat) concat_au_feat = torch.cat((align_feat, global_au_out_feat, local_au_out_feat.detach()), 1) aus_output = au_net(concat_au_feat) loss_au_softmax = au_softmax_loss(aus_output, au, weight=au_weight) loss_au_dice = au_dice_loss(aus_output, au, weight=au_weight) loss_au = loss_au_softmax + loss_au_dice loss_local_au_softmax = au_softmax_loss(local_aus_output, au, weight=au_weight) loss_local_au_dice = au_dice_loss(local_aus_output, au, weight=au_weight) loss_local_au = loss_local_au_softmax + loss_local_au_dice loss_land = landmark_loss(align_output, land, biocular) total_loss = config.lambda_au * (loss_au + loss_local_au) + \ config.lambda_land * loss_land total_loss.backward() optimizer.step() count = count + 1 res_file.close()
def cross_validation_loss(args, feature_network_path, predict_network_path, num_layer, src_list, target_path, val_list, class_num, resize_size, crop_size, batch_size, use_gpu): """ Main function for computing the CV loss :param feature_network: :param predict_network: :param src_cls_list: :param target_path: :param val_cls_list: :param class_num: :param resize_size: :param crop_size: :param batch_size: :return: """ option = 'resnet' + args.resnet G = ResBase(option) F1 = ResClassifier(num_layer=num_layer) G.load_state_dict(torch.load(feature_network_path)) F1.load_state_dict(torch.load(predict_network_path)) if use_gpu: G.cuda() F1.cuda() G.eval() F1.eval() val_list = seperate_data.dimension_rd(val_list) tar_list = open(target_path).readlines() cross_val_loss = 0 prep_dict = prep.image_train(resize_size=resize_size, crop_size=crop_size) # load different class's image dsets_src = ImageList(src_list, transform=prep_dict) dset_loaders_src = util_data.DataLoader(dsets_src, batch_size=batch_size, shuffle=True, num_workers=4) # prepare source feature iter_src = iter(dset_loaders_src) src_input, src_labels = iter_src.next() if use_gpu: src_input, src_labels = Variable(src_input).cuda(), Variable( src_labels).cuda() else: src_input, src_labels = Variable(src_input), Variable(src_labels) feature_val = G(src_input) src_feature_de = feature_val.detach().detach().cpu().numpy() for _ in range(len(dset_loaders_src) - 1): src_input, src_labels = iter_src.next() if use_gpu: src_input, src_labels = Variable(src_input).cuda(), Variable( src_labels).cuda() else: src_input, src_labels = Variable(src_input), Variable(src_labels) src_feature_new = G(src_input) src_feature_new_de = src_feature_new.detach().cpu().numpy() src_feature_de = np.append(src_feature_de, src_feature_new_de, axis=0) print("Created Source feature: {}".format(src_feature_de.shape)) # prepare target feature dsets_tar = ImageList(tar_list, transform=prep_dict) dset_loaders_tar = util_data.DataLoader(dsets_tar, batch_size=batch_size, shuffle=True, num_workers=4) iter_tar = iter(dset_loaders_tar) tar_input, _ = iter_tar.next() if use_gpu: tar_input, _ = Variable(tar_input).cuda(), Variable(_).cuda() else: src_input, _ = Variable(tar_input), Variable(_) tar_feature = G(tar_input) tar_feature_de = tar_feature.detach().cpu().numpy() for _ in range(len(dset_loaders_tar) - 1): tar_input, _ = iter_tar.next() if use_gpu: tar_input, _ = Variable(tar_input).cuda(), Variable(_).cuda() else: src_input, _ = Variable(tar_input), Variable(_) tar_feature_new = G(tar_input) tar_feature_new_de = tar_feature_new.detach().cpu().numpy() tar_feature_de = np.append(tar_feature_de, tar_feature_new_de, axis=0) print("Created Target feature: {}".format(tar_feature_de.shape)) # prepare validation feature dsets_val = ImageList(val_list, transform=prep_dict) dset_loaders_val = util_data.DataLoader(dsets_val, batch_size=batch_size, shuffle=True, num_workers=4) iter_val = iter(dset_loaders_val) val_input, val_labels = iter_val.next() if use_gpu: val_input, val_labels = Variable(val_input).cuda(), Variable( val_labels).cuda() else: val_input, val_labels = Variable(val_input), Variable(val_labels) val_feature = G(val_input) pred_label = F1(val_feature) val_feature_de = val_feature.detach().cpu().numpy() w = pred_label[0].shape[0] error = np.zeros(1) error[0] = predict_loss(val_labels[0].item(), pred_label[0].reshape(1, w)).item() error = error.reshape(1, 1) print("Before the final") print(pred_label.shape) for num_image in range(1, len(pred_label)): new_error = np.zeros(1) single_pred_label = pred_label[num_image] w = single_pred_label.shape[0] single_val_label = val_labels[num_image] new_error[0] = predict_loss(single_val_label.item(), single_pred_label.reshape(1, w)).item() new_error = new_error.reshape(1, 1) error = np.append(error, new_error, axis=0) for _ in range(len(dset_loaders_val) - 1): val_input, val_labels = iter_val.next() if use_gpu: val_input, val_labels = Variable(val_input).cuda(), Variable( val_labels).cuda() else: val_input, val_labels = Variable(val_input), Variable(val_labels) val_feature_new = G(val_input) val_feature_new_de = val_feature_new.detach().cpu().numpy() val_feature_de = np.append(val_feature_de, val_feature_new_de, axis=0) pred_label = F1(val_feature_new) for num_image in range(len(pred_label)): new_error = np.zeros(1) single_pred_label = pred_label[num_image] w = single_pred_label.shape[0] single_val_label = val_labels[num_image] new_error[0] = predict_loss(single_val_label.item(), single_pred_label.reshape(1, w)).item() new_error = new_error.reshape(1, 1) error = np.append(error, new_error, axis=0) print("Created Validation error shape: {}".format(error.shape)) print("Created Validation feature: {}".format(val_feature_de.shape)) if not os.path.exists(args.save.split("/")[0] + "/feature_np/"): os.makedirs(args.save.split("/")[0] + "/feature_np/") np.save( args.save.split("/")[0] + "/feature_np/" + "src_feature_de.npy", src_feature_de) np.save( args.save.split("/")[0] + "/feature_np/" + "tar_feature_de.npy", tar_feature_de) np.save( args.save.split("/")[0] + "/feature_np/" + "val_feature_de.npy", val_feature_de) src_feature_path = args.save.split( "/")[0] + "/feature_np/" + "_" + "src_feature_de.npy" tar_feature_path = args.save.split( "/")[0] + "/feature_np/" + "_" + "tar_feature_de.npy" val_feature_path = args.save.split( "/")[0] + "/feature_np/" + "_" + "val_feature_de.npy" weight = get_weight(src_feature_path, tar_feature_path, val_feature_path) cross_val_loss = cross_val_loss + get_dev_risk(weight, error) return cross_val_loss