def test_ten(): print("Testing...") # init dataset _, metatest_folders = tg.mini_imagenet_folders() # init network feature_encoder = CNNEncoder() relation_network = RelationNetwork(Size, Hidden_dim, Relation_dim) feature_encoder = feature_encoder.cuda() relation_network = relation_network.cuda() if not os.path.exists(str("./models/" + args.exp)): feature_encoder.apply(weights_init) relation_network.apply(weights_init) print("Testing with random initialization ...") if os.path.exists( str("./models/" + args.exp + "/miniimagenet_feature_encoder_" + str(Val_way) + "way_" + str(Val_shot) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/" + args.exp + "/miniimagenet_feature_encoder_" + str(Val_way) + "way_" + str(Val_shot) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/" + args.exp + "/miniimagenet_relation_network_" + str(Val_way) + "way_" + str(Val_shot) + "shot.pkl")): relation_network.load_state_dict( torch.load( str("./models/" + args.exp + "/miniimagenet_relation_network_" + str(Val_way) + "way_" + str(Val_shot) + "shot.pkl"))) print("load relation network success") accuracy = [] for _ in range(10): accuracy.append( test_one(metatest_folders, feature_encoder, relation_network)) print('accuracies', accuracy) print('mean', np.mean(accuracy), 'std', np.std(accuracy), 'max', np.max(accuracy), 'min', np.min(accuracy))
def main(): # Step 1: init data folders print("init data folders") metatrain_folders,metatest_folders = tg.mini_imagenet_folders() # init character folders for dataset construction # Step 2: init neural networks print("load neural networks") if os.path.exists(str("models/miniimagenet_feature_encoder_" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot")): feature_encoder = tf.keras.models.load_model(str("models/miniimagenet_feature_encoder_" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot")) print("load feature encoder success") if os.path.exists(str("models/miniimagenet_relation_network_"+ str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot")): relation_network = tf.keras.models.load_model(str("models/miniimagenet_relation_network_" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot")) print("load relation network success") # Step 3: build graph total_accuracy = 0.0 for episode in range(EPISODE): print("Testing...") accuracies = [] for i in range(TEST_EPISODE): task = tg.MiniImagenetTask(metatest_folders,CLASS_NUM,SAMPLE_NUM_PER_CLASS,BATCH_NUM_PER_CLASS) sample_dataset = tg.dataset(task,SAMPLE_NUM_PER_CLASS,split='train',shuffle=False) batch_dataset = tg.dataset(task,3,split='test',shuffle=True) sample_dataloader = tf.data.Dataset.from_generator(sample_dataset.generator, output_types=(tf.float32, tf.float32), output_shapes = ((84,84,3),(5,1))).batch(SAMPLE_NUM_PER_CLASS*CLASS_NUM).take(1) batch_dataloader = tf.data.Dataset.from_generator(batch_dataset.generator, output_types=(tf.float32, tf.float32), output_shapes = ((84,84,3),(5,1))).batch(3*CLASS_NUM).take(1) samples,sample_labels = next(iter(sample_dataloader)) batches,batch_labels = next(iter(batch_dataloader)) accuracies.append(test(feature_encoder, relation_network, samples, sample_labels, batches, batch_labels)) test_accuracy,h = mean_confidence_interval(accuracies) print("test accuracy:",test_accuracy,"h:",h) total_accuracy += test_accuracy print("aver_accuracy:",total_accuracy/EPISODE)
def main(): # Step 1: init data folders print("init data folders") # init character folders for dataset construction metatrain_folders, metatest_folders = tg.mini_imagenet_folders() # Step 2: init neural networks print("init neural networks") feature_encoder = CNNEncoder() relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM) feature_encoder = nn.DataParallel(feature_encoder) relation_network = nn.DataParallel(relation_network) feature_encoder.apply(weights_init) #??????????? relation_network.apply(weights_init) feature_encoder.cuda(GPU) relation_network.cuda(GPU) feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=100000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=100000, gamma=0.5) if os.path.exists( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") # Step 3: build graph print("Training...") last_accuracy = 0.0 ######################################################################################## year = datetime.datetime.now().year month = datetime.datetime.now().month day = datetime.datetime.now().day filename = "miniimagenet_train_fewshot_" + str(year) + '_' + str( month) + '_' + str(day) + ".txt" with open("models/" + filename, "w") as f: for episode in range(EPISODE): feature_encoder_scheduler.step(episode) relation_network_scheduler.step(episode) # init dataset # sample_dataloader is to obtain previous samples for compare # batch_dataloader is to batch samples for training task = tg.MiniImagenetTask(metatrain_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=BATCH_NUM_PER_CLASS, split="test", shuffle=True) # sample datas samples, sample_labels = sample_dataloader.__iter__().next( ) #25*3*84*84 batches, batch_labels = batch_dataloader.__iter__().next() # calculate features sample_features = feature_encoder( Variable(samples).cuda(GPU)) # 25*64*19*19 sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) batch_features = feature_encoder( Variable(batches).cuda(GPU)) # 20x64*5*5 # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) batch_features_ext = batch_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) batch_features_ext = torch.transpose(batch_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view(-1, CLASS_NUM) mse = nn.MSELoss().cuda(GPU) one_hot_labels = Variable( torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).scatter_(1, batch_labels.view(-1, 1), 1).cuda(GPU)) loss = mse(relations, one_hot_labels) # training feature_encoder.zero_grad() relation_network.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(feature_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm(relation_network.parameters(), 0.5) feature_encoder_optim.step() relation_network_optim.step() if (episode + 1) % 100 == 0: print("episode:", episode + 1, "loss", loss.item()) newcontext = "episode: " + str(episode + 1) + " loss " + str( loss.item()) + '\n' f.writelines(newcontext) if episode % 5000 == 0: # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, 15) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 5 test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=False) sample_images, sample_labels = sample_dataloader.__iter__( ).next() for test_images, test_labels in test_dataloader: batch_size = test_labels.shape[0] # calculate features sample_features = feature_encoder( Variable(sample_images).cuda(GPU)) # 5x64 sample_features = sample_features.view( CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) test_features = feature_encoder( Variable(test_images).cuda(GPU)) # 20x64 # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze( 0).repeat(batch_size, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose( test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j].cuda() == test_labels[j].cuda() else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) accuracy = total_rewards / 1.0 / CLASS_NUM / 15 accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:", test_accuracy, "h:", h) newcontext = "episode: " + str( episode + 1) + "test accuracy: " + str(test_accuracy) + '\n' f.writelines(newcontext) if test_accuracy > last_accuracy: # save networks torch.save( feature_encoder.state_dict(), str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")) torch.save( relation_network.state_dict(), str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")) print("save networks for episode:", episode) last_accuracy = test_accuracy
def main(): # Step 1: init data folders print("init data folders") # init character folders for dataset construction metatrain_folders, metatest_folders = tg.mini_imagenet_folders() # Step 2: init neural networks print("init neural networks") feature_encoder = CNNEncoder() relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM) feature_encoder.cuda(GPU) relation_network.cuda(GPU) if os.path.exists( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") # Step 3: build graph total_accuracy = 0.0 for episode in range(EPISODE): # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, 15) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 5 test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=False) sample_images, sample_labels = sample_dataloader.__iter__().next() for test_images, test_labels in test_dataloader: batch_size = test_labels.shape[0] # calculate features sample_features = feature_encoder( Variable(sample_images).cuda(GPU)) # 5x64 sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1) / 5.0 sample_features = sample_features.squeeze(1) test_features = feature_encoder( Variable(test_images).cuda(GPU)) # 20x64 # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose(test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) predict_labels = predict_labels.cuda(GPU) test_labels = test_labels.cuda(GPU) rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) accuracy = total_rewards / 1.0 / CLASS_NUM / 15 accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:", test_accuracy, "h:", h) total_accuracy += test_accuracy print("aver_accuracy:", total_accuracy / EPISODE)
def main(): print("init data folders") metatrain_folders, metaquery_folders = tg.mini_imagenet_folders() print("init neural networks") foreground_encoder = models.FeatureEncoder().apply(weights_init).cuda(GPU) background_encoder = models.FeatureEncoder().apply(weights_init).cuda(GPU) mixture_network = models.MixtureNetwork().apply(weights_init).cuda(GPU) relation_network = models.SimilarityNetwork( FEATURE_DIM, RELATION_DIM).apply(weights_init).cuda(GPU) # Loading models if os.path.exists( str(METHOD + "/miniImagenet_foreground_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): foreground_encoder.load_state_dict( torch.load( str(METHOD + "/miniImagenet_foreground_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load foreground encoder success") if os.path.exists( str(METHOD + "/miniImagenet_background_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): background_encoder.load_state_dict( torch.load( str(METHOD + "/miniImagenet_background_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load background encoder success") if os.path.exists( str(METHOD + "/miniImagenet_mixture_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): mixture_network.load_state_dict( torch.load( str(METHOD + "/miniImagenet_mixture_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load mixture network success") if os.path.exists( str(METHOD + "/miniImagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str(METHOD + "/miniImagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") best_accuracy = 0.0 best_h = 0.0 for episode in range(EPISODE): with torch.no_grad(): # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 counter = 0 task = tg.MiniImagenetTask(metaquery_folders, CLASS_NUM, SUPPORT_NUM_PER_CLASS, 15) support_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SUPPORT_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 2 query_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=True) support_img, support_sal, support_labels = support_dataloader.__iter__( ).next() for query_img, query_sal, query_labels in query_dataloader: query_size = query_labels.shape[0] # calculate foreground and background features support_foreground_features = foreground_encoder( Variable(support_img * support_sal).cuda(GPU)).view( CLASS_NUM, SUPPORT_NUM_PER_CLASS, 64, 19, 19) support_background_features = background_encoder( Variable( support_img * (1 - support_sal)).cuda(GPU)).view( CLASS_NUM, SUPPORT_NUM_PER_CLASS, 64, 19, 19) query_foreground_features = foreground_encoder( Variable(query_img * query_sal).cuda(GPU)) query_background_features = background_encoder( Variable(query_img * (1 - query_sal)).cuda(GPU)) # Inter-class Hallucination support_foreground_features = support_foreground_features.unsqueeze( 2).repeat(1, 1, CLASS_NUM * SUPPORT_NUM_PER_CLASS, 1, 1, 1) support_background_features = support_background_features.view( 1, 1, CLASS_NUM * SUPPORT_NUM_PER_CLASS, 64, 19, 19).repeat(CLASS_NUM, SUPPORT_NUM_PER_CLASS, 1, 1, 1, 1) similarity_measure = similarity_func( support_background_features, CLASS_NUM, SUPPORT_NUM_PER_CLASS).view(CLASS_NUM, SUPPORT_NUM_PER_CLASS, -1, 1, 1) support_mix_features = mixture_network( (support_foreground_features + support_background_features).view( (CLASS_NUM * SUPPORT_NUM_PER_CLASS)**2, 64, 19, 19)).view(CLASS_NUM, SUPPORT_NUM_PER_CLASS, -1, 64, 19**2) support_mix_features = (support_mix_features * similarity_measure).sum(2).sum(1) query_mix_features = mixture_network( query_foreground_features + query_background_features).view(-1, 64, 19**2) so_support_features = Variable( torch.Tensor(CLASS_NUM, 1, 64, 64)).cuda(GPU) so_query_features = Variable( torch.Tensor(query_size, 1, 64, 64)).cuda(GPU) # second-order features for d in range(support_mix_features.size()[0]): s = support_mix_features[d, :, :].squeeze(0) s = (1.0 / support_mix_features.size()[2]) * s.mm( s.transpose(0, 1)) so_support_features[d, :, :, :] = power_norm( s / s.trace(), SIGMA) for d in range(query_mix_features.size()[0]): s = query_mix_features[d, :, :].squeeze(0) s = (1.0 / query_mix_features.size()[2]) * s.mm( s.transpose(0, 1)) so_query_features[d, :, :, :] = power_norm( s / s.trace(), SIGMA) # calculate relations with 64x64 second-order features support_features_ext = so_support_features.unsqueeze( 0).repeat(query_size, 1, 1, 1, 1) query_features_ext = so_query_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) query_features_ext = torch.transpose( query_features_ext, 0, 1) relation_pairs = torch.cat( (support_features_ext, query_features_ext), 2).view(-1, 2, 64, 64) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == query_labels[j].cuda(GPU) else 0 for j in range(query_size) ] total_rewards += np.sum(rewards) counter += query_size accuracy = total_rewards / 1.0 / counter accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:", test_accuracy, "h:", h) if test_accuracy > best_accuracy: best_accuracy = test_accuracy best_h = h print("best accuracy:", best_accuracy, "h:", best_h)
def main(): # Step 1: init data folders print("init data folders") # init character folders for dataset construction metatrain_folders, metatest_folders = tg.mini_imagenet_folders( trainval=False) # Step 2: init neural networks print("init neural networks") feature_encoder = CNNEncoder() relation_network = RelationNetwork(Size, Hidden_dim, Relation_dim) feature_encoder.apply(weights_init) relation_network.apply(weights_init) feature_encoder = feature_encoder.cuda() relation_network = relation_network.cuda() # feature_encoder.cuda() # relation_network.cuda() feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=Learning_rate) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=20000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=Learning_rate) relation_network_scheduler = StepLR(relation_network_optim, step_size=20000, gamma=0.5) # Step 3: build graph print("Training...") last_accuracy = 0.0 train_counter = 0 train_total_rewards = 0 for episode in range(Episode): feature_encoder.train() relation_network.train() feature_encoder_scheduler.step(episode) relation_network_scheduler.step(episode) # init train dataset task = tg.MiniImagenetTask(metatrain_folders, Train_way, Train_shot, Train_query) support_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=Train_shot, split="train", shuffle=False, train=True) query_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=Train_query, split="test", shuffle=True, train=True) # sample datas samples, sample_labels = support_dataloader.__iter__().next() batches, batch_labels = query_dataloader.__iter__().next() if Train_shot > 1: sample_labels = sample_labels.view(Train_way, Train_shot) sample_labels = torch.mean(sample_labels.float(), 1).long() for _ in range(1): # calculate support features and query features support_features = feature_encoder(Variable(samples).cuda()) query_features = feature_encoder(Variable(batches).cuda()) _, Channel, Height, Width = support_features.size() if Train_shot > 1: support_features = support_features.view( Train_way, Train_shot, Channel, Height, Width) support_features = torch.mean(support_features, 1).squeeze(1) # calculate features support_features = support_features.view(support_features.size(0), Channel, Height * Width).transpose( 1, 2) query_features = query_features.view(query_features.size(0), Channel, Height * Width).transpose( 1, 2) # calculate mean support_mean = support_features.mean(2, keepdim=True) query_mean = query_features.mean(2, keepdim=True) # centered features support_centered = support_features - support_mean query_centered = query_features - query_mean relation_matrix = (1.0 / (Height * Width - 1)) * torch.matmul( query_centered.unsqueeze(1), support_centered.transpose( 1, 2)).view(query_centered.size(0) * Train_way, -1) relations = relation_network(relation_matrix).view( query_features.size(0), Train_way) cre = nn.CrossEntropyLoss().cuda() loss = cre(relations, Variable(batch_labels.cuda())) # training feature_encoder.zero_grad() relation_network.zero_grad() loss.backward() feature_encoder_optim.step() relation_network_optim.step() _, train_predict_labels = torch.max(relations.data, 1) rewards = [ 1 if train_predict_labels.cpu()[j] == batch_labels[j] else 0 for j in range(query_features.size(0)) ] train_total_rewards += np.sum(rewards) train_counter += query_features.size(0) if (episode + 1) % 100 == 0: train_accuracy = train_total_rewards / 1.0 / train_counter train_total_rewards = 0 train_counter = 0 print("episode:", episode + 1, "loss", loss.data[0], 'accuracy aver 100 episode', train_accuracy) if episode % 500 == 0: # test print("Testing...") accuracies = [] for i in range(Test_episode): feature_encoder.eval() relation_network.eval() total_rewards = 0 counter = 0 task = tg.MiniImagenetTask(metatest_folders, Val_way, Val_shot, Val_query) support_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=Val_shot, split="train", shuffle=False, train=False) test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=Val_query, split="test", shuffle=True, train=False) sample_images, sample_labels = support_dataloader.__iter__( ).next() # calculate support features support_features = feature_encoder( Variable(sample_images).cuda()) _, Channel, Height, Width = support_features.size() if Val_shot > 1: support_features = support_features.view( Val_way, Val_shot, Channel, Height, Width) support_features = torch.mean(support_features, 1).squeeze(1) # calculate centered support features support_features = support_features.view( support_features.size(0), Channel, Height * Width).transpose(1, 2) support_mean = support_features.mean(2, keepdim=True) support_centered = support_features - support_mean for test_images, test_labels in test_dataloader: batch_size = test_labels.shape[0] # calculate centered test features test_features = feature_encoder( Variable(test_images).cuda()) test_features = test_features.view(test_features.size(0), Channel, Height * Width).transpose(1, 2) test_mean = test_features.mean(2, keepdim=True) test_centered = test_features - test_mean # calculate relation matrix relation_matrix = ( 1.0 / (Height * Width - 1)) * torch.matmul( test_centered.unsqueeze(1), support_centered.transpose(1, 2)).view( test_centered.size(0) * Val_way, -1) relations = relation_network(relation_matrix).view( batch_size, Val_way) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels.cpu()[j] == test_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) counter += batch_size accuracy = total_rewards / 1.0 / counter accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:", test_accuracy, "h:", h) if test_accuracy > last_accuracy: # create exp directory if not os.path.exists(str("./models/" + args.exp)): os.makedirs(str("./models/" + args.exp)) # save networks torch.save( feature_encoder.state_dict(), str("./models/" + args.exp + "/miniimagenet_feature_encoder_" + str(Val_way) + "way_" + str(Val_shot) + "shot.pkl")) torch.save( relation_network.state_dict(), str("./models/" + args.exp + "/miniimagenet_relation_network_" + str(Val_way) + "way_" + str(Val_shot) + "shot.pkl")) print("save networks for episode:", episode) last_accuracy = test_accuracy
def main(): # Step 0: Experiment info date = datetime.datetime.today() today = date.date() now = date.time() exp_date = '{}{:02d}{:02d}-{:02d}{:02d}'.format(today.year % 100, today.month, today.day, now.hour, now.minute) print('\n{} {} {}-way {}-shot Mini ImageNet training\n'.format( today, now, CLASS_NUM, SAMPLE_NUM_PER_CLASS)) # Step 1: init data folders print("init data folders") # init character folders for dataset construction metatrain_folders, metatest_folders = tg.mini_imagenet_folders() # Step 2: init neural networks print("init neural networks") feature_encoder = CNNEncoder() relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM) feature_encoder.apply(weights_init) relation_network.apply(weights_init) print("Construct model success") feature_encoder.cuda(GPU) relation_network.cuda(GPU) feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=100000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=100000, gamma=0.5) # if os.path.exists(str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str( # SAMPLE_NUM_PER_CLASS) + "shot.pkl")): # feature_encoder.load_state_dict(torch.load(str( # "./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str( # SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) # print("load feature encoder success") # if os.path.exists(str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str( # SAMPLE_NUM_PER_CLASS) + "shot.pkl")): # relation_network.load_state_dict(torch.load(str( # "./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str( # SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) # print("load relation network success") # Step 3: build graph print("Training...") last_accuracy = 0.0 last_h = 0.0 for episode in range(EPISODE): feature_encoder_scheduler.step(episode) relation_network_scheduler.step(episode) # init dataset # sample_dataloader is to obtain previous samples for compare # batch_dataloader is to batch samples for training task = tg.MiniImagenetTask(metatrain_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=BATCH_NUM_PER_CLASS, split="test", shuffle=True) # sample datas samples, sample_labels = sample_dataloader.__iter__().next( ) # 25*3*84*84 (support set Way*Shot)*3*H*W batches, batch_labels = batch_dataloader.__iter__().next() # calculate features sample_features = feature_encoder( Variable(samples).cuda(GPU)) # 25*64*19*19 sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) batch_features = feature_encoder( Variable(batches).cuda(GPU)) # 20x64*5*5 # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) batch_features_ext = batch_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) batch_features_ext = torch.transpose(batch_features_ext, 0, 1) relation_pairs = torch.cat((sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view(-1, CLASS_NUM) mse = nn.MSELoss().cuda(GPU) one_hot_labels = Variable( torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).scatter_(1, batch_labels.view(-1, 1), 1).cuda(GPU)) loss = mse(relations, one_hot_labels) # training feature_encoder.zero_grad() relation_network.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(feature_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm_(relation_network.parameters(), 0.5) feature_encoder_optim.step() relation_network_optim.step() if (episode + 1) % 100 == 0: print("episode: {}\tloss:{:.6f}".format(episode + 1, loss.item())) if episode % 5000 == 0: # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, 15) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 5 test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=False) sample_images, sample_labels = sample_dataloader.__iter__( ).next() for test_images, test_labels in test_dataloader: batch_size = test_labels.shape[0] # calculate features sample_features = feature_encoder( Variable(sample_images).cuda(GPU)) # 5x64 sample_features = sample_features.view( CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) test_features = feature_encoder( Variable(test_images).cuda(GPU)) # 20x64 # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose( test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) test_labels = test_labels.cuda(GPU) rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) accuracy = total_rewards / 1.0 / CLASS_NUM / 15 accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:{:.3f}\th:{:.5f}\thighest: {:.3f}".format( test_accuracy, h, last_accuracy)) if test_accuracy > last_accuracy: # save networks torch.save( feature_encoder.state_dict(), "./models/{}_miniimagenet_feature_encoder_{}way_{}shot.pkl" .format(exp_date, CLASS_NUM, SAMPLE_NUM_PER_CLASS)) torch.save( relation_network.state_dict(), "./models/{}_miniimagenet_relation_network_{}way_{}shot.pkl" .format(exp_date, CLASS_NUM, SAMPLE_NUM_PER_CLASS)) print("save networks for episode:", episode) last_accuracy = test_accuracy last_h = h print("Training end\nhighest test accuracy: {:.3f}\th:{:.3f}".format( last_accuracy, last_h))
def main(): # Step 1: init data folders print("init data folders") # init character folders for dataset construction metatrain_folders, metatest_folders = tg.mini_imagenet_folders() # Step 2: init neural networks print("init neural networks") feature_encoder = CNNEncoder() relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM) feature_encoder.cuda(GPU) relation_network.cuda(GPU) feature_encoder_fn = os.path.join( './models', "{}_miniimagenet_feature_encoder_{}way_{}shot.pkl".format( EXP_DATE, CLASS_NUM, SAMPLE_NUM_PER_CLASS)) relation_network_fn = os.path.join( './models', "{}_miniimagenet_relation_network_{}way_{}shot.pkl".format( EXP_DATE, CLASS_NUM, SAMPLE_NUM_PER_CLASS)) if os.path.exists(feature_encoder_fn): feature_encoder.load_state_dict(torch.load(feature_encoder_fn)) print("load feature encoder success") if os.path.exists(relation_network_fn): relation_network.load_state_dict(torch.load(relation_network_fn)) print("load relation network success") total_accuracy = 0.0 for episode in range(EPISODE): # test print("Episode-{}\tTesting...".format(episode), end='\t') accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, 15) # test_dir, 5, 5, 15(test_num)) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 5 test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=False) sample_images, sample_labels = sample_dataloader.__iter__().next() for test_images, test_labels in test_dataloader: batch_size = test_labels.shape[0] print(test_labels) # print(test_images.size()) # calculate features sample_features = feature_encoder( Variable(sample_images).cuda(GPU)) # [25, 64, 19, 19] # print('sample feature(1): ', sample_features.size()) sample_features = sample_features.view( CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) # [5, 5, 64, 19, 19] # print('sample feature(2): ', sample_features.size()) sample_features = torch.sum(sample_features, 1).squeeze(1) # [5, 64, 19, 19] # print('sample feature(3): ', sample_features.size()) test_features = feature_encoder( Variable(test_images).cuda(GPU)) # [25, 64, 19, 19] # print('test_features(1): ', test_features.size()) # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) # [25, 5, 64, 19, 19] # print('sample feature(4): ', sample_features_ext.size()) test_features_ext = test_features.unsqueeze( 0) # [1, 25, 64, 19, 19] # print('test_features(2): ', test_features_ext.size()) test_features_ext = test_features_ext.repeat( 1 * CLASS_NUM, 1, 1, 1, 1) # [5, 25, 64, 19, 19] # print('test_features(3): ', test_features_ext.size()) test_features_ext = torch.transpose(test_features_ext, 0, 1) # [25, 5, 64, 19, 19] # print('test_features(4): ', test_features_ext.size()) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2) # [25, 5, 128, 19, 19] # print('relation_pairs(1): ', relation_pairs.size()) relation_pairs = relation_pairs.view(-1, FEATURE_DIM * 2, 19, 19) # [125, 128, 19, 19] # print('relation_pairs(2): ', relation_pairs.size()) relations = relation_network(relation_pairs) # [125, 1] # print('relations(1): ', relations.size()) relations = relations.view(-1, CLASS_NUM) # [25, 5] # print('relations(2): ', relations.size()) print(relations.data) rel_score = relations.detach().cpu().numpy() print(rel_score) _, predict_labels = torch.max(relations.data, 1) print(predict_labels) exit() test_labels = test_labels.cuda(GPU) rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) accuracy = total_rewards / 1.0 / CLASS_NUM / 15 accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:{:.4f}\th:{:.3f}".format(test_accuracy, h)) total_accuracy += test_accuracy print("aver_accuracy: {:.4f}".format(total_accuracy / EPISODE))
def main(): metatrain_folders, metatest_folders = tg.mini_imagenet_folders() print("init neural networks") feature_encoder = models.FeatureEncoder().apply(weights_init).cuda(GPU) relation_network = models.SimilarityNetwork( FEATURE_DIM, RELATION_DIM).apply(weights_init).cuda(GPU) feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=50000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=50000, gamma=0.5) if os.path.exists( str(METHOD + "/miniImagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str(METHOD + "/miniImagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str(METHOD + "/miniImagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str(METHOD + "/miniImagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") if os.path.exists(METHOD) == False: os.system('mkdir ' + METHOD) # Step 3: build graph print("Training...") best_accuracy = 0.0 best_h = 0.0 start = time.time() for episode in range(EPISODE): with torch.no_grad(): # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 counter = 0 task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, SUPPORT_NUM_PER_CLASS, 15) support_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SUPPORT_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 5 query_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=False) support_images, support_labels = support_dataloader.__iter__( ).next() for query_images, query_labels in query_dataloader: query_size = query_labels.shape[0] support_features = feature_encoder( Variable(support_images).cuda(GPU)) support_features = support_features.view( CLASS_NUM, SUPPORT_NUM_PER_CLASS, FEATURE_DIM, 19 * 19).sum(1) query_features = feature_encoder( Variable(query_images).cuda(GPU)).view( num_per_class * CLASS_NUM, 64, 19 * 19) H_support_features = Variable( torch.Tensor(CLASS_NUM, 1, 64, 64)).cuda(GPU) H_query_features = Variable( torch.Tensor(num_per_class * CLASS_NUM, 1, 64, 64)).cuda(GPU) for d in range(support_features.size()[0]): s = support_features[d, :, :].squeeze(0) s = (1.0 / support_features.size()[2]) * s.mm( s.transpose(0, 1)) H_support_features[d, :, :, :] = power_norm( s / s.trace(), SIGMA) for d in range(query_features.size()[0]): s = query_features[d, :, :].squeeze(0) s = (1.0 / query_features.size()[2]) * s.mm( s.transpose(0, 1)) H_query_features[d, :, :, :] = power_norm( s / s.trace(), SIGMA) support_features_ext = H_support_features.unsqueeze( 0).repeat(query_size, 1, 1, 1, 1) query_features_ext = H_query_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) query_features_ext = torch.transpose( query_features_ext, 0, 1) relation_pairs = torch.cat( (support_features_ext, query_features_ext), 2).view(-1, 2, 64, 64) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == query_labels[j].cuda(GPU) else 0 for j in range(query_size) ] total_rewards += np.sum(rewards) counter += query_size accuracy = total_rewards / 1.0 / counter accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:", test_accuracy, "h:", h) print("best accuracy:", best_accuracy, "h:", best_h) if test_accuracy > best_accuracy: # save networks torch.save( feature_encoder.state_dict(), str(METHOD + "/miniImagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")) torch.save( relation_network.state_dict(), str(METHOD + "/miniImagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")) print("save networks for episode:", episode) best_accuracy = test_accuracy best_h = h