def main(): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # * Step 1: init data folders print("init data folders") # * Init character folders for dataset construction metatrain_character_folders, metatest_character_folders = tg.mini_imagenet_folders( ) # * Step 2: init neural networks print("init neural networks") feature_encoder = CNNEncoder() RFT = RandomForestClassifier(n_estimators=100, random_state=1, n_jobs=-1, warm_start=True) #RFT = RandomForestClassifier(n_estimators=100, random_state=1, max_depth=10, n_jobs=-1, warm_start=True) relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM) feature_encoder.apply(weights_init) relation_network.apply(weights_init) feature_encoder.to(device) relation_network.to(device) cross_entropy = nn.CrossEntropyLoss() feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=100000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=100000, gamma=0.5) if os.path.exists( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/miniimagenet_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): RFT = pickle.load( open( str("./models/miniimagenet_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"), 'rb')) print("load random forest success") if os.path.exists( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") # * Step 3: build graph print("Training...") last_accuracy = 0.0 last_RFT_accuracy = 0 test_RFT_accuracy = 0 # embedding_loss_list = [] RFT_loss_list = [] relation_loss_list = [] loss_list = [] RFT_fit_index = 100 for episode in range(EPISODE): # * init dataset # * sample_dataloader is to obtain previous samples for compare # * batch_dataloader is to batch samples for training # degrees = random.choice([0, 90, 180, 270]) number_of_query_image = 10 task = tg.MiniImagenetTask(metatrain_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, number_of_query_image) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=number_of_query_image, split="test", shuffle=True) # * num_per_class : number of query images # * sample datas # samples, sample_labels = sample_dataloader.__iter__().next() # batches, batch_labels = batch_dataloader.__iter__().next() samples, sample_labels = next(iter(sample_dataloader)) batches, batch_labels = next(iter(batch_dataloader)) RFT_batches, RFT_batch_labels = batches, batch_labels samples, sample_labels = samples.to(device), sample_labels.to(device) batches, batch_labels = batches.to(device), batch_labels.to(device) # * calculates features sample_features = feature_encoder(samples) sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) # * Testing : mean vs sum sample_features = torch.sum(sample_features, 1).squeeze(1) batch_features = feature_encoder(batches) # * calculate relations # * each batch sample link to every samples to calculate relations # * to form a 100 * 128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( number_of_query_image * CLASS_NUM, 1, 1, 1, 1) batch_features_ext = batch_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) batch_features_ext = torch.transpose(batch_features_ext, 0, 1) relation_pairs = torch.cat((sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view(-1, CLASS_NUM) if episode > 30000: RFT_fit_index = 1000 if episode % RFT_fit_index == 0: RFT.fit(relations.detach().cpu(), RFT_batch_labels) RFT.n_estimators += 1 # one_hot_labels = torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).to(device).scatter_(1, batch_labels.view(-1, 1), 1) RFT_prob = torch.tensor(RFT.predict_proba( relations.detach().cpu())).to(device) _, RFT_labels = torch.max(RFT_prob, 1) RFT_loss = cross_entropy(relations, RFT_labels) * 0.7 relation_loss = cross_entropy(relations, batch_labels) loss = relation_loss + RFT_loss feature_encoder.zero_grad() relation_network.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(feature_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm_(relation_network.parameters(), 0.5) feature_encoder_optim.step() relation_network_optim.step() feature_encoder_scheduler.step(episode) relation_network_scheduler.step(episode) if (episode + 1) % 100 == 0: print( f"episode : {episode+1}, loss : {loss.cpu().detach().numpy()}") loss_list.append(loss.cpu().detach().numpy()) RFT_loss_list.append(RFT_loss.cpu().detach().numpy()) relation_loss_list.append(relation_loss.cpu().detach().numpy()) if (episode + 1) % 1000 == 0: print("Testing...") total_reward = 0 for i in range(TEST_EPISODE): number_of_query_image = 10 task = tg.MiniImagenetTask(metatest_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, number_of_query_image) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=number_of_query_image, split="test", shuffle=True) sample_images, sample_labels = next(iter(sample_dataloader)) test_images, test_labels = next(iter(test_dataloader)) sample_images, sample_labels = sample_images.to( device), sample_labels.to(device) test_images, test_labels = test_images.to( device), test_labels.to(device) # * calculate features sample_features = feature_encoder(sample_images) sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) test_features = feature_encoder(test_images) # * calculate relations # * each batch sample link to every samples to calculate relations # * to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( number_of_query_image * CLASS_NUM, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose(test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(CLASS_NUM * SAMPLE_NUM_PER_CLASS) ] total_reward += np.sum(rewards) if i % 200 == 0: RFT_predict = RFT.predict(relations.detach().cpu()) assert RFT_predict.shape == test_labels.detach().cpu( ).shape print( accuracy_score(RFT_predict, test_labels.detach().cpu())) test_RFT_accuracy += accuracy_score( RFT_predict, test_labels.detach().cpu()) test_accuracy = total_reward / ( 1.0 * CLASS_NUM * SAMPLE_NUM_PER_CLASS * TEST_EPISODE) test_RFT_accuracy /= (TEST_EPISODE // 200) # print("test accuracy : ", test_accuracy) print(f"{test_RFT_accuracy:.3f} %") mean_loss = np.mean(loss_list) mean_RFT_loss = np.mean(RFT_loss_list) mean_relation_loss = np.mean(relation_loss_list) print(f'mean loss : {mean_loss}') print(f'RFT loss : {mean_RFT_loss}') # writer.add_scalar('1.embedding loss', mean_embedding_loss, episode + 1) writer.add_scalar('1.RFT loss', mean_RFT_loss, episode + 1) writer.add_scalar('RFT_accuracy', test_RFT_accuracy, episode + 1) writer.add_scalar('2.relation loss', mean_relation_loss, episode + 1) writer.add_scalar('loss', mean_loss, episode + 1) writer.add_scalar('test accuracy', test_accuracy, episode + 1) loss_list = [] # embedding_loss_list = [] relation_loss_list = [] RFT_loss_list = [] if test_RFT_accuracy > last_RFT_accuracy: pickle.dump( RFT, open( str("./models/miniimagenet_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"), 'wb')) last_RFT_accuracy = test_RFT_accuracy print("save random forest for episode:", episode) test_RFT_accuracy = 0 print("test accuracy : ", test_accuracy) if test_accuracy > last_accuracy: # save networks torch.save( feature_encoder.state_dict(), str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_exp.pkl")) torch.save( relation_network.state_dict(), str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_exp.pkl")) print("save networks for episode:", episode) last_accuracy = test_accuracy
def main(): # Step 1: init data folders print("init data folders") # init character folders for dataset construction,遍历train、val所有类别并将其顺序打乱,metatrain_folders,metatest_folders分别记录训练、验证用list metatrain_folders, metatest_folders = tg.mini_imagenet_folders() # Step 2: init neural networks,分特征编码网络和关系网络 print("init neural networks") feature_encoder = CNNEncoder() relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM) feature_encoder.apply( weights_init ) ## pytorch网络初始化https://blog.csdn.net/dss_dssssd/article/details/83990511 relation_network.apply(weights_init) feature_encoder.cuda(GPU) relation_network.cuda(GPU) feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) #Adam feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=100000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=100000, gamma=0.5) if os.path.exists( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") # Step 3: build graph print("Training...") last_accuracy = 0.0 for episode in range(EPISODE): feature_encoder_scheduler.step(episode) relation_network_scheduler.step(episode) # init dataset # sample_dataloader is to obtain previous samples for compare # batch_dataloader is to batch samples for training,共100类,每类600个样本,存在有一些类别一直没有参与训练的问题,此处可以略作完善 task = tg.MiniImagenetTask( metatrain_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS ) #从train所有文件夹中随机抽5个文件夹,每个文件夹均随机取1张训练15张测试,每张图均有lable范围(0~4) #如下从train中取[[0],[1],[2],[3],[4]] sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) #内部用的均值方差比较特别,如何计算得到的 #如下从test中取[[0~15],[15~30],[30~45],[45~60],[60~75]] batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=BATCH_NUM_PER_CLASS, split="test", shuffle=True) # sample datas,https://blog.csdn.net/g11d111/article/details/81504637 samples, sample_labels = sample_dataloader.__iter__().next( ) ##这一步具体做什么,取出的数是什么样子的 batches, batch_labels = batch_dataloader.__iter__().next() # calculate features sample_features = feature_encoder( Variable(samples).cuda(GPU)) # 5x64*19*19 ##注释错了,原来注释为5*64*5*5 batch_features = feature_encoder(Variable(batches).cuda( GPU)) # 75x64*19*19#####注释错了,而原来注释为20*64*5*5) # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) ##75*5*64*19*19,repeat是扩增, batch_features_ext = batch_features.unsqueeze(0).repeat( SAMPLE_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) ##5*75*64*19*19 batch_features_ext = torch.transpose(batch_features_ext, 0, 1) #transpose是矩阵转制,75*5*64*19*19 relation_pairs = torch.cat((sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) #375*128*19*19 relations = relation_network(relation_pairs).view( -1, CLASS_NUM * SAMPLE_NUM_PER_CLASS) ##75*5 mse = nn.MSELoss().cuda( GPU) ##scatter_解析https://www.cnblogs.com/shiyublog/p/10924287.html one_hot_labels = Variable( torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).scatter_(1, batch_labels.view(-1, 1), 1)).cuda(GPU) loss = mse(relations, one_hot_labels) # training feature_encoder.zero_grad() relation_network.zero_grad() loss.backward() #clip_grad_norm梯度裁剪:在BP过程中会产生梯度消失(就是偏导无限接近0,导致长时记忆无法更新),那么最简单粗暴的方法,设定阈值,当梯度小于阈值时,更新的梯度为阈值 #https://blog.csdn.net/qq_29340857/article/details/70574528 torch.nn.utils.clip_grad_norm(feature_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm(relation_network.parameters(), 0.5) feature_encoder_optim.step() relation_network_optim.step() if (episode + 1) % 100 == 0: print( "episode:", episode + 1, "loss", loss.item()) #print("episode:",episode+1,"loss",loss.data[0]) if episode % 5000 == 0: # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 counter = 0 task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, 1, 15) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=1, split="train", shuffle=False) num_per_class = 3 test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=True) sample_images, sample_labels = sample_dataloader.__iter__( ).next() for test_images, test_labels in test_dataloader: test_labels = test_labels.cuda() batch_size = test_labels.shape[0] # calculate features sample_features = feature_encoder( Variable(sample_images).cuda(GPU)) # 5x64 test_features = feature_encoder( Variable(test_images).cuda(GPU)) # 20x64 # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose( test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) counter += batch_size accuracy = total_rewards / 1.0 / counter accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:", test_accuracy, "h:", h) if test_accuracy > last_accuracy: # save networks torch.save( feature_encoder.state_dict(), str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")) torch.save( relation_network.state_dict(), str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")) print("save networks for episode:", episode) last_accuracy = test_accuracy
def main(): # Step 1: init data folders print("init data folders") metatrain_folders,metatest_folders = tg.mini_imagenet_folders() # init character folders for dataset construction # Step 2: init neural networks print("init neural networks") feature_encoder = CNNEncoder() relation_network = RelationNetwork() feature_encoder_scheduler = tf.keras.optimizers.schedules.ExponentialDecay(LEARNING_RATE,100000,0.5,staircase=True) feature_encoder_optim = tf.keras.optimizers.Adam(learning_rate=0.001,epsilon=1e-08) relation_network_scheduler = tf.keras.optimizers.schedules.ExponentialDecay(LEARNING_RATE,100000,0.5,staircase=True) relation_network_optim = tf.keras.optimizers.Adam(learning_rate=0.001,epsilon=1e-08) if os.path.exists(str("models/miniimagenet_feature_encoder_" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot")): feature_encoder = tf.keras.models.load_model(str("models/miniimagenet_feature_encoder_" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot")) print("load feature encoder success") if os.path.exists(str("models/miniimagenet_relation_network_"+ str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot")): relation_network = tf.keras.models.load_model(str("models/miniimagenet_relation_network_" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot")) print("load relation network success") # Step 3: build graph print("Training...") last_accuracy = 0.0 for episode in range(EPISODE): task = tg.MiniImagenetTask(metatrain_folders,CLASS_NUM,SAMPLE_NUM_PER_CLASS,BATCH_NUM_PER_CLASS) sample_dataset = tg.dataset(task,SAMPLE_NUM_PER_CLASS,split='train',shuffle=False) batch_dataset = tg.dataset(task,BATCH_NUM_PER_CLASS,split='test',shuffle=True) sample_dataloader = tf.data.Dataset.from_generator(sample_dataset.generator, output_types=(tf.float32, tf.float32), output_shapes = ((84,84,3),(5,1))).batch(SAMPLE_NUM_PER_CLASS*CLASS_NUM).take(1) batch_dataloader = tf.data.Dataset.from_generator(batch_dataset.generator, output_types=(tf.float32, tf.float32), output_shapes = ((84,84,3),(5,1))).batch(BATCH_NUM_PER_CLASS*CLASS_NUM).take(1) samples,sample_labels = next(iter(sample_dataloader)) batches,batch_labels = next(iter(batch_dataloader)) loss = train_one_step(feature_encoder, relation_network, feature_encoder_optim, relation_network_optim, samples, sample_labels, batches, batch_labels).numpy() if (episode+1)%100 == 0: print("episode:",episode+1,"loss",loss) if episode%5000 == 0: # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): task = tg.MiniImagenetTask(metatest_folders,CLASS_NUM,SAMPLE_NUM_PER_CLASS,BATCH_NUM_PER_CLASS) sample_dataset = tg.dataset(task,SAMPLE_NUM_PER_CLASS,split='train',shuffle=False) batch_dataset = tg.dataset(task,5,split='test',shuffle=True) sample_dataloader = tf.data.Dataset.from_generator(sample_dataset.generator, output_types=(tf.float32, tf.float32), output_shapes = ((84,84,3),(5,1))).batch(SAMPLE_NUM_PER_CLASS*CLASS_NUM).take(1) batch_dataloader = tf.data.Dataset.from_generator(batch_dataset.generator, output_types=(tf.float32, tf.float32), output_shapes = ((84,84,3),(5,1))).batch(5*CLASS_NUM).take(1) samples,sample_labels = next(iter(sample_dataloader)) batches,batch_labels = next(iter(batch_dataloader)) accuracies.append(test(feature_encoder, relation_network, samples, sample_labels, batches, batch_labels)) test_accuracy,h = mean_confidence_interval(accuracies) print("test accuracy:",test_accuracy,"h:",h) if test_accuracy > last_accuracy: # save networks feature_encoder.save(str("models/miniimagenet_feature_encoder_" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot"),save_format='tf') relation_network.save(str("models/miniimagenet_relation_network_" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot"),save_format='tf') print("save networks for episode:",episode) last_accuracy = test_accuracy
def main(): print("init data folders") metatrain_folders, metaquery_folders = tg.mini_imagenet_folders() print("init neural networks") foreground_encoder = models.FeatureEncoder().apply(weights_init).cuda(GPU) background_encoder = models.FeatureEncoder().apply(weights_init).cuda(GPU) mixture_network = models.MixtureNetwork().apply(weights_init).cuda(GPU) relation_network = models.SimilarityNetwork( FEATURE_DIM, RELATION_DIM).apply(weights_init).cuda(GPU) vanilla_foreground_encoder = models.FeatureEncoder().apply( weights_init).cuda(GPU) vanilla_background_encoder = models.FeatureEncoder().apply( weights_init).cuda(GPU) vanilla_mixture_network = models.MixtureNetwork().apply(weights_init).cuda( GPU) foreground_encoder_optim = torch.optim.Adam( foreground_encoder.parameters(), lr=LEARNING_RATE) foreground_encoder_scheduler = StepLR(foreground_encoder_optim, step_size=100000, gamma=0.5) background_encoder_optim = torch.optim.Adam( background_encoder.parameters(), lr=LEARNING_RATE) background_encoder_scheduler = StepLR(background_encoder_optim, step_size=100000, gamma=0.5) mixture_network_optim = torch.optim.Adam(mixture_network.parameters(), lr=LEARNING_RATE) mixture_network_scheduler = StepLR(mixture_network_optim, step_size=100000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=100000, gamma=0.5) # Loading models if os.path.exists( str(METHOD + "/miniImagenet_foreground_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): foreground_encoder.load_state_dict( torch.load( str(METHOD + "/miniImagenet_foreground_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load foreground encoder success") if os.path.exists( str(METHOD + "/miniImagenet_background_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): background_encoder.load_state_dict( torch.load( str(METHOD + "/miniImagenet_background_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load background encoder success") if os.path.exists( str(METHOD + "/miniImagenet_mixture_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): mixture_network.load_state_dict( torch.load( str(METHOD + "/miniImagenet_mixture_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load mixture network success") if os.path.exists( str(METHOD + "/miniImagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str(METHOD + "/miniImagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") # Loading vanilla models if os.path.exists( str("./vanilla_models/miniImagenet_foreground_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): vanilla_foreground_encoder.load_state_dict( torch.load( str("./vanilla_models/miniImagenet_foreground_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load vanilla foreground encoder success") if os.path.exists( str("./vanilla_models/miniImagenet_background_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): vanilla_background_encoder.load_state_dict( torch.load( str("./vanilla_models/miniImagenet_background_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load vanilla background encoder success") if os.path.exists( str("./vanilla_models/miniImagenet_mixture_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): vanilla_mixture_network.load_state_dict( torch.load( str("./vanilla_models/miniImagenet_mixture_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load vanilla mixture network success") if os.path.exists(METHOD) == False: os.system('mkdir ' + METHOD) print("Training...") best_accuracy = 0.0 start = time.time() for episode in range(EPISODE): mse = nn.MSELoss().cuda(GPU) foreground_encoder_scheduler.step(episode) background_encoder_scheduler.step(episode) mixture_network_scheduler.step(episode) relation_network_scheduler.step(episode) # init dataset task = tg.MiniImagenetTask(metatrain_folders, CLASS_NUM, SUPPORT_NUM_PER_CLASS, BATCH_NUM_PER_CLASS) support_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SUPPORT_NUM_PER_CLASS, split="train", shuffle=False) query_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=BATCH_NUM_PER_CLASS, split="test", shuffle=True) # support datas support_img, support_sal, support_labels = support_dataloader.__iter__( ).next() query_img, query_sal, query_labels = query_dataloader.__iter__().next() # calculate features support_foreground_features = foreground_encoder( Variable(support_img * support_sal).cuda(GPU)).view( CLASS_NUM, SUPPORT_NUM_PER_CLASS, 64, 19, 19) support_background_features = background_encoder( Variable(support_img * (1 - support_sal)).cuda(GPU)).view( CLASS_NUM, SUPPORT_NUM_PER_CLASS, 64, 19, 19) query_foreground_features = foreground_encoder( Variable(query_img * query_sal).cuda(GPU)) query_background_features = background_encoder( Variable(query_img * (1 - query_sal)).cuda(GPU)) # Real-Representation Regularization (TriR), teacher network support_foreground_features_ = vanilla_foreground_encoder( Variable(support_img * support_sal).cuda(GPU)) support_background_features_ = vanilla_background_encoder( Variable(support_img * (1 - support_sal)).cuda(GPU)) support_mix_features_ = vanilla_mixture_network( support_foreground_features_ + support_background_features_) support_mix_features__ = mixture_network( (support_foreground_features + support_background_features).view( -1, 64, 19, 19)) TriR = args.beta * mse( support_mix_features__, Variable(support_mix_features_, requires_grad=False)) # Inter-class Hallucination support_foreground_features = support_foreground_features.unsqueeze( 2).repeat(1, 1, CLASS_NUM * SUPPORT_NUM_PER_CLASS, 1, 1, 1) support_background_features = support_background_features.view( 1, 1, CLASS_NUM * SUPPORT_NUM_PER_CLASS, 64, 19, 19).repeat(CLASS_NUM, SUPPORT_NUM_PER_CLASS, 1, 1, 1, 1) similarity_measure = similarity_func( support_background_features, CLASS_NUM, SUPPORT_NUM_PER_CLASS).view(CLASS_NUM, SUPPORT_NUM_PER_CLASS, -1, 1, 1) support_mix_features = mixture_network( (support_foreground_features + support_background_features).view( (CLASS_NUM**2) * (SUPPORT_NUM_PER_CLASS**2), 64, 19, 19)).view(CLASS_NUM, SUPPORT_NUM_PER_CLASS, -1, 64, 19**2) support_mix_features = (support_mix_features * similarity_measure).sum(2).sum(1) query_mix_features = mixture_network(query_foreground_features + query_background_features).view( -1, 64, 19**2) so_support_features = Variable(torch.Tensor(CLASS_NUM, 1, 64, 64)).cuda(GPU) so_query_features = Variable( torch.Tensor(BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 64, 64)).cuda(GPU) # second-order features for d in range(support_mix_features.size()[0]): s = support_mix_features[d, :, :].squeeze(0) s = (1.0 / support_mix_features.size()[2]) * s.mm(s.transpose( 0, 1)) so_support_features[d, :, :, :] = power_norm(s / s.trace(), SIGMA) for d in range(query_mix_features.size()[0]): s = query_mix_features[d, :, :].squeeze(0) s = (1.0 / query_mix_features.size()[2]) * s.mm(s.transpose(0, 1)) so_query_features[d, :, :, :] = power_norm(s / s.trace(), SIGMA) # calculate relations with 64x64 second-order features support_features_ext = so_support_features.unsqueeze(0).repeat( BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) query_features_ext = so_query_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) query_features_ext = torch.transpose(query_features_ext, 0, 1) relation_pairs = torch.cat((support_features_ext, query_features_ext), 2).view(-1, 2, 64, 64) relations = relation_network(relation_pairs).view(-1, CLASS_NUM) one_hot_labels = Variable( torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).scatter_(1, query_labels.view(-1, 1), 1)).cuda(GPU) loss = mse(relations, one_hot_labels) + TriR # update network parameters foreground_encoder.zero_grad() background_encoder.zero_grad() mixture_network.zero_grad() relation_network.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(foreground_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm_(background_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm_(mixture_network.parameters(), 0.5) torch.nn.utils.clip_grad_norm_(relation_network.parameters(), 0.5) foreground_encoder_optim.step() background_encoder_optim.step() mixture_network_optim.step() relation_network_optim.step() if np.mod(episode + 1, 100) == 0: print("episode:", episode + 1, "loss", loss.item()) if np.mod(episode, 2500) == 0: # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 counter = 0 task = tg.MiniImagenetTask(metaquery_folders, CLASS_NUM, SUPPORT_NUM_PER_CLASS, 15) support_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SUPPORT_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 2 query_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=True) support_img, support_sal, support_labels = support_dataloader.__iter__( ).next() for query_img, query_sal, query_labels in query_dataloader: query_size = query_labels.shape[0] # calculate foreground and background features support_foreground_features = foreground_encoder( Variable(support_img * support_sal).cuda(GPU)).view( CLASS_NUM, SUPPORT_NUM_PER_CLASS, 64, 19, 19) support_background_features = background_encoder( Variable( support_img * (1 - support_sal)).cuda(GPU)).view( CLASS_NUM, SUPPORT_NUM_PER_CLASS, 64, 19, 19) query_foreground_features = foreground_encoder( Variable(query_img * query_sal).cuda(GPU)) query_background_features = background_encoder( Variable(query_img * (1 - query_sal)).cuda(GPU)) # Inter-class Hallucination support_foreground_features = support_foreground_features.unsqueeze( 2).repeat(1, 1, CLASS_NUM * SUPPORT_NUM_PER_CLASS, 1, 1, 1) support_background_features = support_background_features.view( 1, 1, CLASS_NUM * SUPPORT_NUM_PER_CLASS, 64, 19, 19).repeat(CLASS_NUM, SUPPORT_NUM_PER_CLASS, 1, 1, 1, 1) similarity_measure = similarity_func( support_background_features, CLASS_NUM, SUPPORT_NUM_PER_CLASS).view(CLASS_NUM, SUPPORT_NUM_PER_CLASS, -1, 1, 1) support_mix_features = mixture_network( (support_foreground_features + support_background_features).view( (CLASS_NUM * SUPPORT_NUM_PER_CLASS)**2, 64, 19, 19)).view(CLASS_NUM, SUPPORT_NUM_PER_CLASS, -1, 64, 19**2) support_mix_features = (support_mix_features * similarity_measure).sum(2).sum(1) query_mix_features = mixture_network( query_foreground_features + query_background_features).view(-1, 64, 19**2) so_support_features = Variable( torch.Tensor(CLASS_NUM, 1, 64, 64)).cuda(GPU) so_query_features = Variable( torch.Tensor(query_size, 1, 64, 64)).cuda(GPU) # second-order features for d in range(support_mix_features.size()[0]): s = support_mix_features[d, :, :].squeeze(0) s = (1.0 / support_mix_features.size()[2]) * s.mm( s.transpose(0, 1)) so_support_features[d, :, :, :] = power_norm( s / s.trace(), SIGMA) for d in range(query_mix_features.size()[0]): s = query_mix_features[d, :, :].squeeze(0) s = (1.0 / query_mix_features.size()[2]) * s.mm( s.transpose(0, 1)) so_query_features[d, :, :, :] = power_norm( s / s.trace(), SIGMA) # calculate relations with 64x64 second-order features support_features_ext = so_support_features.unsqueeze( 0).repeat(query_size, 1, 1, 1, 1) query_features_ext = so_query_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) query_features_ext = torch.transpose( query_features_ext, 0, 1) relation_pairs = torch.cat( (support_features_ext, query_features_ext), 2).view(-1, 2, 64, 64) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == query_labels[j].cuda(GPU) else 0 for j in range(query_size) ] total_rewards += np.sum(rewards) counter += query_size accuracy = total_rewards / 1.0 / counter accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:", test_accuracy, "h:", h) if test_accuracy > best_accuracy: # save networks torch.save( foreground_encoder.state_dict(), str(METHOD + "/miniImagenet_foreground_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")) torch.save( background_encoder.state_dict(), str(METHOD + "/miniImagenet_background_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")) torch.save( mixture_network.state_dict(), str(METHOD + "/miniImagenet_mixture_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")) torch.save( relation_network.state_dict(), str(METHOD + "/miniImagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")) print("save networks for episode:", episode) best_accuracy = test_accuracy print("best accuracy:", best_accuracy)
from __future__ import division import torch import task_generator as tg import train metatrain_character_folders, metatest_character_folders = tg.mini_imagenet_folders( ) FEATURE_DIM = 64 RELATION_DIM = 8 CLASS_NUM = 5 SAMPLE_NUM_PER_CLASS = 1 BATCH_NUM_PER_CLASS = 15 EPISODE = 10 TEST_EPISODE = 600 DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') rn_trainer = train.RNTrainer(metatrain_character_folders, metatest_character_folders, FEATURE_DIM, RELATION_DIM, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS, DEVICE) rn_trainer.load_models() last_accuracy = 0.0 for r in range(EPISODE): test_accuracy = rn_trainer.validate(TEST_EPISODE) print('Completed episodes')
def main(): metatrain_folders, metaquery_folders = tg.mini_imagenet_folders() print("init neural networks") feature_encoder = models.FeatureEncoder().apply(weights_init).cuda(GPU) relation_network = models.SimilarityNetwork( FEATURE_DIM, RELATION_DIM).apply(weights_init).cuda(GPU) feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=50000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=50000, gamma=0.5) if os.path.exists( str(METHOD + "/feature_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str(METHOD + "/feature_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str(METHOD + "/relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str(METHOD + "/relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") if os.path.exists(METHOD) == False: os.system('mkdir ' + METHOD) # Step 3: build graph print("Training...") best_accuracy = 0.0 best_h = 0.0 for episode in range(EPISODE): with torch.no_grad(): print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 counter = 0 task = tg.MiniImagenetTask(metaquery_folders, CLASS_NUM, 1, 2) support_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=1, split="train", shuffle=False) num_per_class = 2 query_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="query", shuffle=True) support_images, support_labels = support_dataloader.__iter__( ).next() for query_images, query_labels in query_dataloader: query_size = query_labels.shape[0] # calculate features support_features = feature_encoder( Variable(support_images).cuda(GPU)).view( CLASS_NUM, SUPPORT_NUM_PER_CLASS, 64, 19**2).sum(1) query_features = feature_encoder( Variable(query_images).cuda(GPU)).view( num_per_class * CLASS_NUM, 64, 19**2) H_support_features = Variable( torch.Tensor(CLASS_NUM, 1, 64, 64)).cuda(GPU) H_query_features = Variable( torch.Tensor(num_per_class * CLASS_NUM, 1, 64, 64)).cuda(GPU) # HOP features for d in range(support_features.size()[0]): s = support_features[d, :, :].squeeze(0) s = s - LAMBDA * s.mean(1).repeat(1, s.size()[1]).view( s.size()) s = (1.0 / support_features.size()[2]) * s.mm( s.transpose(0, 1)) H_support_features[d, :, :, :] = power_norm( s / s.trace(), SIGMA) for d in range(query_features.size()[0]): s = query_features[d, :, :].squeeze(0) s = s - LAMBDA * s.mean(1).repeat(1, s.size()[1]).view( s.size()) s = (1.0 / query_features.size()[2]) * s.mm( s.transpose(0, 1)) H_query_features[d, :, :, :] = power_norm( s / s.trace(), SIGMA) # form relation pairs support_features_ext = H_support_features.unsqueeze( 0).repeat(query_size, 1, 1, 1, 1) query_features_ext = H_query_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) query_features_ext = torch.transpose( query_features_ext, 0, 1) relation_pairs = torch.cat( (support_features_ext, query_features_ext), 2).view(-1, 2, 64, 64) # calculate relation scores relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == query_labels[j].cuda(GPU) else 0 for j in range(query_size) ] total_rewards += np.sum(rewards) counter += query_size accuracy = total_rewards / 1.0 / counter accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("Test accuracy:", test_accuracy, "h:", h) print("Best accuracy: ", best_accuracy, "h:", best_h) if test_accuracy > best_accuracy: best_accuracy = test_accuracy best_h = h
def main(): # Step 1: init data folders print("init data folders") # init character folders for dataset construction metatrain_folders, metaquery_folders = tg.mini_imagenet_folders() # Step 2: init neural networks print("init neural networks") feature_encoder = models.FeatureEncoder().apply(weights_init).cuda(GPU) relation_network = models.SimilarityNetwork( FEATURE_DIM, RELATION_DIM).apply(weights_init).cuda(GPU) feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=50000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=50000, gamma=0.5) if os.path.exists( str(METHOD + "/feature_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str(METHOD + "/feature_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str(METHOD + "/relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str(METHOD + "/relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") if os.path.exists(METHOD) == False: os.system('mkdir ' + METHOD) # Step 3: build graph print("Training...") best_accuracy = 0.0 best_h = 0.0 for episode in range(EPISODE): feature_encoder_scheduler.step(episode) relation_network_scheduler.step(episode) # init dataset # support_dataloader is to obtain previous supports for compare # query_dataloader is to query supports for training task = tg.MiniImagenetTask(metatrain_folders, CLASS_NUM, SUPPORT_NUM_PER_CLASS, QUERY_NUM_PER_CLASS) support_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SUPPORT_NUM_PER_CLASS, split="train", shuffle=False) query_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=QUERY_NUM_PER_CLASS, split="test", shuffle=True) # support datas supports, support_labels = support_dataloader.__iter__().next() queries, query_labels = query_dataloader.__iter__().next() # calculate features support_features = feature_encoder(Variable(supports).cuda(GPU)).view( CLASS_NUM, SUPPORT_NUM_PER_CLASS, 64, 19**2).sum(1) # 5x64*19*19 query_features = feature_encoder(Variable(queries).cuda(GPU)).view( QUERY_NUM_PER_CLASS * CLASS_NUM, 64, 19**2) # 20x64*19*19 H_support_features = Variable( torch.Tensor(SUPPORT_NUM_PER_CLASS * CLASS_NUM, 1, 64, 64)).cuda(GPU) H_query_features = Variable( torch.Tensor(QUERY_NUM_PER_CLASS * CLASS_NUM, 1, 64, 64)).cuda(GPU) # HOP features for d in range(support_features.size()[0]): s = support_features[d, :, :].squeeze(0) s = s - LAMBDA * s.mean(1).repeat(1, s.size()[1]).view(s.size()) s = (1.0 / support_features.size()[2]) * s.mm(s.transpose(0, 1)) H_support_features[d, :, :, :] = power_norm(s / s.trace(), SIGMA) for d in range(query_features.size()[0]): s = query_features[d, :, :].squeeze(0) s = s - LAMBDA * s.mean(1).repeat(1, s.size()[1]).view(s.size()) s = (1.0 / query_features.size()[2]) * s.mm(s.transpose(0, 1)) H_query_features[d, :, :, :] = power_norm(s / s.trace(), SIGMA) # form relation pairs support_features_ext = H_support_features.unsqueeze(0).repeat( QUERY_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) query_features_ext = H_query_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) query_features_ext = torch.transpose(query_features_ext, 0, 1) relation_pairs = torch.cat((support_features_ext, query_features_ext), 2).view(-1, 2, 64, 64) # calculate relation scores relations = relation_network(relation_pairs).view( -1, CLASS_NUM * SUPPORT_NUM_PER_CLASS) # define loss function mse = nn.MSELoss().cuda(GPU) one_hot_labels = Variable( torch.zeros(QUERY_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).scatter_(1, query_labels.view(-1, 1), 1)).cuda(GPU) loss = mse(relations, one_hot_labels) # updating network parameters with their gradients feature_encoder.zero_grad() relation_network.zero_grad() loss.backward() feature_encoder_optim.step() relation_network_optim.step() if (episode + 1) % 100 == 0: print("episode:", episode + 1, "loss", loss.data[0]) if episode % 500 == 0: # query print("Testing...") accuracies = [] for i in range(TEST_EPISODE): with torch.no_grad(): total_rewards = 0 counter = 0 task = tg.MiniImagenetTask(metaquery_folders, CLASS_NUM, 1, 2) support_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=1, split="train", shuffle=False) num_per_class = 2 query_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="query", shuffle=True) support_images, support_labels = support_dataloader.__iter__( ).next() for query_images, query_labels in query_dataloader: query_size = query_labels.shape[0] # calculate features support_features = feature_encoder( Variable(support_images).cuda(GPU)).view( CLASS_NUM, SUPPORT_NUM_PER_CLASS, 64, 19**2).sum(1) query_features = feature_encoder( Variable(query_images).cuda(GPU)).view( num_per_class * CLASS_NUM, 64, 19**2) H_support_features = Variable( torch.Tensor(SUPPORT_NUM_PER_CLASS * CLASS_NUM, 1, 64, 64)).cuda(GPU) H_query_features = Variable( torch.Tensor(num_per_class * CLASS_NUM, 1, 64, 64)).cuda(GPU) # HOP features for d in range(support_features.size()[0]): s = support_features[d, :, :].squeeze(0) s = s - LAMBDA * s.mean(1).repeat( 1, s.size()[1]).view(s.size()) s = (1.0 / support_features.size()[2]) * s.mm( s.transpose(0, 1)) H_support_features[d, :, :, :] = power_norm( s / s.trace(), SIGMA) for d in range(query_features.size()[0]): s = query_features[d, :, :].squeeze(0) s = s - LAMBDA * s.mean(1).repeat( 1, s.size()[1]).view(s.size()) s = (1.0 / query_features.size()[2]) * s.mm( s.transpose(0, 1)) H_query_features[d, :, :, :] = power_norm( s / s.trace(), SIGMA) # form relation pairs support_features_ext = H_support_features.unsqueeze( 0).repeat(query_size, 1, 1, 1, 1) query_features_ext = H_query_features.unsqueeze( 0).repeat(1 * CLASS_NUM, 1, 1, 1, 1) query_features_ext = torch.transpose( query_features_ext, 0, 1) relation_pairs = torch.cat( (support_features_ext, query_features_ext), 2).view(-1, 2, 64, 64) # calculate relation scores relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == query_labels[j].cuda(GPU) else 0 for j in range(query_size) ] total_rewards += np.sum(rewards) counter += query_size accuracy = total_rewards / 1.0 / counter accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("Test accuracy:", test_accuracy, "h:", h) print("Best accuracy: ", best_accuracy, "h:", best_h) if test_accuracy > best_accuracy: # save networks torch.save( feature_encoder.state_dict(), str(METHOD + "/feature_encoder_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")) torch.save( relation_network.state_dict(), str(METHOD + "/relation_network_" + str(CLASS_NUM) + "way_" + str(SUPPORT_NUM_PER_CLASS) + "shot.pkl")) print("save networks for episode:", episode) best_accuracy = test_accuracy best_h = h
def main(): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # * Step 1: init data folders print("init data folders") # * Init character folders for dataset construction metatrain_character_folders, metatest_character_folders = tg.mini_imagenet_folders( ) # * Step 2: init neural networks print("init neural networks") feature_encoder = models.CNNEncoder() actor = models.Actor(FEATURE_DIM, RELATION_DIM, CLASS_NUM) critic = models.Critic(FEATURE_DIM, RELATION_DIM) #feature_encoder = torch.nn.DataParallel(feature_encoder) #actor = torch.nn.DataParallel(actor) #critic = torch.nn.DataParallel(critic) feature_encoder.train() actor.train() critic.train() feature_encoder.apply(models.weights_init) actor.apply(models.weights_init) critic.apply(models.weights_init) feature_encoder.to(device) actor.to(device) critic.to(device) cross_entropy = nn.CrossEntropyLoss() feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=10000, gamma=0.5) actor_optim = torch.optim.Adam(actor.parameters(), lr=2.5 * LEARNING_RATE) actor_scheduler = StepLR(actor_optim, step_size=10000, gamma=0.5) critic_optim = torch.optim.Adam(critic.parameters(), lr=2.5 * LEARNING_RATE * 10) critic_scheduler = StepLR(critic_optim, step_size=10000, gamma=0.5) agent = a2cAgent.A2CAgent(actor, critic, GAMMA, ENTROPY_WEIGHT, CLASS_NUM, device) if os.path.exists( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/miniimagenet_actor_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): actor.load_state_dict( torch.load( str("./models/miniimagenet_actor_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load actor network success") if os.path.exists( str("./models/miniimagenet_critic_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): critic.load_state_dict( torch.load( str("./models/miniimagenet_critic_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load critic network success") # * Step 3: build graph print("Training...") last_accuracy = 0.0 mbal_loss_list = [] mbcl_loss_list = [] loss_list = [] number_of_query_image = 15 for episode in range(EPISODE): #print(f"EPISODE : {episode}") policy_losses = [] value_losses = [] for meta_batch in range(META_BATCH_RANGE): meta_env_states_list = [] meta_env_labels_list = [] for inner_batch in range(INNER_BATCH_RANGE): # * Generate environment env_states_list = [] env_labels_list = [] for env in range(ENV_LENGTH): task = tg.MiniImagenetTask(metatrain_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, number_of_query_image) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=5, split="test", shuffle=True) samples, sample_labels = next(iter(sample_dataloader)) samples, sample_labels = samples.to( device), sample_labels.to(device) for batches, batch_labels in batch_dataloader: batches, batch_labels = batches.to( device), batch_labels.to(device) inner_sample_features = feature_encoder(samples) inner_sample_features = inner_sample_features.view( CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) inner_sample_features = torch.sum( inner_sample_features, 1).squeeze(1) inner_batch_features = feature_encoder(batches) inner_sample_feature_ext = inner_sample_features.unsqueeze( 0).repeat(5 * CLASS_NUM, 1, 1, 1, 1) inner_batch_features_ext = inner_batch_features.unsqueeze( 0).repeat(CLASS_NUM, 1, 1, 1, 1) inner_batch_features_ext = torch.transpose( inner_batch_features_ext, 0, 1) inner_relation_pairs = torch.cat( (inner_sample_feature_ext, inner_batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) env_states_list.append(inner_relation_pairs) env_labels_list.append(batch_labels) inner_env = a2cAgent.env(env_states_list, env_labels_list) agent.train(inner_env, inner_update=True) for meta_env in range(META_ENV_LENGTH): task = tg.MiniImagenetTask(metatrain_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, number_of_query_image) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=number_of_query_image, split="test", shuffle=True) # * num_per_class : number of query images # * sample datas samples, sample_labels = next(iter(sample_dataloader)) samples, sample_labels = samples.to(device), sample_labels.to( device) # * Generate env for meta update batches, batch_labels = next(iter(batch_dataloader)) # * init dataset # * sample_dataloader is to obtain previous samples for compare # * batch_dataloader is to batch samples for training batches, batch_labels = batches.to(device), batch_labels.to( device) # * calculates features #feature_encoder.weight = feature_fast_weights sample_features = feature_encoder(samples) sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) batch_features = feature_encoder(batches) # * calculate relations # * each batch sample link to every samples to calculate relations # * to form a 100 * 128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( number_of_query_image * CLASS_NUM, 1, 1, 1, 1) batch_features_ext = batch_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) batch_features_ext = torch.transpose(batch_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) meta_env_states_list.append(relation_pairs) meta_env_labels_list.append(batch_labels) meta_env = a2cAgent.env(meta_env_states_list, meta_env_labels_list) agent.train(meta_env, policy_loss_list=policy_losses, value_loss_list=value_losses) feature_encoder_optim.zero_grad() actor_optim.zero_grad() critic_optim.zero_grad() torch.nn.utils.clip_grad_norm_(feature_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm_(actor.parameters(), 0.5) torch.nn.utils.clip_grad_norm_(critic.parameters(), 0.5) meta_batch_actor_loss = torch.stack(policy_losses).mean() meta_batch_critic_loss = torch.stack(value_losses).mean() meta_batch_actor_loss.backward(retain_graph=True) meta_batch_critic_loss.backward() feature_encoder_optim.step() actor_optim.step() critic_optim.step() feature_encoder_scheduler.step() actor_scheduler.step() critic_scheduler.step() if (episode + 1) % 100 == 0: mbal = meta_batch_actor_loss.cpu().detach().numpy() mbcl = meta_batch_critic_loss.cpu().detach().numpy() print( f"episode : {episode+1}, meta_batch_actor_loss : {mbal:.4f}, meta_batch_critic_loss : {mbcl:.4f}" ) mbal_loss_list.append(mbal) mbcl_loss_list.append(mbcl) loss_list.append(mbal + mbcl) if (episode + 1) % 500 == 0: print("Testing...") total_reward = 0 total_num_of_test_samples = 0 for i in range(TEST_EPISODE): # * Generate env env_states_list = [] env_labels_list = [] number_of_query_image = 10 task = tg.MiniImagenetTask(metatest_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, number_of_query_image) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=number_of_query_image, split="test", shuffle=True) sample_images, sample_labels = next(iter(sample_dataloader)) sample_images, sample_labels = sample_images.to( device), sample_labels.to(device) test_images, test_labels = next(iter(test_dataloader)) total_num_of_test_samples += len(test_labels) test_images, test_labels = test_images.to( device), test_labels.to(device) # * calculate features sample_features = feature_encoder(sample_images) sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) test_features = feature_encoder(test_images) # * calculate relations # * each batch sample link to every samples to calculate relations # * to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( number_of_query_image * CLASS_NUM, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose(test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) env_states_list.append(relation_pairs) env_labels_list.append(test_labels) test_env = a2cAgent.env(env_states_list, env_labels_list) rewards = agent.test(test_env) total_reward += rewards test_accuracy = total_reward / (1.0 * total_num_of_test_samples) mean_loss = np.mean(loss_list) mean_actor_loss = np.mean(mbal_loss_list) mean_critic_loss = np.mean(mbcl_loss_list) print(f'mean loss : {mean_loss}') print("test accuracy : ", test_accuracy) writer.add_scalar('1.loss', mean_loss, episode + 1) writer.add_scalar('2.mean_actor_loss', mean_actor_loss, episode + 1) writer.add_scalar('3.mean_critic_loss', mean_critic_loss, episode + 1) writer.add_scalar('4.test accuracy', test_accuracy, episode + 1) loss_list = [] mbal_loss_list = [] mbcl_loss_list = [] if test_accuracy > last_accuracy: # save networks torch.save( feature_encoder.state_dict(), str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")) torch.save( actor.state_dict(), str("./models/miniimagenet_actor_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")) torch.save( critic.state_dict(), str("./models/miniimagenet_critic_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")) print("save networks for episode:", episode) last_accuracy = test_accuracy
def main(): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # * Step 1: init data folders print("init data folders") # * Init character folders for dataset construction metatrain_character_folders, metatest_character_folders = tg.mini_imagenet_folders( ) # * Step 2: init neural networks print("init neural networks") feature_encoder = models.CNNEncoder() actor = models.Actor(FEATURE_DIM, RELATION_DIM, CLASS_NUM) critic = models.Critic(FEATURE_DIM, RELATION_DIM) #feature_encoder = torch.nn.DataParallel(feature_encoder) #actor = torch.nn.DataParallel(actor) #critic = torch.nn.DataParallel(critic) feature_encoder.train() actor.train() critic.train() feature_encoder.apply(models.weights_init) actor.apply(models.weights_init) critic.apply(models.weights_init) feature_encoder.to(device) actor.to(device) critic.to(device) agent = a2cAgent.A2CAgent(actor, critic, GAMMA, ENTROPY_WEIGHT, FEATURE_DIM, RELATION_DIM, CLASS_NUM, device) #feature_encoder.eval() #relation_network.eval() if os.path.exists( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/miniimagenet_actor_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): actor.load_state_dict( torch.load( str("./models/miniimagenet_actor_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load actor network success") if os.path.exists( str("./models/miniimagenet_critic_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): critic.load_state_dict( torch.load( str("./models/miniimagenet_critic_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load critic network success") max_accuracy_list = [] mean_accuracy_list = [] for episode in range(1): total_accuracy = [] for i in range(TEST_EPISODE): # * Generate env env_states_list = [] env_labels_list = [] number_of_query_image = 15 task = tg.MiniImagenetTask(metatest_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, number_of_query_image) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=number_of_query_image, split="test", shuffle=True) sample_images, sample_labels = next(iter(sample_dataloader)) test_images, test_labels = next(iter(test_dataloader)) sample_images, sample_labels = sample_images.to( device), sample_labels.to(device) test_images, test_labels = test_images.to(device), test_labels.to( device) # * calculate features sample_features = feature_encoder(sample_images) sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) test_features = feature_encoder(test_images) # * calculate relations # * each batch sample link to every samples to calculate relations # * to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( number_of_query_image * CLASS_NUM, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose(test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) env_states_list.append(relation_pairs) env_labels_list.append(test_labels) test_env = a2cAgent.env(env_states_list, env_labels_list) rewards = agent.test(test_env) test_accuracy = rewards / len(test_labels) print(test_accuracy) total_accuracy.append(test_accuracy) mean_accuracy, conf_int = mean_confidence_interval(total_accuracy) print(f"Total accuracy : {mean_accuracy:.4f}") print(f"confidence interval : {conf_int:.4f}")
def main(): # * Step 1: init data folders print("init data folders") # * init character folders for dataset construction metatrain_character_folders, metatest_character_folders = tg.mini_imagenet_folders( ) # * Step 2: init neural networks print("init neural networks") feature_encoder = ot.CNNEncoder().to(device) RFT = RandomForestClassifier(n_estimators=100, random_state=1, warm_start=True) relation_network = ot.RelationNetwork(FEATURE_DIM, RELATION_DIM).to(device) #feature_encoder.eval() #relation_network.eval() if os.path.exists( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_exp.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_exp.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/miniimagenet_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): RFT = pickle.load( open( str("./models/miniimagenet_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"), 'rb')) print("load random forest success") if os.path.exists( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_exp.pkl")): relation_network.load_state_dict( torch.load( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_exp.pkl"))) print("load relation network success") total_accuracy = 0.0 max_accuracy_list = [] mean_accuracy_list = [] for test in range(5): print("Testing...") max_accuracy = 0 total_accuracy = [] number_of_query_image = 15 print(f"Test {test}") for i in range(600): total_reward = 0 task = tg.MiniImagenetTask(metatest_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, number_of_query_image) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=number_of_query_image, split="test", shuffle=False) sample_images, sample_labels = next(iter(sample_dataloader)) sample_images, sample_labels = sample_images.to( device), sample_labels.to(device) # print(f"Episode {i}") for test_images, test_labels in test_dataloader: #print(test_labels.shape) test_images, test_labels = test_images.to( device), test_labels.to(device) batch_size = test_labels.shape[0] # * calculate features sample_features = feature_encoder(sample_images) sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) test_features = feature_encoder(test_images) # * calculate relations # * each batch sample link to every samples to calculate relations # * to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose(test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) #RFT_prob = RFT.predict_proba(relations.detach().cpu()) #relation_prob = torch.softmax(relations.data, dim=1) #RFT_prob_tensor = torch.tensor(RFT_prob).to(device) #soft_voting = (RFT_prob_tensor * 0.7) + relation_prob #soft_voting = (RFT_prob_tensor / relation_prob) #_, soft_voting_predicted_labels = torch.max(soft_voting, 1) _, predict_labels = torch.max(relations.data, 1) #print(predict_labels.item()) #print(test_labels.item()) rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(CLASS_NUM * 5) ] #rewards = [1 if soft_voting_predicted_labels[j] == test_labels[j] else 0 for j in range(CLASS_NUM * number_of_query_image)] total_reward += np.sum(rewards) #print(total_reward) test_accuracy = total_reward / (1.0 * CLASS_NUM * 15) #print(test_accuracy) total_accuracy.append(test_accuracy) if test_accuracy > max_accuracy: max_accuracy = test_accuracy test_accuracy, h = mean_confidence_interval(total_accuracy) print(f"Final result : {test_accuracy:.4f}, h : {h:.4f} ") mean_accuracy = np.mean(total_accuracy) mean_accuracy_list.append(mean_accuracy) print(f"Total accuracy : {mean_accuracy:.4f}") print(f"max accuracy : {max_accuracy:.4f}") max_accuracy_list.append(max_accuracy) ''' test_accuracy, h = mean_confidence_interval(accuracies) print(f'test accuracy : {test_accuracy:.4f}, h : {h:.4f}') total_accuracy += test_accuracy print(f"average accuracy : {total_accuracy/10 :.4f}") ''' final_accuracy, h = mean_confidence_interval(max_accuracy_list) print(f"Final result : {final_accuracy:.4f}, h : {h:.4f} ") print(np.sort(mean_accuracy_list))
def main(): save_path = make_dir(way=CLASS_NUM, shot=SAMPLE_NUM_PER_CLASS, batch_size=BATCH_NUM_PER_CLASS) # Step 1: init data folders print("init data folders") # init character folders for dataset construction metatrain_folders, metatest_folders, metaval_folders = tg.mini_imagenet_folders( ) # Step 2: init neural networks print("init neural networks") feature_encoder = CNNEncoder() RP = RP_Network(GPU=GPU) RM = RM_Network(64, HIDDEN_UNIT, GPU=GPU) RP.apply(weights_init_kaiming) RM.apply(weights_init_kaiming) feature_encoder.cuda(GPU) RP.cuda(GPU) RM.cuda(GPU) # Implement Optimizer and Scheduler feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = ReduceLROnPlateau(feature_encoder_optim, mode="max", factor=0.5, patience=2, verbose=True) RP_optim = torch.optim.Adam(RP.parameters(), lr=LEARNING_RATE) RP_scheduler = ReduceLROnPlateau(RP_optim, mode="max", factor=0.5, patience=2, verbose=True) RM_optim = torch.optim.Adam(RM.parameters(), lr=LEARNING_RATE) RM_optim_scheduler = ReduceLROnPlateau(RM_optim, mode='max', factor=0.5, patience=2, verbose=True) # somethings to save the data loss_list = [] acc_list = [] episode_list = [] print("Training...") last_accuracy = 0.0 for episode in range(EPISODE): #feature_encoder_scheduler.step(episode) # RM_scheduler.step(episode) # init dataset # sample_dataloader is to obtain previous samples for compare # batch_dataloader is to batch samples for training task = tg.MiniImagenetTask(metatrain_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=BATCH_NUM_PER_CLASS, split="test", shuffle=True, train_query_argue=True) # sample datas samples, sample_labels = sample_dataloader.__iter__().next( ) #25*3*84*84 batches, batch_labels = batch_dataloader.__iter__().next() # calculate features sample_features = feature_encoder( Variable(samples).cuda(GPU)) # 25*64*19*19 sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) batch_features = feature_encoder( Variable(batches).cuda(GPU)) # 20x64*5*5 """--------------- Phrase of RPN ----------------------""" sample_features_ext = sample_features.unsqueeze(0).repeat( BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) batch_features_ext = batch_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) batch_features_ext = torch.transpose(batch_features_ext, 0, 1) relation_pairs = torch.cat((sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) Att = RP(relation_pairs) """----------------------------------------------------------------""" """------------------Phrase of RMN-----------------------------------------""" sample_features_ext = sample_features.unsqueeze(0).repeat( BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) sample_features_ext = sample_features_ext.view(-1, FEATURE_DIM, 19, 19).contiguous() batch_features_ext = batch_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) batch_features_ext = torch.transpose(batch_features_ext, 0, 1) batch_features_ext = batch_features_ext.contiguous().view( -1, FEATURE_DIM, 19, 19) batch_features_ext_att = batch_features_ext + batch_features_ext * Att.expand_as( batch_features_ext) relations = RM(sample_features_ext, batch_features_ext_att).view(-1, CLASS_NUM) """----------------------------------------------------------------""" # BP and Optimize mse = nn.MSELoss().cuda(GPU) one_hot_labels = Variable( torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).scatter_(1, batch_labels.view(-1, 1), 1).cuda(GPU)) loss = mse(relations, one_hot_labels) # training feature_encoder.zero_grad() RP.zero_grad() RM.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(feature_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm(RP.parameters(), 0.5) torch.nn.utils.clip_grad_norm(RM.parameters(), 0.5) feature_encoder_optim.step() RP_optim.step() RM_optim.step() if (episode + 1) % 100 == 0: print("episode:", episode + 1, "loss", loss.item()) # Validation if episode % 5000 == 0: print("validation...") accuracies_val = [] for i in range(VAL_EPISODE): total_rewards = 0 task = tg.MiniImagenetTask(metaval_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, 15) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 5 val_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=False) sample_images, sample_labels = sample_dataloader.__iter__( ).next() for val_images, val_labels in val_dataloader: val_images, val_labels = val_images.cuda( GPU), val_labels.cuda(GPU) batch_size = val_labels.shape[0] # calculate features sample_features = feature_encoder( Variable(sample_images).cuda(GPU)) # 5x64 sample_features = sample_features.view( CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) val_features = feature_encoder( Variable(val_images).cuda(GPU)) # 20x64 """---------------RPN----------------------""" # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) val_features_ext = val_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) val_features_ext = torch.transpose(val_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, val_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) # relations = RP(sample_features_ext,test_features_ext).view(-1,CLASS_NUM) Att = RP(relation_pairs) # print(Att) """-----------------------------------------------""" """------------------RMN-----------------------------------------""" sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) sample_features_ext = sample_features_ext.view( -1, FEATURE_DIM, 19, 19) val_features_ext = val_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) val_features_ext = torch.transpose(val_features_ext, 0, 1) val_features_ext = val_features_ext.contiguous().view( -1, FEATURE_DIM, 19, 19) val_features_ext_att = val_features_ext + val_features_ext * Att.expand_as( val_features_ext) relations = RM(sample_features_ext, val_features_ext_att).view(-1, CLASS_NUM) """-----------------------------------------------""" _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == val_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) acc_val = total_rewards / 1.0 / CLASS_NUM / 15 accuracies_val.append(acc_val) val_accuracy, h = mean_confidence_interval(accuracies_val) feature_encoder_scheduler.step(val_accuracy) RP_scheduler.step(val_accuracy) RM_optim_scheduler.step(val_accuracy) print("Acc_Val", val_accuracy, 'Episode', episode) if episode % 500 == 0: # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, 15) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 5 test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=False) sample_images, sample_labels = sample_dataloader.__iter__( ).next() for test_images, test_labels in test_dataloader: test_images, test_labels = test_images.cuda( GPU), test_labels.cuda(GPU) batch_size = test_labels.shape[0] # calculate features sample_features = feature_encoder( Variable(sample_images).cuda(GPU)) # 5x64 sample_features = sample_features.view( CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) test_features = feature_encoder( Variable(test_images).cuda(GPU)) # 20x64 """---------------RPN----------------------""" # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose( test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) # relations = RP(sample_features_ext,test_features_ext).view(-1,CLASS_NUM) Att = RP(relation_pairs) """-----------------------------------------------""" """------------------RMN-----------------------------------------""" sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) sample_features_ext = sample_features_ext.view( -1, FEATURE_DIM, 19, 19) test_features_ext = test_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose( test_features_ext, 0, 1) test_features_ext = test_features_ext.contiguous().view( -1, FEATURE_DIM, 19, 19) test_features_ext_att = test_features_ext + test_features_ext * Att.expand_as( test_features_ext) relations = RM(sample_features_ext, test_features_ext_att).view(-1, CLASS_NUM) """-----------------------------------------------""" _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) accuracy = total_rewards / 1.0 / CLASS_NUM / 15 accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:", test_accuracy, "h:", h) acc_list.append(test_accuracy) episode_list.append(episode) if test_accuracy > last_accuracy: # save networks torch.save(feature_encoder.state_dict(), save_path + "encoder.pkl") torch.save(RP.state_dict(), save_path + "RP.pkl") torch.save(RM.state_dict(), save_path + "RM.pkl") print("save networks for episode:", episode) last_accuracy = test_accuracy #save data np.savetxt(save_path + "acc.txt", acc_list) np.savetxt(save_path + "episode.txt", episode_list)
def main(): # Step 1: init data folders print("init data folders") # init character folders for dataset construction metatrain_folders, metatest_folders = tg.mini_imagenet_folders() # Step 2: init neural networks print("init neural networks") feature_encoder = CNNEncoder() relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM) feature_encoder.apply(weights_init) relation_network.apply(weights_init) feature_encoder.cuda() relation_network.cuda() feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=100000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=100000, gamma=0.5) if os.path.exists( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") path = os.path.join( 'logs', str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot") if not os.path.exists(path): os.makedirs(path) writer = SummaryWriter(path) # Step 3: build graph print("Training...") last_accuracy = 0.0 for episode in range(EPISODE): feature_encoder_scheduler.step(episode) relation_network_scheduler.step(episode) # init dataset # sample_dataloader is to obtain previous samples for compare # batch_dataloader is to batch samples for training task = tg.MiniImagenetTask(metatrain_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=BATCH_NUM_PER_CLASS, split="test", shuffle=True) # sample datas samples, sample_labels = sample_dataloader.__iter__().next( ) #25*3*84*84 batches, batch_labels = batch_dataloader.__iter__().next() # calculate features sample_features = feature_encoder( Variable(samples).cuda()) # 25*64*19*19 sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) batch_features = feature_encoder(Variable(batches).cuda()) # 20x64*5*5 # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) batch_features_ext = batch_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) batch_features_ext = torch.transpose(batch_features_ext, 0, 1) relation_pairs = torch.cat((sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view(-1, CLASS_NUM) mse = nn.MSELoss().cuda() one_hot_labels = Variable( torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).scatter_(1, batch_labels.view(-1, 1), 1).cuda()) loss = mse(relations, one_hot_labels) # training feature_encoder.zero_grad() relation_network.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(feature_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm_(relation_network.parameters(), 0.5) feature_encoder_optim.step() relation_network_optim.step() writer.add_scalar('training loss', loss.data.item(), episode + 1) if (episode + 1) % 100 == 0: print("episode:", episode + 1, "loss", loss.data.item()) if episode % 5000 == 0: # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, 15) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 5 test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=False) sample_images, sample_labels = sample_dataloader.__iter__( ).next() for test_images, test_labels in test_dataloader: batch_size = test_labels.shape[0] # calculate features sample_features = feature_encoder( Variable(sample_images).cuda()) # 5x64 sample_features = sample_features.view( CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) test_features = feature_encoder( Variable(test_images).cuda()) # 20x64 # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose( test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) predict_labels = predict_labels.cpu() rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) accuracy = total_rewards / 1.0 / CLASS_NUM / 15 accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) writer.add_scalar('test accuracy', test_accuracy, episode + 1) print("test accuracy:", test_accuracy, "h:", h) if test_accuracy > last_accuracy: # save networks torch.save( feature_encoder.state_dict(), str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")) torch.save( relation_network.state_dict(), str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")) print("save networks for episode:", episode) last_accuracy = test_accuracy writer.close()
def main(): # * Step 1: init data folders print("init data folders") # * init character folders for dataset construction metatrain_folders, metatest_folders = tg.mini_imagenet_folders() # * Step 2: init neural networks print("init neural networks") feature_encoder = ot.CNNEncoder().to(device) # RFT = RandomForestClassifier(n_estimators=100, random_state=1, warm_start=True) relation_network = ot.RelationNetwork(FEATURE_DIM, RELATION_DIM).to(device) feature_encoder.eval() relation_network.eval() if os.path.exists( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") ''' if os.path.exists(str("./models/miniimagenet_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): RFT = pickle.load(open(str("./models/miniimagenet_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"), 'rb')) print("load random forest success") ''' if os.path.exists( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") total_accuracy = 0.0 for episode in range(10): # * test print("Testing...") accuracies = [] for i in range(100): total_rewards = 0 # degrees = random.choice([0, 90, 180, 270]) task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) test_dataloader = tg.get_mini_imagenet_data_loader(task, num_per_class=5, split="test", shuffle=False) sample_images, sample_labels = next(iter(sample_dataloader)) sample_images, sample_labels = sample_images.to( device), sample_labels.to(device) for test_images, test_labels in test_dataloader: batch_size = test_labels.shape[0] test_images, test_labels = test_images.to( device), test_labels.to(device) # * Calculate features sample_features = feature_encoder(sample_images) sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) test_features = feature_encoder(test_images) sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose(test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) accuracy = total_rewards / (1.0 * CLASS_NUM * 15) accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print(f'test accuracy : {test_accuracy:.4f}, h : {h:.4f}') total_accuracy += test_accuracy print(f"average accuracy : {total_accuracy/10 :.4f}")
def main(): # Step 1: init data folders print("init data folders") # init character folders for dataset construction metatrain_folders, metaval_folders, metatest_folders = tg.mini_imagenet_folders( ) # Step 2: init neural networks and results dir print("init neural networks") path = makedir(name) # the path to save the results feature_encoder = FeatureEncoder() middle_layer = Middle_Moudle_v3(width=WIDTH) explain_network = ExplainModule_v6_meta_pair( n_way=CLASS_NUM, k_shot=SAMPLE_NUM_PER_CLASS, width=WIDTH, batch_size=BATCH_NUM_PER_CLASS) feature_encoder.apply(weights_init_FE) feature_encoder.cuda(GPU) explain_network.cuda(GPU) feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = ReduceLROnPlateau(feature_encoder_optim, mode="max", factor=0.5, patience=2, verbose=True, threshold=1e-2) explain_network_optim = torch.optim.Adam(explain_network.parameters(), lr=LEARNING_RATE) explain_network_scheduler = ReduceLROnPlateau(explain_network_optim, mode="max", factor=0.5, patience=2, verbose=True, threshold=1e-2) total_accuracy = 0.0 last_accuracy = 0.0 # Create some lists to save data loss_list = [] acc_list = [] episode_list = [] h_list = [] episode_list = [] for episode in range(EPISODE): explain_network.batch_size = BATCH_NUM_PER_CLASS # init dataset # sample_dataloader is to obtain previous samples for compare # batch_dataloader is to batch samples for training task = tg.MiniImagenetTask(metatrain_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=BATCH_NUM_PER_CLASS, split="test", shuffle=True, train_query_argue=True) # sample datas samples, sample_labels = sample_dataloader.__iter__().next( ) #(n*s)*3*84*84 batches, batch_labels = batch_dataloader.__iter__().next() # calculate features sample_features = feature_encoder( Variable(samples).cuda(GPU)) # (n*s)*64*19*19 sample_features_ext = sample_features.unsqueeze(0).repeat( BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) # [n*b,n*s,64,19,19] batch_features = feature_encoder( Variable(batches).cuda(GPU)) # (n*b)*64*19*19 batch_features_ext = batch_features.unsqueeze(1).repeat( 1, CLASS_NUM * SAMPLE_NUM_PER_CLASS, 1, 1, 1) # [n*b,n*s,64,19,19] distance_feature = middle_layer.forward( support_x=sample_features_ext, query_x=batch_features_ext) # [n*b,n*s,19*19] pre = explain_network(distance_feature, torch.cat( [sample_features_ext, batch_features_ext], dim=2)) # [n*b,n] # print(pre.size()) mse = nn.MSELoss().cuda(GPU) one_hot_labels = Variable( torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).scatter_(1, batch_labels.view(-1, 1), 1).cuda(GPU)) loss = mse(pre, one_hot_labels) feature_encoder_optim.zero_grad() explain_network_optim.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(feature_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm(explain_network.parameters(), 0.5) feature_encoder_optim.step() explain_network_optim.step() if (episode + 1) % 50 == 0: print("episode:", episode + 1, "loss", loss.item()) if (episode + 1) % 5000 == 0: print("val") val_accuracies = [] with torch.no_grad(): for i in range(TEST_EPISODE): total_rewards = 0 task = tg.MiniImagenetTask(metaval_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, 15) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 5 explain_network.batch_size = num_per_class val_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=False) sample_images, sample_labels = sample_dataloader.__iter__( ).next() for val_images, val_labels in val_dataloader: val_images, val_labels = val_images.cuda( GPU), val_labels.cuda(GPU) batch_size = val_labels.shape[0] sample_features = feature_encoder( Variable(sample_images).cuda( GPU)) # (n*s)*64*19*19 sample_features_ext = sample_features.unsqueeze( 0).repeat(num_per_class * CLASS_NUM, 1, 1, 1, 1) # [n*b,n*s,64,19,19] val_features = feature_encoder( Variable(val_images).cuda(GPU)) # (n*b)*64*19*19 val_features_ext = val_features.unsqueeze(1).repeat( 1, CLASS_NUM * SAMPLE_NUM_PER_CLASS, 1, 1, 1) # [n*b,n*s,64,19,19] distance_feature = middle_layer.forward( support_x=sample_features_ext, query_x=val_features_ext) # [n*b,n*s,19*19] # distance_feature = distance_feature.view([CLASS_NUM*num_per_class,CLASS_NUM*SAMPLE_NUM_PER_CLASS*9*9]) pre = explain_network( distance_feature, torch.cat([sample_features_ext, val_features_ext], dim=2)) # [n*b,n] # print(pre[0]) # print(test_labels[0]) _, predict_labels = torch.max(pre.data, 1) rewards = [ 1 if predict_labels[j] == val_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) val_accuracy = total_rewards / 1.0 / CLASS_NUM / 15 val_accuracies.append(val_accuracy) accuracy_val, h = mean_confidence_interval(val_accuracies) #迭代 feature_encoder_scheduler.step(accuracy_val) explain_network_scheduler.step(accuracy_val) if episode % 500 == 0: # test print("Testing...") accuracies = [] with torch.no_grad(): for i in range(TEST_EPISODE): total_rewards = 0 task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, 15) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) num_per_class = 5 explain_network.batch_size = num_per_class test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=False) sample_images, sample_labels = sample_dataloader.__iter__( ).next() for test_images, test_labels in test_dataloader: test_images, test_labels = test_images.cuda( GPU), test_labels.cuda(GPU) batch_size = test_labels.shape[0] sample_features = feature_encoder( Variable(sample_images).cuda( GPU)) # (n*s)*64*19*19 sample_features_ext = sample_features.unsqueeze( 0).repeat(num_per_class * CLASS_NUM, 1, 1, 1, 1) # [n*b,n*s,64,19,19] test_features = feature_encoder( Variable(test_images).cuda(GPU)) # (n*b)*64*19*19 test_features_ext = test_features.unsqueeze(1).repeat( 1, CLASS_NUM * SAMPLE_NUM_PER_CLASS, 1, 1, 1) # [n*b,n*s,64,19,19] distance_feature = middle_layer.forward( support_x=sample_features_ext, query_x=test_features_ext) # [n*b,n*s,19*19] # distance_feature = distance_feature.view([CLASS_NUM*num_per_class,CLASS_NUM*SAMPLE_NUM_PER_CLASS*9*9]) pre = explain_network( distance_feature, torch.cat([sample_features_ext, test_features_ext], dim=2)) # [n*b,n] _, predict_labels = torch.max(pre.data, 1) rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) accuracy = total_rewards / 1.0 / CLASS_NUM / 15 accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) loss_list, acc_list, h_list = save_data( loss_list, acc_list, h_list, loss, test_accuracy, h) episode_list.append(episode) print("test accuracy:", test_accuracy, "h:", h) if test_accuracy > last_accuracy: torch.save(feature_encoder.state_dict(), path + "feature_encoder.pkl") torch.save(explain_network.state_dict(), path + "explain_network.pkl") np.savetxt(path + "acc.txt", acc_list) np.savetxt(path + "eposide.txt", episode_list) np.savetxt(path + "h.txt", h_list) np.savetxt(path + "loss.txt", loss_list)
def main(): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # * Step 1: init data folders print("init data folders") # * Init character folders for dataset construction metatrain_character_folders, metatest_character_folders = tg.mini_imagenet_folders( ) # * Step 2: init neural networks print("init neural networks") feature_encoder = CNNEncoder() relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM) feature_encoder.train() relation_network.train() feature_encoder.apply(weights_init) relation_network.apply(weights_init) feature_encoder.to(device) relation_network.to(device) feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=100000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=100000, gamma=0.5) if os.path.exists( str("./models/omniglot_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_args.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_args.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/omniglot_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_args.pkl")): relation_network.load_state_dict( torch.load( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_args.pkl"))) print("load relation network success") # * Step 3: build graph print("Training...") last_accuracy = 0.0 for episode in range(EPISODE): # * init dataset # * sample_dataloader is to obtain previous samples for compare # * batch_dataloader is to batch samples for training degrees = random.choice([0, 90, 180, 270]) task = tg.MiniImagenetTask(metatrain_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=BATCH_NUM_PER_CLASS, split="test", shuffle=True) # * sample datas # samples, sample_labels = sample_dataloader.__iter__().next() # batches, batch_labels = batch_dataloader.__iter__().next() samples, sample_labels = next(iter(sample_dataloader)) batches, batch_labels = next(iter(batch_dataloader)) samples, sample_labels = samples.to(device), sample_labels.to(device) batches, batch_labels = batches.to(device), batch_labels.to(device) # * calculates features sample_features = feature_encoder(samples) sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) batch_features = feature_encoder(batches) # * calculate relations # * each batch sample link to every samples to calculate relations # * to form a 100 * 128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) batch_features_ext = batch_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) batch_features_ext = torch.transpose(batch_features_ext, 0, 1) relation_pairs = torch.cat((sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view(-1, CLASS_NUM) mse = nn.MSELoss() one_hot_labels = torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).to(device).scatter_( 1, batch_labels.view(-1, 1), 1) loss = mse(relations, one_hot_labels) feature_encoder.zero_grad() relation_network.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(feature_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm_(relation_network.parameters(), 0.5) feature_encoder_optim.step() relation_network_optim.step() feature_encoder_scheduler.step(episode) relation_network_scheduler.step(episode) if (episode + 1) % 100 == 0: print( f"episode : {episode+1}, loss : {loss.cpu().detach().numpy()}") if (episode + 1) % 1000 == 0: print("Testing...") total_reward = 0 for i in range(TEST_EPISODE): degrees = random.choice([0, 90, 180, 270]) task = tg.MiniImagenetTask(metatest_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, SAMPLE_NUM_PER_CLASS) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="test", shuffle=True) sample_images, sample_labels = next(iter(sample_dataloader)) test_images, test_labels = next(iter(test_dataloader)) sample_images, sample_labels = sample_images.to( device), sample_labels.to(device) test_images, test_labels = test_images.to( device), test_labels.to(device) # * calculate features sample_features = feature_encoder(sample_images) sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) test_features = feature_encoder(test_images) # * calculate relations # * each batch sample link to every samples to calculate relations # * to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( SAMPLE_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose(test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(CLASS_NUM * SAMPLE_NUM_PER_CLASS) ] total_reward += np.sum(rewards) test_accuracy = total_reward / ( 1.0 * CLASS_NUM * SAMPLE_NUM_PER_CLASS * TEST_EPISODE) print("test accuracy : ", test_accuracy) if test_accuracy > last_accuracy: # save networks torch.save( feature_encoder.state_dict(), str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_args.pkl")) torch.save( relation_network.state_dict(), str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot_args.pkl")) print("save networks for episode:", episode) last_accuracy = test_accuracy
def main(): # Step 1: init data folders print("init data folders") # init character folders for dataset construction metatrain_folders, metatest_folders = tg.mini_imagenet_folders() # Step 2: init neural networks print("init neural networks") feature_encoder = CNNEncoder() relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM) feature_encoder = nn.DataParallel(feature_encoder) relation_network = nn.DataParallel(relation_network) feature_encoder.apply(weights_init) relation_network.apply(weights_init) feature_encoder.cuda(GPU) relation_network.cuda(GPU) feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=100000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=100000, gamma=0.5) if os.path.exists( str("./models/miniimagenet_feature_encoder2" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder2" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/miniimagenet_relation_network2" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str("./models/miniimagenet_relation_network2" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") # Step 3: build graph print("Training...") last_accuracy = 0.0 ######################################################################################## year = datetime.datetime.now().year month = datetime.datetime.now().month day = datetime.datetime.now().day filename = "miniimagenet_train_oneshot_" + str(year) + '_' + str( month) + '_' + str(day) + ".txt" with open("models/" + filename, "w") as f: for episode in range(EPISODE): feature_encoder_scheduler.step(episode) relation_network_scheduler.step(episode) # init dataset # sample_dataloader is to obtain previous samples for compare # batch_dataloader is to batch samples for training task = tg.MiniImagenetTask(metatrain_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=BATCH_NUM_PER_CLASS, split="test", shuffle=True) # sample datas samples, sample_labels = sample_dataloader.__iter__().next() batches, batch_labels = batch_dataloader.__iter__().next() # print(samples.shape) #[75,3,84,84] # print(batches.shape) # calculate features sample_features = feature_encoder(Variable(samples).cuda(GPU)) # print(sample_features.shape) # torch.Size([5, 64, 19, 19]) batch_features = feature_encoder(Variable(batches).cuda(GPU)) # print(batch_features.shape) # torch.Size([75, 64, 19, 19]) # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) #torch.Size([75, 5, 64, 19, 19]) batch_features_ext = batch_features.unsqueeze(0).repeat( SAMPLE_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) #5 75 64 19 19 batch_features_ext = torch.transpose(batch_features_ext, 0, 1) #75 5 64 19 19 relation_pairs = torch.cat( (sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) # print(relation_pairs.shape) #375,128,19,19 relations = relation_network(relation_pairs).view( -1, CLASS_NUM * SAMPLE_NUM_PER_CLASS) print(relations.shape) mse = nn.MSELoss().cuda(GPU) one_hot_labels = Variable( torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).scatter_(1, batch_labels.view(-1, 1), 1)).cuda(GPU) loss = mse(relations, one_hot_labels) # training feature_encoder.zero_grad() relation_network.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(feature_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm(relation_network.parameters(), 0.5) feature_encoder_optim.step() relation_network_optim.step() if (episode + 1) % 100 == 0: print("episode:", episode + 1, "loss", loss.item()) newcontext = "episode: " + str(episode + 1) + " loss " + str( loss.item()) + '\n' f.writelines(newcontext) if episode % 5000 == 0: # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 counter = 0 task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, 1, 15) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=1, split="train", shuffle=False) num_per_class = 3 test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=True) sample_images, sample_labels = sample_dataloader.__iter__( ).next() for test_images, test_labels in test_dataloader: batch_size = test_labels.shape[0] # calculate features sample_features = feature_encoder( Variable(sample_images).cuda(GPU)) # 5x64 test_features = feature_encoder( Variable(test_images).cuda(GPU)) # 20x64 # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze( 0).repeat(batch_size, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose( test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j].cuda() == test_labels[j].cuda() else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) counter += batch_size accuracy = total_rewards / 1.0 / counter accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:", test_accuracy, "h:", h) newcontext = "episode: " + str( episode + 1) + "test accuracy: " + str(test_accuracy) + '\n' f.writelines(newcontext) if test_accuracy > last_accuracy: # save networks # torch.save(feature_encoder.state_dict(),str("./models/miniimagenet_feature_encoder2" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot.pkl")) # torch.save(relation_network.state_dict(),str("./models/miniimagenet_relation_network2"+ str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot.pkl")) print("save networks for episode:", episode) last_accuracy = test_accuracy
def main(): # Step 1: init data folders print("init data folders") # init character folders for dataset construction metatrain_folders, metatest_folders = tg.mini_imagenet_folders() # Step 2: init neural networks print("init neural networks") ### instantiation feature_encoder = CNNEncoder() relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM) ### use member function to load a module, which means to take some functions. feature_encoder.apply(weights_init) relation_network.apply(weights_init) feature_encoder.cuda(GPU) relation_network.cuda(GPU) feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=100000, gamma=0.5) relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE) relation_network_scheduler = StepLR(relation_network_optim, step_size=100000, gamma=0.5) if os.path.exists( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict( torch.load( str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): relation_network.load_state_dict( torch.load( str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load relation network success") # Step 3: build graph print("Training...") last_accuracy = 0.0 for episode in range(EPISODE): ### Sets the learning rate of each parameter group to the initial lr decayed by gamma every step_size epochs. feature_encoder_scheduler.step(episode) relation_network_scheduler.step(episode) # init dataset # sample_dataloader is to obtain previous samples for compare # batch_dataloader is to batch samples for training task = tg.MiniImagenetTask(metatrain_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=BATCH_NUM_PER_CLASS, split="test", shuffle=True) # sample datas samples, sample_labels = sample_dataloader.__iter__().next() batches, batch_labels = batch_dataloader.__iter__().next() # calculate features sample_features = feature_encoder( Variable(samples).cuda(GPU)) # 5x64*5*5 batch_features = feature_encoder( Variable(batches).cuda(GPU)) # 20x64*5*5 # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) batch_features_ext = batch_features.unsqueeze(0).repeat( SAMPLE_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) batch_features_ext = torch.transpose(batch_features_ext, 0, 1) relation_pairs = torch.cat((sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM * SAMPLE_NUM_PER_CLASS) mse = nn.MSELoss().cuda(GPU) one_hot_labels = Variable( torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).scatter_(1, batch_labels.view(-1, 1), 1)).cuda(GPU) loss = mse(relations, one_hot_labels) ### 变量的含义,根据论文设计的算法确定;in the hyper parameters section. ### 代码的由粗到细,模块化,数据流,shape;常用操作的组合,刚开始很需要耐心,弄清每个细节,这是代码部分的基础,如果想要建造一颗参天大树,就得从这里开始。 ### 关键部分的改进和实现。 # training feature_encoder.zero_grad() relation_network.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(feature_encoder.parameters(), 0.5) torch.nn.utils.clip_grad_norm(relation_network.parameters(), 0.5) feature_encoder_optim.step() relation_network_optim.step() if (episode + 1) % 100 == 0: print("episode:", episode + 1, "loss", loss.data[0]) if episode % 5000 == 0: # test print("Testing...") accuracies = [] for i in range(TEST_EPISODE): total_rewards = 0 counter = 0 task = tg.MiniImagenetTask(metatest_folders, CLASS_NUM, 1, 15) sample_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=1, split="train", shuffle=False) num_per_class = 3 test_dataloader = tg.get_mini_imagenet_data_loader( task, num_per_class=num_per_class, split="test", shuffle=True) sample_images, sample_labels = sample_dataloader.__iter__( ).next() for test_images, test_labels in test_dataloader: batch_size = test_labels.shape[0] # calculate features sample_features = feature_encoder( Variable(sample_images).cuda(GPU)) # 5x64 test_features = feature_encoder( Variable(test_images).cuda(GPU)) # 20x64 # calculate relations # each batch sample link to every samples to calculate relations # to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat( batch_size, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat( 1 * CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose( test_features_ext, 0, 1) relation_pairs = torch.cat( (sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) relations = relation_network(relation_pairs).view( -1, CLASS_NUM) _, predict_labels = torch.max(relations.data, 1) rewards = [ 1 if predict_labels[j] == test_labels[j] else 0 for j in range(batch_size) ] total_rewards += np.sum(rewards) counter += batch_size accuracy = total_rewards / 1.0 / counter accuracies.append(accuracy) test_accuracy, h = mean_confidence_interval(accuracies) print("test accuracy:", test_accuracy, "h:", h) if test_accuracy > last_accuracy: # save networks torch.save( feature_encoder.state_dict(), str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")) torch.save( relation_network.state_dict(), str("./models/miniimagenet_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")) print("save networks for episode:", episode) last_accuracy = test_accuracy
def main(): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # * Step 1: init data folders print("init data folders") # * Init character folders for dataset construction metatrain_character_folders, metatest_character_folders = tg.mini_imagenet_folders() # * Step 2: init neural networks print("init neural networks") feature_encoder = models.CNNEncoder() model = models.ActorCritic(FEATURE_DIM, RELATION_DIM, CLASS_NUM) #feature_encoder = torch.nn.DataParallel(feature_encoder) #actor = torch.nn.DataParallel(actor) #critic = torch.nn.DataParallel(critic) feature_encoder.train() model.train() feature_encoder.apply(models.weights_init) model.apply(models.weights_init) feature_encoder.to(device) model.to(device) cross_entropy = nn.CrossEntropyLoss() feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE) feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=10000, gamma=0.5) model_optim = torch.optim.Adam(model.parameters(), lr=2.5 * LEARNING_RATE) model_scheduler = StepLR(model_optim, step_size=10000, gamma=0.5) agent = ppoAgent.PPOAgent(GAMMA, ENTROPY_WEIGHT, CLASS_NUM, device) if os.path.exists(str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): feature_encoder.load_state_dict(torch.load(str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load feature encoder success") if os.path.exists(str("./models/miniimagenet_actor_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")): model.load_state_dict(torch.load(str("./models/miniimagenet_actor_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))) print("load model network success") # * Step 3: build graph print("Training...") loss_list = [] last_accuracy = 0.0 number_of_query_image = 15 clip_param = 0.1 for episode in range(EPISODE): if clip_param > 0 and clip_param % CLIP_DECREASE == 0: clip_param *= 0.5 #print(f"EPISODE : {episode}") losses = [] for meta_batch in range(META_BATCH_RANGE): meta_env_states_list = [] meta_env_labels_list = [] model_fast_weight = OrderedDict(model.named_parameters()) for inner_batch in range(INNER_BATCH_RANGE): # * Generate environment env_states_list = [] env_labels_list = [] inner_loss_list = [] for _ in range(ENV_LENGTH): task = tg.MiniImagenetTask(metatrain_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, number_of_query_image) sample_dataloader = tg.get_mini_imagenet_data_loader(task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader(task, num_per_class=5, split="test", shuffle=True) samples, sample_labels = next(iter(sample_dataloader)) samples, sample_labels = samples.to(device), sample_labels.to(device) for batches, batch_labels in batch_dataloader: batches, batch_labels = batches.to(device), batch_labels.to(device) inner_sample_features = feature_encoder(samples) inner_sample_features = inner_sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) inner_sample_features = torch.sum(inner_sample_features, 1).squeeze(1) inner_batch_features = feature_encoder(batches) inner_sample_feature_ext = inner_sample_features.unsqueeze(0).repeat(5 * CLASS_NUM, 1, 1, 1, 1) inner_batch_features_ext = inner_batch_features.unsqueeze(0).repeat(CLASS_NUM, 1, 1, 1, 1) inner_batch_features_ext = torch.transpose(inner_batch_features_ext, 0, 1) inner_relation_pairs = torch.cat((inner_sample_feature_ext, inner_batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) env_states_list.append(inner_relation_pairs) env_labels_list.append(batch_labels) inner_env = ppoAgent.env(env_states_list, env_labels_list) agent.train(inner_env, model, loss_list=inner_loss_list) inner_loss = torch.stack(inner_loss_list).mean() inner_gradients = torch.autograd.grad(inner_loss.mean(), model_fast_weight.values(), create_graph=True, allow_unused=True) model_fast_weight = OrderedDict( (name, param - INNER_LR * (0 if grad is None else grad)) for ((name, param), grad) in zip(model_fast_weight.items(), inner_gradients) ) model.weight = model_fast_weight # * Generate env for meta update for _ in range(META_ENV_LENGTH): # * init dataset # * sample_dataloader is to obtain previous samples for compare # * batch_dataloader is to batch samples for training task = tg.MiniImagenetTask(metatrain_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, number_of_query_image) sample_dataloader = tg.get_mini_imagenet_data_loader(task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) batch_dataloader = tg.get_mini_imagenet_data_loader(task, num_per_class=number_of_query_image, split="test", shuffle=True) # * num_per_class : number of query images # * sample datas samples, sample_labels = next(iter(sample_dataloader)) batches, batch_labels = next(iter(batch_dataloader)) samples, sample_labels = samples.to(device), sample_labels.to(device) batches, batch_labels = batches.to(device), batch_labels.to(device) # * calculates features #feature_encoder.weight = feature_fast_weights sample_features = feature_encoder(samples) sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) batch_features = feature_encoder(batches) # * calculate relations # * each batch sample link to every samples to calculate relations # * to form a 100 * 128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat(number_of_query_image * CLASS_NUM, 1, 1, 1, 1) batch_features_ext = batch_features.unsqueeze(0).repeat(CLASS_NUM, 1, 1, 1, 1) batch_features_ext = torch.transpose(batch_features_ext, 0, 1) relation_pairs = torch.cat((sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) meta_env_states_list.append(relation_pairs) meta_env_labels_list.append(batch_labels) meta_env = ppoAgent.env(meta_env_states_list, meta_env_labels_list) agent.train(meta_env, model, loss_list=losses, clip_param=clip_param) feature_encoder_optim.zero_grad() model_optim.zero_grad() torch.nn.utils.clip_grad_norm_(feature_encoder.parameters(), 0.5) meta_batch_loss = torch.stack(losses).mean() meta_batch_loss.backward() feature_encoder_optim.step() model_optim.step() feature_encoder_scheduler.step() model_scheduler.step() mean_loss = None if (episode + 1) % 100 == 0: mean_loss = meta_batch_loss.cpu().detach().numpy() print(f"episode : {episode+1}, meta_loss : {mean_loss:.4f}") loss_list.append(mean_loss) if (episode + 1) % 500 == 0: print("Testing...") total_reward = 0 total_test_samples = 0 for i in range(TEST_EPISODE): # * Generate env env_states_list = [] env_labels_list = [] number_of_query_image = 10 task = tg.MiniImagenetTask(metatest_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, number_of_query_image) sample_dataloader = tg.get_mini_imagenet_data_loader(task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False) test_dataloader = tg.get_mini_imagenet_data_loader(task, num_per_class=number_of_query_image, split="test", shuffle=True) sample_images, sample_labels = next(iter(sample_dataloader)) test_images, test_labels = next(iter(test_dataloader)) total_test_samples += len(test_labels) sample_images, sample_labels = sample_images.to(device), sample_labels.to(device) test_images, test_labels = test_images.to(device), test_labels.to(device) # * calculate features sample_features = feature_encoder(sample_images) sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 19, 19) sample_features = torch.sum(sample_features, 1).squeeze(1) test_features = feature_encoder(test_images) # * calculate relations # * each batch sample link to every samples to calculate relations # * to form a 100x128 matrix for relation network sample_features_ext = sample_features.unsqueeze(0).repeat(number_of_query_image * CLASS_NUM, 1, 1, 1, 1) test_features_ext = test_features.unsqueeze(0).repeat(CLASS_NUM, 1, 1, 1, 1) test_features_ext = torch.transpose(test_features_ext, 0, 1) relation_pairs = torch.cat((sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 19, 19) env_states_list.append(relation_pairs) env_labels_list.append(test_labels) test_env = ppoAgent.env(env_states_list, env_labels_list) rewards = agent.test(test_env, model) total_reward += rewards test_accuracy = total_reward / (1.0 * total_test_samples) print(f'mean loss : {mean_loss}') print("test accuracy : ", test_accuracy) writer.add_scalar('1.loss', mean_loss, episode + 1) writer.add_scalar('4.test accuracy', test_accuracy, episode + 1) loss_list = [] if test_accuracy > last_accuracy: # save networks torch.save( feature_encoder.state_dict(), str("./models/miniimagenet_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl") ) torch.save( model.state_dict(), str("./models/miniimagenet_actor_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl") ) print("save networks for episode:", episode) last_accuracy = test_accuracy