def main():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# * Step 1: init data folders
print("init data folders")
# * Init character folders for dataset construction
metatrain_character_folders, metatest_character_folders = tg.omniglot_character_folders(
)
# * Step 2: init neural networks
print("init neural networks")
feature_encoder = CNNEncoder()
relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM)
feature_encoder.apply(weights_init)
relation_network.apply(weights_init)
feature_encoder.to(device)
relation_network.to(device)
feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(),
lr=LEARNING_RATE)
feature_encoder_scheduler = StepLR(feature_encoder_optim,
step_size=100000,
gamma=0.5)
relation_network_optim = torch.optim.Adam(relation_network.parameters(),
lr=LEARNING_RATE)
relation_network_scheduler = StepLR(relation_network_optim,
step_size=100000,
gamma=0.5)
if os.path.exists(
str("./models/omniglot_feature_encoder_" + str(CLASS_NUM) +
"way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")):
feature_encoder.load_state_dict(
torch.load(
str("./models/omniglot_feature_encoder_" + str(CLASS_NUM) +
"way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")))
print("load feature encoder success")
if os.path.exists(
str("./models/omniglot_relation_network_" + str(CLASS_NUM) +
"way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")):
relation_network.load_state_dict(
torch.load(
str("./models/omniglot_relation_network_" + str(CLASS_NUM) +
"way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")))
print("load relation network success")
# * Step 3: build graph
print("Training...")
last_accuracy = 0.0
for episode in range(EPISODE):
# * init dataset
# * sample_dataloader is to obtain previous samples for compare
# * batch_dataloader is to batch samples for training
degrees = random.choice([0, 90, 180, 270])
task = tg.OmniglotTask(metatrain_character_folders, CLASS_NUM,
SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS)
sample_dataloader = tg.get_data_loader(
task,
num_per_class=SAMPLE_NUM_PER_CLASS,
split="train",
shuffle=False,
rotation=degrees)
batch_dataloader = tg.get_data_loader(
task,
num_per_class=BATCH_NUM_PER_CLASS,
split="test",
shuffle=True,
rotation=degrees)
# * sample datas
# samples, sample_labels = sample_dataloader.__iter__().next()
# batches, batch_labels = batch_dataloader.__iter__().next()
samples, sample_labels = next(iter(sample_dataloader))
batches, batch_labels = next(iter(batch_dataloader))
samples, sample_labels = samples.to(device), sample_labels.to(device)
batches, batch_labels = batches.to(device), batch_labels.to(device)
# * calculates features
sample_features = feature_encoder(samples)
sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS,
FEATURE_DIM, 5, 5)
sample_features = torch.sum(sample_features, 1).squeeze(1)
batch_features = feature_encoder(batches)
# * calculate relations
# * each batch sample link to every samples to calculate relations
# * to form a 100 * 128 matrix for relation network
sample_features_ext = sample_features.unsqueeze(0).repeat(
BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1)
batch_features_ext = batch_features.unsqueeze(0).repeat(
CLASS_NUM, 1, 1, 1, 1)
batch_features_ext = torch.transpose(batch_features_ext, 0, 1)
relation_pairs = torch.cat((sample_features_ext, batch_features_ext),
2).view(-1, FEATURE_DIM * 2, 5, 5)
relations = relation_network(relation_pairs).view(-1, CLASS_NUM)
mse = nn.MSELoss()
one_hot_labels = torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM,
CLASS_NUM).to(device).scatter_(
1, batch_labels.view(-1, 1), 1)
loss = mse(relations, one_hot_labels)
feature_encoder.zero_grad()
relation_network.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(feature_encoder.parameters(), 0.5)
torch.nn.utils.clip_grad_norm_(relation_network.parameters(), 0.5)
feature_encoder_optim.step()
relation_network_optim.step()
feature_encoder_scheduler.step(episode)
relation_network_scheduler.step(episode)
if (episode + 1) % 100 == 0:
print(
f"episode : {episode+1}, loss : {loss.cpu().detach().numpy()}")
if (episode + 1) % 500 == 0:
print("Testing...")
total_reward = 0
for i in range(TEST_EPISODE):
degrees = random.choice([0, 90, 180, 270])
task = tg.OmniglotTask(metatest_character_folders, CLASS_NUM,
SAMPLE_NUM_PER_CLASS,
SAMPLE_NUM_PER_CLASS)
sample_dataloader = tg.get_data_loader(
task,
num_per_class=SAMPLE_NUM_PER_CLASS,
split="train",
shuffle=False,
rotation=degrees)
test_dataloader = tg.get_data_loader(
task,
num_per_class=SAMPLE_NUM_PER_CLASS,
split="test",
shuffle=True,
rotation=degrees)
sample_images, sample_labels = next(iter(sample_dataloader))
test_images, test_labels = next(iter(test_dataloader))
sample_images, sample_labels = sample_images.to(
device), sample_labels.to(device)
test_images, test_labels = test_images.to(
device), test_labels.to(device)
# * calculate features
sample_features = feature_encoder(sample_images)
sample_features = sample_features.view(CLASS_NUM,
SAMPLE_NUM_PER_CLASS,
FEATURE_DIM, 5, 5)
sample_features = torch.sum(sample_features, 1).squeeze(1)
test_features = feature_encoder(test_images)
# * calculate relations
# * each batch sample link to every samples to calculate relations
# * to form a 100x128 matrix for relation network
sample_features_ext = sample_features.unsqueeze(0).repeat(
SAMPLE_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1)
test_features_ext = test_features.unsqueeze(0).repeat(
CLASS_NUM, 1, 1, 1, 1)
test_features_ext = torch.transpose(test_features_ext, 0, 1)
relation_pairs = torch.cat(
(sample_features_ext, test_features_ext),
2).view(-1, FEATURE_DIM * 2, 5, 5)
relations = relation_network(relation_pairs).view(
-1, CLASS_NUM)
_, predict_labels = torch.max(relations.data, 1)
rewards = [
1 if predict_labels[j] == test_labels[j] else 0
for j in range(CLASS_NUM * SAMPLE_NUM_PER_CLASS)
]
total_reward += np.sum(rewards)
test_accuracy = total_reward / (
1.0 * CLASS_NUM * SAMPLE_NUM_PER_CLASS * TEST_EPISODE)
print("test accuracy : ", test_accuracy)
if test_accuracy > last_accuracy:
# save networks
torch.save(
feature_encoder.state_dict(),
str("./models/omniglot_feature_encoder_" + str(CLASS_NUM) +
"way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"))
torch.save(
relation_network.state_dict(),
str("./models/omniglot_relation_network_" +
str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) +
"shot.pkl"))
print("save networks for episode:", episode)
last_accuracy = test_accuracy
def main():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# * Step 1: init data folders
print("init data folders")
# * Init character folders for dataset construction
metatrain_character_folders, metatest_character_folders = tg.omniglot_character_folders()
# * Step 2: init neural networks
print("init neural networks")
feature_encoder = CNNEncoder()
# RFT = RandomForestClassifier(n_estimators=100, random_state=1, n_jobs=-1, warm_start=True)
# * It decrease bias and increase variance
'''
When RFT has high bias and low variance, then RFT dominate accuracy when it is combined with relation network
'''
# RFT = RandomForestClassifier(n_estimators=100, random_state=1, min_samples_leaf=5, n_jobs=-1, warm_start=True)
# * It increase bias and decrease variance
RFT = RandomForestClassifier(n_estimators=100, random_state=1, n_jobs=-1, warm_start=True)
# RFT = RandomForestClassifier(n_estimators=100, random_state=1, n_jobs=-1)
# RFT = RandomForestClassifier(n_estimators=100, random_state=1, min_samples_leaf=5, n_jobs=-1)
relation_network = RelationNetwork(FEATURE_DIM, RELATION_DIM)
feature_encoder.apply(weights_init)
relation_network.apply(weights_init)
feature_encoder.to(device)
relation_network.to(device)
# mse = nn.MSELoss()
cross_entropy = nn.CrossEntropyLoss()
feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(), lr=LEARNING_RATE)
feature_encoder_scheduler = StepLR(feature_encoder_optim, step_size=100000, gamma=0.5)
relation_network_optim = torch.optim.Adam(relation_network.parameters(), lr=LEARNING_RATE)
relation_network_scheduler = StepLR(relation_network_optim, step_size=100000, gamma=0.5)
if os.path.exists(str("./models/omniglot_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")):
feature_encoder.load_state_dict(torch.load(str("./models/omniglot_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")))
print("load feature encoder success")
if os.path.exists(str("./models/omniglot_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")):
RFT = pickle.load(open(str("./models/omniglot_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"), 'rb'))
print("load random forest success")
if os.path.exists(str("./models/omniglot_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")):
relation_network.load_state_dict(torch.load(str("./models/omniglot_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")))
print("load relation network success")
# * Step 3: build graph
print("Training...")
last_accuracy = 0.0
last_RFT_accuracy = 0
test_RFT_accuracy = 0
# embedding_loss_list = []
RFT_loss_list = []
relation_loss_list = []
loss_list = []
for episode in range(EPISODE):
# * init dataset
# * sample_dataloader is to obtain previous samples for compare
# * batch_dataloader is to batch samples for training
degrees = random.choice([0, 90, 180, 270])
task = tg.OmniglotTask(metatrain_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, BATCH_NUM_PER_CLASS)
sample_dataloader = tg.get_data_loader(task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False, rotation=degrees)
batch_dataloader = tg.get_data_loader(task, num_per_class=BATCH_NUM_PER_CLASS, split="test", shuffle=True, rotation=degrees)
# * sample datas
# samples, sample_labels = sample_dataloader.__iter__().next()
# batches, batch_labels = batch_dataloader.__iter__().next()
samples, sample_labels = next(iter(sample_dataloader))
batches, batch_labels = next(iter(batch_dataloader))
# RFT_samples, RFT_sample_labels = samples, sample_labels
RFT_batches, RFT_batch_labels = batches, batch_labels
samples, sample_labels = samples.to(device), sample_labels.to(device)
batches, batch_labels = batches.to(device), batch_labels.to(device)
# one_hot_sample_labels = torch.zeros(SAMPLE_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).to(device).scatter_(1, sample_labels.view(-1, 1), 1)
# * calculates features
linear, sample_features = feature_encoder(samples)
# RFT_sample_features = sample_features.detach().cpu().reshape(RFT_samples.shape[0], -1)
sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 5, 5)
sample_features = torch.sum(sample_features, 1).squeeze(1)
_, batch_features = feature_encoder(batches)
# RFT_batch_features = batch_features.detach().cpu().reshape(RFT_batches.shape[0], -1)
# embedding_loss = mse(linear, one_hot_sample_labels)
# embedding_loss = cross_entropy(linear, sample_labels)
# * calculate relations
# * each batch sample link to every samples to calculate relations
# * to form a 100 * 128 matrix for relation network
sample_features_ext = sample_features.unsqueeze(0).repeat(BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1)
batch_features_ext = batch_features.unsqueeze(0).repeat(CLASS_NUM, 1, 1, 1, 1)
batch_features_ext = torch.transpose(batch_features_ext, 0, 1)
relation_pairs = torch.cat((sample_features_ext, batch_features_ext), 2).view(-1, FEATURE_DIM * 2, 5, 5)
relations = relation_network(relation_pairs).view(-1, CLASS_NUM)
if episode % 10 == 0:
RFT.fit(relations.detach().cpu(), RFT_batch_labels)
RFT.n_estimators += 1
RFT_prob = torch.tensor(RFT.predict_proba(relations.detach().cpu())).to(device)
_, RFT_labels = torch.max(RFT_prob, 1)
RFT_loss = cross_entropy(relations, RFT_labels) * 0.5
# one_hot_labels = torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM, CLASS_NUM).to(device).scatter_(1, batch_labels.view(-1, 1), 1)
# loss = mse(relations, one_hot_labels)
# soft_voting = torch.softmax(relations, dim=1) + torch.tensor(RFT.predict_proba(relations.detach().cpu())).to(device)
# relation_loss = cross_entropy(relations, batch_labels)
relation_loss = cross_entropy(relations, batch_labels)
# loss = embedding_loss + relation_loss
# embedding_loss.detach()
loss = relation_loss + RFT_loss
feature_encoder_optim.zero_grad()
relation_network_optim.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(feature_encoder.parameters(), 0.5)
torch.nn.utils.clip_grad_norm_(relation_network.parameters(), 0.5)
feature_encoder_optim.step()
relation_network_optim.step()
feature_encoder_scheduler.step(episode)
relation_network_scheduler.step(episode)
if (episode + 1) % 100 == 0:
print(f"episode : {episode+1}, loss : {loss.cpu().detach().numpy()}")
loss_list.append(loss.cpu().detach().numpy())
RFT_loss_list.append(RFT_loss.cpu().detach().numpy())
relation_loss_list.append(relation_loss.cpu().detach().numpy())
if (episode + 1) % 500 == 0:
print("Testing...")
total_reward = 0
for i in range(TEST_EPISODE):
degrees = random.choice([0, 90, 180, 270])
task = tg.OmniglotTask(metatest_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, SAMPLE_NUM_PER_CLASS)
sample_dataloader = tg.get_data_loader(task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False, rotation=degrees)
test_dataloader = tg.get_data_loader(task, num_per_class=SAMPLE_NUM_PER_CLASS, split="test", shuffle=True, rotation=degrees)
sample_images, sample_labels = next(iter(sample_dataloader))
test_images, test_labels = next(iter(test_dataloader))
sample_images, sample_labels = sample_images.to(device), sample_labels.to(device)
test_images, test_labels = test_images.to(device), test_labels.to(device)
# * calculate features
_, sample_features = feature_encoder(sample_images)
sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 5, 5)
sample_features = torch.sum(sample_features, 1).squeeze(1)
_, test_features = feature_encoder(test_images)
# * calculate relations
# * each batch sample link to every samples to calculate relations
# * to form a 100x128 matrix for relation network
sample_features_ext = sample_features.unsqueeze(0).repeat(SAMPLE_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1)
test_features_ext = test_features.unsqueeze(0).repeat(CLASS_NUM, 1, 1, 1, 1)
test_features_ext = torch.transpose(test_features_ext, 0, 1)
relation_pairs = torch.cat((sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 5, 5)
relations = relation_network(relation_pairs).view(-1, CLASS_NUM)
_, predict_labels = torch.max(relations.data, 1)
rewards = [1 if predict_labels[j] == test_labels[j] else 0 for j in range(CLASS_NUM * SAMPLE_NUM_PER_CLASS)]
total_reward += np.sum(rewards)
if i % 200 == 0:
RFT_predict = RFT.predict(relations.detach().cpu())
assert RFT_predict.shape == test_labels.detach().cpu().shape
print(accuracy_score(RFT_predict, test_labels.detach().cpu()))
test_RFT_accuracy += accuracy_score(RFT_predict, test_labels.detach().cpu())
test_accuracy = total_reward / (1.0 * CLASS_NUM * SAMPLE_NUM_PER_CLASS * TEST_EPISODE)
test_RFT_accuracy /= (TEST_EPISODE // 200)
print("test accuracy : ", test_accuracy)
print(f"{test_RFT_accuracy:.3f} %")
mean_loss = np.mean(loss_list)
mean_RFT_loss = np.mean(RFT_loss_list)
mean_relation_loss = np.mean(relation_loss_list)
print(f'mean loss : {mean_loss}')
print(f'RFT loss : {mean_RFT_loss}')
# writer.add_scalar('1.embedding loss', mean_embedding_loss, episode + 1)
writer.add_scalar('1.RFT loss', mean_RFT_loss, episode + 1)
writer.add_scalar('RFT_accuracy', test_RFT_accuracy, episode + 1)
writer.add_scalar('2.relation loss', mean_relation_loss, episode + 1)
writer.add_scalar('loss', mean_loss, episode + 1)
writer.add_scalar('test accuracy', test_accuracy, episode + 1)
loss_list = []
# embedding_loss_list = []
relation_loss_list = []
RFT_loss_list = []
if test_RFT_accuracy > last_RFT_accuracy:
pickle.dump(RFT, open(str("./models/omniglot_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"), 'wb'))
last_RFT_accuracy = test_RFT_accuracy
test_RFT_accuracy = 0
if test_accuracy > last_accuracy:
# save networks
torch.save(
feature_encoder.state_dict(),
str("./models/omniglot_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")
)
torch.save(
relation_network.state_dict(),
str("./models/omniglot_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")
)
print("save networks for episode:", episode)
last_accuracy = test_accuracy
def main():
# * Step 1: init data folders
print("init data folders")
# * init character folders for dataset construction
metartrain_character_folders, metatest_character_folders = tg.omniglot_character_folders()
# * Step 2: init neural networks
print("init neural networks")
feature_encoder = ot.CNNEncoder().to(device)
RFT = RandomForestClassifier(n_estimators=100, random_state=1, warm_start=True)
relation_network = ot.RelationNetwork(FEATURE_DIM, RELATION_DIM).to(device)
feature_encoder.eval()
relation_network.eval()
if os.path.exists(str("./models/omniglot_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")):
feature_encoder.load_state_dict(torch.load(str("./models/omniglot_feature_encoder_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")))
print("load feature encoder success")
if os.path.exists(str("./models/omniglot_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")):
RFT = pickle.load(open(str("./models/omniglot_random_forest_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl"), 'rb'))
print("load random forest success")
if os.path.exists(str("./models/omniglot_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")):
relation_network.load_state_dict(torch.load(str("./models/omniglot_relation_network_" + str(CLASS_NUM) + "way_" + str(SAMPLE_NUM_PER_CLASS) + "shot.pkl")))
print("load relation network success")
RFT_total_accuracy = 0.0
total_accuracy = 0.0
soft_voting_total_accuracy = 0.0
for episode in range(1):
# * test
print("Testing...")
total_rewards = 0
soft_voting_total_rewards = 0
accuracies = []
RFT_accuracies = []
soft_voting_accuracies = []
for i in range(100):
degrees = random.choice([0, 90, 180, 270])
task = tg.OmniglotTask(metatest_character_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS, SAMPLE_NUM_PER_CLASS)
sample_dataloader = tg.get_data_loader(task, num_per_class=SAMPLE_NUM_PER_CLASS, split="train", shuffle=False, rotation=degrees)
test_dataloader = tg.get_data_loader(task, num_per_class=SAMPLE_NUM_PER_CLASS, split="test", shuffle=True, rotation=degrees)
sample_images, sample_labels = next(iter(sample_dataloader))
test_images, test_labels = next(iter(test_dataloader))
RFT_samples, RFT_sample_labels = sample_images, sample_labels
RFT_test, RFT_test_labels = test_images, test_labels
sample_images, sample_labels = sample_images.to(device), sample_labels.to(device)
test_images, test_labels = test_images.to(device), test_labels.to(device)
# * Calculate features
_, sample_features = feature_encoder(sample_images)
sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 5, 5)
sample_features = torch.sum(sample_features, 1).squeeze(1)
_, test_features = feature_encoder(test_images)
sample_features_ext = sample_features.unsqueeze(0).repeat(SAMPLE_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1)
test_features_ext = test_features.unsqueeze(0).repeat(CLASS_NUM, 1, 1, 1, 1)
test_features_ext = torch.transpose(test_features_ext, 0, 1)
relation_pairs = torch.cat((sample_features_ext, test_features_ext), 2).view(-1, FEATURE_DIM * 2, 5, 5)
relations = relation_network(relation_pairs).view(-1, CLASS_NUM)
RFT_predict = RFT.predict(relations.detach().cpu())
assert RFT_predict.shape == test_labels.detach().cpu().shape
RFT_score = accuracy_score(RFT_predict, test_labels.detach().cpu())
_, predict_labels = torch.max(relations.data, 1)
rewards = [1 if predict_labels[j] == test_labels[j] else 0 for j in range(CLASS_NUM * SAMPLE_NUM_PER_CLASS)]
total_rewards += np.sum(rewards)
accuracy = np.sum(rewards) / (1.0 * CLASS_NUM * SAMPLE_NUM_PER_CLASS)
RFT_prob = RFT.predict_proba(relations.detach().cpu())
relation_prob = torch.softmax(relations.data, dim=1)
RFT_prob_tensor = torch.tensor(RFT_prob).to(device)
# soft_voting = (RFT_prob_tensor + relation_prob) / 2
soft_voting = (RFT_prob_tensor / relation_prob)
_, soft_voting_predicted_labels = torch.max(soft_voting, 1)
'''
for i in range(len(RFT_predict)):
if RFT_predict[i] != predict_labels[i]:
print(f'RFT_prob : {RFT_prob[i]}, label : {RFT_predict[i]}, answer : {test_labels[i]}')
print(f'relation_prob : {relation_prob[i].detach().cpu().numpy()}, label : {predict_labels[i]}, answer : {test_labels[i]}')
print(f'soft_voting : {soft_voting[i].detach().cpu().numpy()}, label : {soft_voting_predicted_labels[i]}, answer : {test_labels[i]}')
print("----------------------------------")
'''
'''
print(relations.data)
print(RFT_prob)
print(relation_prob)
assert 1 == 2
'''
soft_voting_rewards = [1 if soft_voting_predicted_labels[j] == test_labels[j] else 0 for j in range(CLASS_NUM * SAMPLE_NUM_PER_CLASS)]
soft_voting_total_rewards += np.sum(soft_voting_rewards)
soft_voting_accuracy = np.sum(soft_voting_rewards) / (1.0 * CLASS_NUM * SAMPLE_NUM_PER_CLASS)
print(f"{i+1}th RFT accuracy : {RFT_score}, CNN accuracy : {accuracy}, combined_accuracy : {soft_voting_accuracy}")
RFT_accuracies.append(RFT_score)
accuracies.append(accuracy)
soft_voting_accuracies.append(soft_voting_accuracy)
RFT_test_accuracy, RFT_h = mean_confidence_interval(RFT_accuracies)
test_accuracy, h = mean_confidence_interval(accuracies)
soft_voting_test_accuracy, soft_voting_h = mean_confidence_interval(soft_voting_accuracies)
print(f'RFT_test_accuracy accuracy : {RFT_test_accuracy:.4f}, h : {RFT_h:.4f}')
print(f'test accuracy : {test_accuracy:.4f}, h : {h:.4f}')
print(f'test soft_voting_test_accuracy : {soft_voting_test_accuracy:.4f}, h : {soft_voting_h:.4f}')
RFT_total_accuracy += RFT_test_accuracy
total_accuracy += test_accuracy
soft_voting_total_accuracy += soft_voting_test_accuracy
print(f"average RFT_total_accuracy : {RFT_total_accuracy :.4f}")
print(f"average accuracy : {total_accuracy :.4f}")
print(f"soft_voting_total_accuracy : {soft_voting_total_accuracy :.4f}")