def run_best_params(opt):
best_params_dir = get_best_params_dir(opt)
with open(best_params_dir + '/params.json') as f:
best_params = json.loads(f.read())
# allow params specified at the cmd line to override
best_params_ret = {**best_params, **opt}
# the exception is number of epochs as we want to use more here than we would for hyperparameter tuning.
best_params_ret['epoch'] = opt['epoch']
print("Running with parameters {}".format(best_params_ret))
data_dir = os.path.abspath("../data")
reporter = CLIReporter(metric_columns=[
"accuracy", "loss", "test_acc", "train_acc", "best_time", "best_epoch",
"training_iteration"
])
if opt['name'] is None:
name = opt['folder'] + '_test'
else:
name = opt['name']
result = tune.run(
partial(train_ray_int, data_dir=data_dir),
name=name,
resources_per_trial={
"cpu": opt['cpus'],
"gpu": opt['gpus']
},
search_alg=None,
keep_checkpoints_num=3,
checkpoint_score_attr='accuracy',
config=best_params_ret,
num_samples=opt['reps']
if opt["num_splits"] == 0 else opt["num_splits"] * opt["reps"],
scheduler=None,
max_failures=
1, # early stop solver can't recover from failure as it doesn't own m2.
local_dir='../ray_tune',
progress_reporter=reporter,
raise_on_failed_trial=False)
df = result.dataframe(metric=opt['metric'],
mode="max").sort_values(opt['metric'],
ascending=False)
try:
df.to_csv('../ray_results/{}_{}.csv'.format(
name, time.strftime("%Y%m%d-%H%M%S")))
except:
pass
print(df[['accuracy', 'test_acc', 'train_acc', 'best_time', 'best_epoch']])
test_accs = df['test_acc'].values
print("test accuracy {}".format(test_accs))
log = "mean test {:04f}, test std {:04f}, test sem {:04f}, test 95% conf {:04f}"
print(
log.format(test_accs.mean(), np.std(test_accs), get_sem(test_accs),
mean_confidence_interval(test_accs)))
def analyse_disc_trees(lang_stats, analysis_dir):
ranges = [(0, 0.5), (0.5, 1), (0, 0.25), (0.25, 0.5), (0.5, 0.75),
(0.75, 1)]
# (0, 0.125), (0.125, 0.25), (0.25, 0.375), (0.375, 0.5)]
most_commons = {0: {}, 1: {}}
for range in ranges:
for channel in most_commons.keys():
most_commons[channel][range] = []
channel_sizes = {}
# most_commons = {}
for run_stats in lang_stats.values():
words_used = {}
for agent_stats in run_stats.values():
for tree in agent_stats['discriminators'][0].trees:
# Calculate tree size
channel = tree.root.channel
if channel not in channel_sizes:
channel_sizes[channel] = []
channel_sizes[channel].append(get_tree_size(tree))
# Calculate words used for every range
if channel not in words_used:
words_used[channel] = {}
for range in ranges:
words_used[channel][range] = {}
for range in ranges:
categoriser = Categoriser(range, channel)
word = agent_stats['memories'][0].get_form(categoriser)
if word is not None:
if word not in words_used[channel][range]:
words_used[channel][range][word] = 0
words_used[channel][range][word] += 1
for channel, ranges in most_commons.items():
for range in ranges:
if len(words_used[channel][range].keys()) > 0:
most_common = max(words_used[channel][range].values())
else:
most_common = 0
most_commons[channel][range].append(most_common)
for channel, ranges in most_commons.items():
print('Channel {}'.format(channel))
for range, counts in ranges.items():
mean, interval = mean_confidence_interval(counts)
print('{}: {}, +-{}'.format(range, mean, interval))
print()
for channel, sizes in channel_sizes.items():
avg_size = sum(sizes) / len(sizes)
print('Channel {} avg size: {}'.format(channel, avg_size))
print('Channel {} min size: {}'.format(channel, min(sizes)))
def test_loop(self, test_loader):
accuracies = []
with torch.no_grad():
for episode, (sample_images,
query_images) in tqdm(enumerate(test_loader)):
relations = self.forward(sample_images, query_images)
relations = (relations[0] + relations[1]) / 2
predicted_results = relations.data.topk(
k=1, dim=1)[1].squeeze().cpu().numpy()
ground_truth = np.repeat(range(self.config.n_way),
self.config.n_query_test)
correct_num = np.sum(predicted_results == ground_truth)
query_num = len(predicted_results)
accuracies.append(correct_num / query_num * 100)
acc, h = utils.mean_confidence_interval(accuracies)
return acc, h
def plot_learning_curve(data, prefix):
"""
Plot a regular learning curve with confidence intervals.
:param data: Result of multiple runs of the algorithm.
:param prefix: string that will be used as the filename of the generated graph
"""
mean, lower_bound, upper_bound = utils.mean_confidence_interval(data)
x_lim = len(data[0])
plt.plot(mean, color='red')
plt.fill_between(range(x_lim),
lower_bound,
upper_bound,
color='red',
alpha=0.4)
plt.xlabel("Episode")
plt.ylabel("Avg. number of steps")
plt.savefig(prefix + 'learning_curve.png')
plt.clf()
def main():
# i3d == the I3D network
encoder = I3D(in_channels=3)
encoder = nn.DataParallel(encoder)
# rn == the relation network
rn = RN(FEATURE_DIM,RELATION_DIM)
# Move the model to GPU
encoder.to(device)
rn.to(device)
# Define Optimizer
encoder_optim = torch.optim.Adam(encoder.parameters(),lr=LEARNING_RATE)
rn_optim = torch.optim.Adam(rn.parameters(),lr=LEARNING_RATE)
# Define Scheduler
encoder_scheduler = StepLR(encoder_optim,step_size=100000,gamma=0.5)
rn_scheduler = StepLR(rn_optim,step_size=100000,gamma=0.5)
# Load Saved Model
if encoder_saved_model != "":
encoder.load_state_dict(torch.load(encoder_saved_model))
if rn_saved_model != "":
rn.load_state_dict(torch.load(rn_saved_model))
max_accuracy = 0
for episode in range(EPISODE):
print("Train_Epi[{}] Pres_Accu = {}".format(episode, max_accuracy), end="\t")
# Update "step" for scheduler
rn_scheduler.step(episode)
encoder_scheduler.step(episode)
# Setup Data #TODO
haaDataset = dataset.HAADataset(DATA_FOLDER, "train", CLASS_NUM, SAMPLE_NUM_PER_CLASS, NUM_INST, NUM_FRAME_PER_VIDEO)
print("Classes", haaDataset.class_names)
sample_dataloader = dataset.get_HAA_data_loader(haaDataset,num_per_class=SAMPLE_NUM_PER_CLASS)
batch_dataloader = dataset.get_HAA_data_loader(haaDataset,num_per_class=BATCH_NUM_PER_CLASS)
samples, _ = sample_dataloader.__iter__().next()
batches,batch_labels = batch_dataloader.__iter__().next()
# Encoding
#sample_features = encoder(Variable(samples).to(device))
samples = encoder(Variable(samples).to(device))
samples = samples.view(CLASS_NUM,SAMPLE_NUM_PER_CLASS,FEATURE_DIM,8,8)
samples = torch.sum(samples,1).squeeze(1)
#batch_features = encoder(Variable(batches).to(device))
batches = encoder(Variable(batches).to(device))
batches = batches.view(CLASS_NUM*BATCH_NUM_PER_CLASS,FEATURE_DIM,8,8)
# Compute Relation
#sample_features_ext = sample_features.unsqueeze(0).repeat(BATCH_NUM_PER_CLASS*CLASS_NUM,1,1,1,1) #
samples = samples.unsqueeze(0).repeat(BATCH_NUM_PER_CLASS*CLASS_NUM,1,1,1,1)
#batch_features_ext = batch_features.unsqueeze(0).repeat(CLASS_NUM,1,1,1,1) #
batches = batches.unsqueeze(0).repeat(CLASS_NUM,1,1,1,1)
batches = torch.transpose(batches,0,1) # TODO
relations = torch.cat((samples,batches),2).view(-1,FEATURE_DIM*2,8,8) #
relations = rn(relations).view(-1,CLASS_NUM)
# return #
# Compute Loss
mse = nn.MSELoss().to(device) #
one_hot_labels = Variable(torch.zeros(BATCH_NUM_PER_CLASS*CLASS_NUM, CLASS_NUM).scatter_(1, batch_labels.view(-1,1), 1).to(device)) # TODO
loss = mse(relations,one_hot_labels) #
if episode % 200:
writer.add_scalar('Train/mse_loss', loss, episode)
# Train Model
encoder.zero_grad()
rn.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(encoder.parameters(),0.5)
nn.utils.clip_grad_norm(rn.parameters(),0.5)
encoder_optim.step()
rn_optim.step()
# Periodically Print Loss #TODO
# Testing
if (episode % 200 == 0 and episode != 0) or episode == EPISODE-1:
with torch.no_grad():
accuracies = []
for test_episode in range(TEST_EPISODE):
print("Test_Epi[{}] Pres_Accu = {}".format(test_episode, max_accuracy), end="\t")
# Data Loading #TODO
total_rewards = 0
total_num_tested = CLASS_NUM * BATCH_NUM_PER_CLASS
haaDataset = dataset.HAADataset(DATA_FOLDER, "test", CLASS_NUM, SAMPLE_NUM_PER_CLASS, NUM_INST, NUM_FRAME_PER_VIDEO)
print("Classes", haaDataset.class_names)
sample_dataloader = dataset.get_HAA_data_loader(haaDataset,num_per_class=SAMPLE_NUM_PER_CLASS) #
test_dataloader = dataset.get_HAA_data_loader(haaDataset,num_per_class=BATCH_NUM_PER_CLASS) #
samples, _ = sample_dataloader.__iter__().next()
batches, batches_labels = test_dataloader.__iter__().next()
samples = encoder(Variable(samples).to(device)) # 5x64 #
samples = samples.view(CLASS_NUM,SAMPLE_NUM_PER_CLASS,FEATURE_DIM,8,8) # TODO
samples = torch.sum(samples,1).squeeze(1) #
# Encoding
samples = samples.unsqueeze(0).repeat(CLASS_NUM*BATCH_NUM_PER_CLASS,1,1,1,1)
batches = encoder(Variable(batches).to(device)) # 20x64 #
batches = batches.view(CLASS_NUM*BATCH_NUM_PER_CLASS,FEATURE_DIM,8,8)
# Compute Relation
batches = batches.unsqueeze(0).repeat(1*CLASS_NUM,1,1,1,1) #
batches = torch.transpose(batches,0,1) # TODO
relations = torch.cat((samples,batches),2).view(-1,FEATURE_DIM*2,8,8) #
relations = rn(relations).view(-1,CLASS_NUM) #
# Predict
_, predict_labels = torch.max(relations.data,1)
# Counting Correct Ones
rewards = [1 if predict_labels[j]==batches_labels[j] else 0 for j in range(len(predict_labels))]
total_rewards += np.sum(rewards)
# Record accuracy
accuracy = total_rewards/1.0/total_num_tested
accuracies.append(accuracy)
# Overall accuracy
test_accuracy, _ = utils.mean_confidence_interval(accuracies)
print("Test_Accu = {}".format(test_accuracy))
writer.add_scalar('Test/accuracy', np.average(test_accuracy), epoch)
# Save Model
if test_accuracy > max_accuracy:
# save networks
torch.save(encoder.state_dict(),str("./models/encoder_" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot_" + str(test_accuracy)+".pkl"))
torch.save(rn.state_dict(),str("./models/rn_"+ str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot_" + str(test_accuracy)+".pkl"))
max_accuracy = test_accuracy
print("Model Saved with accuracy={}".format(max_accuracy))
print("final accuracy = {}".format(max_accuracy))
total_h = np.zeros(repeat_num)
for r in range(repeat_num):
print('==================== The %d-th round ====================' % r)
print('==================== The %d-th round ====================' % r,
file=F_txt_test)
# ======================================= Loaders of Datasets =======================================
opt.current_epoch = repeat_num
_, _, test_loader = FewShotDataloader.get_dataloader(
opt, ['train', 'val', 'test'])
# evaluate on validation/test set
prec1, val_loss, accuracies = validate(test_loader, model, criterion,
epoch_index, best_prec1,
F_txt_test)
test_accuracy, h = utils.mean_confidence_interval(accuracies)
total_accuracy += test_accuracy
total_h[r] = h
print('Test accuracy: %f h: %f \n' % (test_accuracy, h))
print('Test accuracy: %f h: %f \n' % (test_accuracy, h),
file=F_txt_test)
print('Mean_accuracy: %f h: %f' %
(total_accuracy / repeat_num, total_h.mean()))
print('Mean_accuracy: %f h: %f' %
(total_accuracy / repeat_num, total_h.mean()),
file=F_txt_test)
print(
'===================================== Test is END =====================================\n'
)
print(
def main(result_path, epoch_num):
config = json.load(open(os.path.join(result_path, 'config.json')))
fout_path = os.path.join(result_path, 'test_info.txt')
fout_file = open(fout_path, 'a+')
print_func(config, fout_file)
trsfms = transforms.Compose([
transforms.Resize((config['general']['image_size'],
config['general']['image_size'])),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std),
])
model = ALTNet(**config['arch'])
print_func(model, fout_file)
state_dict = torch.load(
os.path.join(result_path,
'{}_best_model.pth'.format(config['data_name'])))
model.load_state_dict(state_dict)
if config['train']['loss']['name'] == 'CrossEntropyLoss':
criterion = nn.CrossEntropyLoss(**config['train']['loss']['args'])
else:
raise RuntimeError
device, _ = prepare_device(config['n_gpu'])
model = model.to(device)
criterion = criterion.to(device)
total_accuracy = 0.0
total_h = np.zeros(epoch_num)
total_accuracy_vector = []
for epoch_idx in range(epoch_num):
test_dataset = ImageFolder(
data_root=config['general']['data_root'],
mode='test',
episode_num=600,
way_num=config['general']['way_num'],
shot_num=config['general']['shot_num'],
query_num=config['general']['query_num'],
transform=trsfms,
)
print_func('The num of the test_dataset: {}'.format(len(test_dataset)),
fout_file)
test_loader = torch.utils.data.DataLoader(
test_dataset,
batch_size=config['test']['batch_size'],
shuffle=True,
num_workers=config['general']['workers_num'],
drop_last=True,
pin_memory=True)
print_func('============ Testing on the test set ============',
fout_file)
_, accuracies = validate(test_loader, model, criterion, epoch_idx,
device, fout_file,
config['general']['image2level'],
config['general']['print_freq'])
test_accuracy, h = mean_confidence_interval(accuracies)
print_func("Test Accuracy: {}\t h: {}".format(test_accuracy, h[0]),
fout_file)
total_accuracy += test_accuracy
total_accuracy_vector.extend(accuracies)
total_h[epoch_idx] = h
aver_accuracy, _ = mean_confidence_interval(total_accuracy_vector)
print_func(
'Aver Accuracy: {:.3f}\t Aver h: {:.3f}'.format(
aver_accuracy, total_h.mean()), fout_file)
print_func('............Testing is end............', fout_file)
def meta_test_per_dataset(self, data_name):
self.nasbench201 = torch.load(
os.path.join(self.data_path, 'nasbench201.pt'))
self.test_dataset = MetaTestDataset(
self.data_path, data_name, self.num_sample, self.num_class)
meta_test_path = os.path.join(
self.save_path, 'meta_test', data_name, 'best_arch')
if not os.path.exists(meta_test_path):
os.makedirs(meta_test_path)
f_arch_str = open(
os.path.join(meta_test_path, 'architecture.txt'), 'w')
save_path = os.path.join(meta_test_path, 'accuracy.txt')
f = open(save_path, 'w')
arch_runs = []
elasped_time = []
if 'cifar' in data_name:
N = 30
runs = 10
acc_runs = []
else:
N = 1
runs = 1
print(f'==> select top architectures for {data_name} by meta-predictor...')
for run in range(1, runs + 1):
print(f'==> run #{run}')
gen_arch_str = self.load_generated_archs(data_name, run)
gen_arch_igraph = self.get_items(
full_target=self.nasbench201['arch']['igraph'],
full_source=self.nasbench201['arch']['str'],
source=gen_arch_str)
y_pred_all = []
self.model.eval()
self.model.to(self.device)
sttime = time.time()
with torch.no_grad():
for i, arch_igraph in enumerate(gen_arch_igraph):
x, g = self.collect_data(arch_igraph)
y_pred = self.forward(x, g)
y_pred = torch.mean(y_pred)
y_pred_all.append(y_pred.cpu().detach().item())
top_arch_lst = self.select_top_arch(
data_name, torch.tensor(y_pred_all), gen_arch_str, N)
arch_runs.append(top_arch_lst[0])
elasped = time.time() - sttime
elasped_time.append(elasped)
if 'cifar' in data_name:
acc = self.select_top_acc(data_name, top_arch_lst)
acc_runs.append(acc)
for run, arch_str in enumerate(arch_runs):
f_arch_str.write(f'{arch_str}\n'); print(f'{arch_str}')
time_path = os.path.join(
self.save_path, 'meta_test', data_name, 'time.txt')
with open(time_path, 'a') as f_time:
msg = f'predictor average elasped time {np.mean(elasped_time):.2f}s'
print(f'==> save time in {time_path}')
f_time.write(msg+'\n'); print(msg)
if self.train_arch:
if not 'cifar' in data_name:
acc_runs = self.train_single_arch(
data_name, arch_runs[0], meta_test_path)
print(f'==> save results in {save_path}')
for r, acc in enumerate(acc_runs):
msg = f'run {r+1} {acc:.2f} (%)'
f.write(msg+'\n'); print(msg)
m, h = mean_confidence_interval(acc_runs)
msg = f'Avg {m:.2f}+-{h.item():.2f} (%)'
f.write(msg+'\n'); print(msg)
def main(args):
'''
main function
'''
if args.logport:
args.logport = Dashboard(args.logport, 'dashboard')
EPOCH_SIZE = args.num_episode*args.num_query*args.way_train
EPOCH_SIZE_TEST = args.num_episode_test*args.num_query*args.way_test
'''define network'''
net = Net(args.num_in_channel, args.num_filter)
relationNet = RelationNet(args.num_filter*2, args.num_filter, 5*5*args.num_filter, args.num_fc, args.drop_prob)
if torch.cuda.is_available():
net.cuda()
relationNet.cuda()
'''
load model if needed
'''
if args.model_load_path_net!='':
net.load_state_dict(torch.load(args.model_load_path_net))
net.cuda()
relationNet.load_state_dict(torch.load(args.model_load_path_relationNet))
relationNet.cuda()
print('model loaded')
''' define loss, optimizer'''
criterion = nn.MSELoss()
params = list(net.parameters()) + list(relationNet.parameters())
optimizer = optim.Adam(params, lr=args.learning_rate)
'''get data'''
trainList = read_miniImageNet_pathonly(TESTMODE=False,
miniImageNetPath='/home/fujenchu/projects/dataset/miniImageNet_Ravi/',
imgPerCls=600)
testList = read_miniImageNet_pathonly(TESTMODE=True,
miniImageNetPath='/home/fujenchu/projects/dataset/miniImageNet_Ravi/',
imgPerCls=600)
scheduler = StepLR(optimizer, step_size=40, gamma=0.5)
''' training'''
for epoch in range(1000):
scheduler.step()
running_loss = 0.0
avg_accu_Train = 0.0
accu_Test_stats = []
net.train()
relationNet.train()
# epoch training list
trainList_combo = Producer(trainList, args.way_train, args.num_episode, "training") # combo contains [query_label, query_path ]
list_trainset = tnt.dataset.ListDataset(trainList_combo, loadImg)
trainloader = list_trainset.parallel(batch_size=args.batch_size, num_workers=args.num_workers, shuffle=True)
for i, data in enumerate(tqdm(trainloader), 0):
#for i, data in enumerate(trainloader, 0):
# get inputs
batchSize = data[0].size()[0]
labels = torch.unsqueeze(data[0], 1)
images = data[1:]
images_all = torch.cat(images).permute(0, 3, 1, 2).float()
labels_one_hot = torch.zeros(data[0].size()[0], args.way_train)
labels_one_hot.scatter_(1, labels, 1.0)
# wrap in Variable
if torch.cuda.is_available():
images_all, labels_one_hot = Variable(images_all.cuda()), Variable(labels_one_hot.cuda())
else:
images_all, labels_one_hot = Variable(images_all), Variable(labels_one_hot)
# zero gradients
optimizer.zero_grad()
# forward + backward + optimizer
feature_s_all_t0_p = net(images_all)
feature_s_all_t0_p = torch.split(feature_s_all_t0_p, batchSize, 0)
concatenatedFeat_list = [[] for _ in range(args.way_train)]
for idx in range(args.way_train):
concatenatedFeat_list[idx] = torch.cat((feature_s_all_t0_p[idx], feature_s_all_t0_p[-1]), 1)
concatenatedFeat_all = torch.cat(concatenatedFeat_list, 0)
relationScore_all = relationNet(concatenatedFeat_all)
relationScore_list = torch.split(relationScore_all, batchSize, 0)
relationScore = torch.cat(relationScore_list, 1)
#loss = criterion(relationScore, labels_one_hot)
weights = labels_one_hot.clone()
weights[labels_one_hot == 0] = 1.0/(args.way_train)
weights[labels_one_hot != 0] = (args.way_train-1.0)/(args.way_train)
loss = weighted_mse_loss(relationScore, labels_one_hot, weights)/data[0].size()[0]
loss.backward()
optimizer.step()
# summing up
running_loss += loss.data[0]
_, predicted = torch.max(relationScore.data, 1)
labels = torch.squeeze(labels, 1)
avg_accu_Train += (predicted == labels.cuda()).sum()
if i % args.log_step == args.log_step-1:
#print('[%d, %5d] train loss: %.3f train accuracy: %.3f' % (epoch + 1, i + 1, running_loss / args.log_step, avg_accu_Train/(args.log_step*batchSize)))
if args.logport:
args.logport.appendlog(running_loss / args.log_step, 'Training Loss')
args.logport.appendlog(avg_accu_Train/(args.log_step*batchSize), 'Training Accuracy')
args.logport.image((images[-1][0, :, :, :]).permute(2, 0, 1), 'query img', mode='img')
for idx in range(args.way_train):
args.logport.image((images[idx][0, :, :, :]).permute(2, 0, 1), 'support img', mode='img')
running_loss = 0.0
avg_accu_Train = 0.0
if (i+1) % args.save_step == 0:
torch.save(net.state_dict(),
os.path.join(args.model_path,
'net-model-%d-%d.pkl' %(epoch+1, i+1)))
torch.save(relationNet.state_dict(),
os.path.join(args.model_path,
'relationNet-model-%d-%d.pkl' %(epoch+1, i+1)))
net.eval()
relationNet.eval()
# epoch training list
testList_combo = Producer(testList, args.way_test, args.num_episode_test, "testing") # combo contains [query_label, query_path ]
list_testset = tnt.dataset.ListDataset(testList_combo, loadImg_testing)
testloader = list_testset.parallel(batch_size=args.batch_size_test, num_workers=args.num_workers, shuffle=False)
#for i, data in enumerate(tqdm(testloader), 0):
for i, data in enumerate(testloader, 0):
# get inputs
batchSize = data[0].size()[0]
labels = torch.unsqueeze(data[0], 1)
images = data[1:]
images_all = torch.cat(images).permute(0, 3, 1, 2).float()
labels_one_hot = torch.zeros(batchSize, args.way_test)
labels_one_hot.scatter_(1, labels, 1.0)
# wrap in Variable
if torch.cuda.is_available():
images_all, labels_one_hot = Variable(images_all.cuda(), volatile=True), Variable(labels_one_hot.cuda(), volatile=True)
else:
images_all, labels_one_hot = Variable(images_all, volatile=True), Variable(labels_one_hot, volatile=True)
# forward
feature_s_all_t0_p = net(images_all)
feature_s_all_t0_p = torch.split(feature_s_all_t0_p, batchSize, 0)
concatenatedFeat_list = [[] for _ in range(args.way_test)]
for idx in range(args.way_test):
concatenatedFeat_list[idx] = torch.cat((feature_s_all_t0_p[idx], feature_s_all_t0_p[-1]), 1)
concatenatedFeat_all = torch.cat(concatenatedFeat_list, 0)
relationScore_all = relationNet(concatenatedFeat_all)
relationScore_list = torch.split(relationScore_all, batchSize, 0)
relationScore = torch.cat(relationScore_list, 1)
_, predicted = torch.max(relationScore.data, 1)
#avg_accu_Test += (predicted == torch.squeeze(labels, 1).cuda()).sum()
accu_Test_stats.append((predicted == torch.squeeze(labels, 1).cuda()).sum()/float(batchSize))
m, h = mean_confidence_interval(np.asarray(accu_Test_stats), confidence=0.95)
print('[epoch %3d] test accuracy with 0.95 confidence: %.4f, +-: %.4f' % (epoch + 1, m, h))
#avg_accu_Test = 0.0
accu_Test_stats = []
analysis_dir = 'run_analysis'
with open(os.path.join(result_dir, 'params.txt'), 'r') as file:
params_s = file.read().replace('\n', '')
param_dict = ast.literal_eval(params_s)
shutil.rmtree(analysis_dir, ignore_errors=True)
print('')
os.mkdir(analysis_dir)
run_dirs = sorted(get_dirs_in_path(result_dir))
i = 0
deliveries = []
collisions = []
delivery_times = []
for run_dir in run_dirs:
i += 1
print('Analysing run {}/{}'.format(i, len(run_dirs)))
run_deliveries, run_collisions, agents, run_times = analyse_run(
run_dir, param_dict['steps'])
deliveries.append(run_deliveries / agents)
collisions.append(run_collisions / agents)
delivery_times += run_times
mean, h = mean_confidence_interval(deliveries)
print('Deliveries mean: {} {}'.format(mean, h))
mean, h = mean_confidence_interval(collisions)
print('Collisions mean: {} {}'.format(mean, h))
print('Delivery mean: {}'.format(
sum(delivery_times) / len(delivery_times)))
def test(args):
setup_seed(2333)
import warnings
warnings.filterwarnings('ignore')
if args.dataset == 'cub':
num_classes = 100
elif args.dataset == 'tieredimagenet':
num_classes = 351
else:
num_classes = 64
if args.resume is not None:
from models.resnet12 import resnet12
model = resnet12(num_classes).to(args.device)
state_dict = torch.load(args.resume)
model.load_state_dict(state_dict)
model.to(args.device)
model.eval()
ici = ICI(classifier=args.classifier,
num_class=args.num_test_ways,
step=args.step,
reduce=args.embed,
d=args.dim)
data_root = os.path.join(args.folder, args.dataset)
dataset = DataSet(data_root, 'test', args.img_size)
sampler = CategoriesSampler(dataset.label, args.num_batches,
args.num_test_ways,
(args.num_shots, 15, args.unlabel))
testloader = DataLoader(dataset,
batch_sampler=sampler,
shuffle=False,
num_workers=0,
pin_memory=True)
k = args.num_shots * args.num_test_ways
loader = tqdm(testloader, ncols=0)
iterations = math.ceil(args.unlabel/args.step) + \
2 if args.unlabel != 0 else math.ceil(15/args.step) + 2
acc_list = [[] for _ in range(iterations)]
for data, indicator in loader:
targets = torch.arange(args.num_test_ways).repeat(
args.num_shots + 15 + args.unlabel).long()[
indicator[:args.num_test_ways *
(args.num_shots + 15 + args.unlabel)] != 0]
data = data[indicator != 0].to(args.device)
train_inputs = data[:k]
train_targets = targets[:k].cpu().numpy()
test_inputs = data[k:k + 15 * args.num_test_ways]
test_targets = targets[k:k + 15 * args.num_test_ways].cpu().numpy()
train_embeddings = get_embedding(model, train_inputs, args.device)
ici.fit(train_embeddings, train_targets)
test_embeddings = get_embedding(model, test_inputs, args.device)
if args.unlabel != 0:
unlabel_inputs = data[k + 15 * args.num_test_ways:]
unlabel_embeddings = get_embedding(model, unlabel_inputs,
args.device)
else:
unlabel_embeddings = None
acc = ici.predict(test_embeddings, unlabel_embeddings, True,
test_targets)
for i in range(min(iterations - 1, len(acc))):
acc_list[i].append(acc[i])
acc_list[-1].append(acc[-1])
mean_list = []
ci_list = []
for item in acc_list:
mean, ci = mean_confidence_interval(item)
mean_list.append(mean)
ci_list.append(ci)
print("Test Acc Mean{}".format(' '.join(
[str(i * 100)[:5] for i in mean_list])))
print("Test Acc ci{}".format(' '.join([str(i * 100)[:5]
for i in ci_list])))
def main():
utils.print_stage("Program Starts")
metatrain_folders, metatest_folders = tg.HAA_folders()
# i3d == the I3D network
encoder = cnn()
# rn == the relation network
rn = RN(FEATURE_DIM, RELATION_DIM)
# Move the model to GPU
encoder.to(device)
rn.to(device)
# Define Optimizer
encoder_optim = torch.optim.Adam(encoder.parameters(), lr=LEARNING_RATE)
rn_optim = torch.optim.Adam(rn.parameters(), lr=LEARNING_RATE)
# Define Scheduler
encoder_scheduler = StepLR(encoder_optim, step_size=100000, gamma=0.5)
rn_scheduler = StepLR(rn_optim, step_size=100000, gamma=0.5)
# Load Saved Model #TODO
# Training Starts Here
utils.print_stage("Start Training")
# Accuracy Record
max_accuracy = 0 #TODO read accuracy from somewhere
for episode in range(EPISODE):
print("{}\tepisode{}".format(max_accuracy, episode))
# Update "step" for scheduler
rn_scheduler.step(episode)
encoder_scheduler.step(episode)
# Setup Data #TODO
task = tg.HAATask(metatrain_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS,
BATCH_NUM_PER_CLASS)
sample_dataloader = tg.get_HAA_data_loader(
task,
num_per_class=SAMPLE_NUM_PER_CLASS,
split="train",
shuffle=False)
batch_dataloader = tg.get_HAA_data_loader(
task,
num_per_class=BATCH_NUM_PER_CLASS,
split="test",
shuffle=True)
samples, sample_labels = sample_dataloader.__iter__().next(
) #25*3*84*84
batches, batch_labels = batch_dataloader.__iter__().next()
# Encoding
sample_features = encoder(
Variable(samples).to(device)) # 25*64*19*19 #
sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS,
FEATURE_DIM, 62, 62) # TODO
sample_features = torch.sum(sample_features, 1).squeeze(1) #
batch_features = encoder(Variable(batches).to(
device)) # 20x64*5*5 #
# Compute Relation
sample_features_ext = sample_features.unsqueeze(0).repeat(
BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) #
batch_features_ext = batch_features.unsqueeze(0).repeat(
CLASS_NUM, 1, 1, 1, 1) #
batch_features_ext = torch.transpose(batch_features_ext, 0, 1) # TODO
relation_pairs = torch.cat((sample_features_ext, batch_features_ext),
2).view(-1, FEATURE_DIM * 2, 62, 62) #
relations = rn(relation_pairs).view(-1, CLASS_NUM) #
# Compute Loss
mse = nn.MSELoss().to(device) #
one_hot_labels = Variable(
torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM,
CLASS_NUM).scatter_(1, batch_labels.view(-1, 1),
1).to(device)) # TODO
loss = mse(relations, one_hot_labels) #
# Train Model
encoder.zero_grad()
rn.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(encoder.parameters(), 0.5)
nn.utils.clip_grad_norm(rn.parameters(), 0.5)
encoder_optim.step()
rn_optim.step()
# Periodically Print Loss #TODO
# Testing
if (episode % 200 == 0 and episode != 0) or episode == EPISODE - 1:
utils.print_stage("Testing")
accuracies = []
for _ in range(TEST_EPISODE):
# Data Loading #TODO
total_rewards = 0
task = tg.HAATask(metatest_folders, CLASS_NUM,
SAMPLE_NUM_PER_CLASS, 15) #
sample_dataloader = tg.get_HAA_data_loader(
task,
num_per_class=SAMPLE_NUM_PER_CLASS,
split="train",
shuffle=False) #
num_per_class = 5 # TODO
test_dataloader = tg.get_HAA_data_loader(
task,
num_per_class=num_per_class,
split="test",
shuffle=False) #
sample_images, sample_labels = sample_dataloader.__iter__(
).next() #
for test_images, test_labels in test_dataloader:
batch_size = test_labels.shape[0]
# Encoding
sample_features = encoder(
Variable(sample_images).to(
device)) # 5x64 #
sample_features = sample_features.view(
CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 62,
62) # TODO
sample_features = torch.sum(sample_features,
1).squeeze(1) #
test_features = encoder(Variable(test_images).to(
device)) # 20x64 #
# Compute Relation
sample_features_ext = sample_features.unsqueeze(0).repeat(
batch_size, 1, 1, 1, 1) #
test_features_ext = test_features.unsqueeze(0).repeat(
1 * CLASS_NUM, 1, 1, 1, 1) #
test_features_ext = torch.transpose(
test_features_ext, 0, 1) # TODO
relation_pairs = torch.cat(
(sample_features_ext, test_features_ext),
2).view(-1, FEATURE_DIM * 2, 62, 62) #
relations = rn(relation_pairs).view(-1, CLASS_NUM) #
# Predict
_, predict_labels = torch.max(relations.data, 1)
# Counting Correct Ones
rewards = [
1 if predict_labels[j] == test_labels[j] else 0
for j in range(batch_size)
]
total_rewards += np.sum(rewards)
# Record accuracy
accuracy = total_rewards / 1.0 / CLASS_NUM / 15
accuracies.append(accuracy)
# Overall accuracy
test_accuracy, _ = utils.mean_confidence_interval(accuracies)
# Save Model
if test_accuracy > max_accuracy:
# save networks
torch.save(
encoder.state_dict(),
str("./models/encoder_" + str(CLASS_NUM) + "way_" +
str(SAMPLE_NUM_PER_CLASS) + "shot_" +
str(test_accuracy) + ".pkl"))
torch.save(
rn.state_dict(),
str("./models/rn_" + str(CLASS_NUM) + "way_" +
str(SAMPLE_NUM_PER_CLASS) + "shot_" +
str(test_accuracy) + ".pkl"))
max_accuracy = test_accuracy
print("Model Saved with accuracy={}".format(max_accuracy))
print("final accuracy = {}".format(max_accuracy))
def main(args):
'''
main function
'''
if args.logport:
args.logport = Dashboard(args.logport, 'dashboard')
EPOCH_SIZE = args.num_episode * args.num_query * args.way_train
EPOCH_SIZE_TEST = args.num_episode_test * args.num_query * args.way_test
SM_CONSTANT = 50
'''define network'''
net = Net(args.num_in_channel, args.num_filter)
if torch.cuda.is_available():
net.cuda()
'''
load model if needed
'''
if args.model_load_path_net != '':
net.load_state_dict(torch.load(args.model_load_path_net))
net.cuda()
print('model loaded')
''' define loss, optimizer'''
criterion = nn.CrossEntropyLoss()
params = list(net.parameters())
optimizer = optim.Adam(params, lr=args.learning_rate)
'''get data'''
trainList = read_miniImageNet_pathonly(
TESTMODE=False,
miniImageNetPath='/media/fujenchu/data/miniImageNet_Ravi/',
imgPerCls=600)
testList = read_miniImageNet_pathonly(
TESTMODE=True,
miniImageNetPath='/media/fujenchu/data/miniImageNet_Ravi/',
imgPerCls=600)
scheduler = StepLR(optimizer, step_size=40, gamma=0.5)
''' training'''
for epoch in range(1000):
scheduler.step()
running_loss = 0.0
avg_accu_Train = 0.0
accu_Test_stats = []
net.train()
# epoch training list
trainList_combo = Producer(
trainList, args.way_train, args.num_episode,
"training") # combo contains [query_label, query_path ]
list_trainset = tnt.dataset.ListDataset(trainList_combo, loadImg)
trainloader = list_trainset.parallel(batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=True)
for i, data in enumerate(tqdm(trainloader), 0):
#for i, data in enumerate(trainloader, 0):
# get inputs
batchSize = data[0].size()[0]
labels = torch.unsqueeze(data[0], 1)
images = data[1:]
images_all = torch.cat(images).permute(0, 3, 1, 2).float()
labels_one_hot = torch.zeros(data[0].size()[0], args.way_train)
labels_one_hot.scatter_(1, labels, 1.0)
# wrap in Variable
if torch.cuda.is_available():
images_all, labels = Variable(images_all.cuda()), Variable(
labels.cuda())
else:
images_all, labels = Variable(images_all), Variable(labels)
# zero gradients
optimizer.zero_grad()
# forward + backward + optimizer
feature_s_all_t0_p = net(images_all)
feature_s_all_t0_p = torch.split(feature_s_all_t0_p, batchSize, 0)
cosineDist_list = [[] for _ in range(args.way_train)]
for idx in range(args.way_train):
cosineDist_list[idx] = SM_CONSTANT * torch.sum(torch.mul(
feature_s_all_t0_p[-1].div(
torch.norm(
feature_s_all_t0_p[-1], p=2, dim=1,
keepdim=True).expand_as(feature_s_all_t0_p[-1])),
feature_s_all_t0_p[idx].div(
torch.norm(
feature_s_all_t0_p[idx], p=2, dim=1,
keepdim=True).expand_as(feature_s_all_t0_p[idx]))),
dim=1,
keepdim=True)
cosineDist_all = torch.cat(cosineDist_list, 1)
labels = labels.squeeze(1)
loss = criterion(cosineDist_all, labels)
loss.backward()
optimizer.step()
# summing up
running_loss += loss.data[0]
_, predicted = torch.max(cosineDist_all.data, 1)
avg_accu_Train += (predicted == labels.data).sum()
if i % args.log_step == args.log_step - 1:
#print('[%d, %5d] train loss: %.3f train accuracy: %.3f' % (epoch + 1, i + 1, running_loss / args.log_step, avg_accu_Train/(args.log_step*batchSize)))
if args.logport:
args.logport.appendlog(running_loss / args.log_step,
'Training Loss')
args.logport.appendlog(
avg_accu_Train / (args.log_step * batchSize),
'Training Accuracy')
args.logport.image(
(images[-1][0, :, :, :]).permute(2, 0, 1),
'query img',
mode='img')
for idx in range(args.way_train):
args.logport.image(
(images[idx][0, :, :, :]).permute(2, 0, 1),
'support img',
mode='img')
running_loss = 0.0
avg_accu_Train = 0.0
if (i + 1) % args.save_step == 0:
torch.save(
net.state_dict(),
os.path.join(args.model_path,
'net-model-%d-%d.pkl' % (epoch + 1, i + 1)))
net.eval()
# epoch training list
testList_combo = Producer(
testList, args.way_test, args.num_episode_test,
"testing") # combo contains [query_label, query_path ]
list_testset = tnt.dataset.ListDataset(testList_combo, loadImg_testing)
testloader = list_testset.parallel(batch_size=args.batch_size_test,
num_workers=args.num_workers,
shuffle=False)
#for i, data in enumerate(tqdm(testloader), 0):
for i, data in enumerate(testloader, 0):
# get inputs
batchSize = data[0].size()[0]
labels = torch.unsqueeze(data[0], 1)
images = data[1:]
images_all = torch.cat(images).permute(0, 3, 1, 2).float()
labels_one_hot = torch.zeros(batchSize, args.way_test)
labels_one_hot.scatter_(1, labels, 1.0)
# wrap in Variable
if torch.cuda.is_available():
images_all, labels = Variable(
images_all.cuda(), volatile=True), Variable(labels.cuda(),
volatile=True)
else:
images_all, labels = Variable(
images_all, volatile=True), Variable(labels, volatile=True)
# forward
feature_s_all_t0_p = net(images_all)
feature_s_all_t0_p = torch.split(feature_s_all_t0_p, batchSize, 0)
cosineDist_list = [[] for _ in range(args.way_train)]
for idx in range(args.way_train):
cosineDist_list[idx] = SM_CONSTANT * torch.sum(torch.mul(
feature_s_all_t0_p[-1].div(
torch.norm(
feature_s_all_t0_p[-1], p=2, dim=1,
keepdim=True).expand_as(feature_s_all_t0_p[-1])),
feature_s_all_t0_p[idx].div(
torch.norm(
feature_s_all_t0_p[idx], p=2, dim=1,
keepdim=True).expand_as(feature_s_all_t0_p[idx]))),
dim=1,
keepdim=True)
cosineDist_all = torch.cat(cosineDist_list, 1)
_, predicted = torch.max(cosineDist_all.data, 1)
accu_Test_stats.append(
(predicted == torch.squeeze(labels, 1).data.cuda()).sum() /
float(batchSize))
equality = (predicted != torch.squeeze(labels, 1).data.cuda())
equality_s = (predicted == torch.squeeze(labels, 1).data.cuda())
equality_idx = equality.nonzero()
equality_idx_s = equality_s.nonzero()
if i % args.log_step == args.log_step - 1:
if args.logport:
pred_np = predicted.cpu().numpy()
labels_np = labels.cpu().data.numpy()
batch_idx = equality_idx[0].cpu().numpy().astype(int)
bb = batch_idx[0]
args.logport.image(
(images[-1][bb, :, :, :]).permute(2, 0, 1),
np.array_str(labels_np[bb]) +
np.array_str(pred_np[bb]) + ' query img',
mode='img-test')
support_image = []
for idx in range(args.way_train):
support_image.append(images[idx][bb, :, :, :].permute(
2, 0, 1))
args.logport.image(torch.cat(support_image, 2),
'support img',
mode='img-test')
batch_idx = equality_idx_s[0].cpu().numpy().astype(int)
bb = batch_idx[0]
args.logport.image(
(images[-1][bb, :, :, :]).permute(2, 0, 1),
np.array_str(labels_np[bb]) +
np.array_str(pred_np[bb]) + ' query img',
mode='img-test')
support_image = []
for idx in range(args.way_train):
support_image.append(images[idx][bb, :, :, :].permute(
2, 0, 1))
args.logport.image(torch.cat(support_image, 2),
'support img',
mode='img-test')
m, h = mean_confidence_interval(np.asarray(accu_Test_stats),
confidence=0.95)
print(
'[epoch %3d] test accuracy with 0.95 confidence: %.4f, +-: %.4f' %
(epoch + 1, m, h))
#avg_accu_Test = 0.0
accu_Test_stats = []
actions_turn.data[:, :, 1].fill_(2)
u_i, u_s, u_c, _, _, _, _ = planning.compute_uncertainty_batch(
model, input_images, input_states, actions_turn, targets, car_sizes)
pred, loss_p = model(inputs, actions_turn, targets, z_dropout=0)
ui_turn.append(u_i)
us_turn.append(u_s)
ui_truth = torch.stack(ui_truth)
ui_turn = torch.stack(ui_turn)
ui_perm = torch.stack(ui_perm)
ui_truth = ui_truth.view(-1, opt.npred).cpu().numpy()
ui_turn = ui_turn.view(-1, opt.npred).cpu().numpy()
ui_perm = ui_perm.view(-1, opt.npred).cpu().numpy()
us_truth = torch.stack(us_truth)
us_turn = torch.stack(us_turn)
us_perm = torch.stack(us_perm)
us_truth = us_truth.view(-1, opt.npred).cpu().numpy()
us_turn = us_turn.view(-1, opt.npred).cpu().numpy()
us_perm = us_perm.view(-1, opt.npred).cpu().numpy()
mean, low, hi = utils.mean_confidence_interval(ui_truth)
utils.plot_mean_and_CI(mean,
low,
hi,
color_mean='magenta',
color_shading='magenta')
#mean,low,hi=utils.mean_confidence_interval(us_perm)
#utils.plot_mean_and_CI(mean, low, hi, color_mean='cyan', color_shading='cyan')
mean, low, hi = utils.mean_confidence_interval(ui_turn)
utils.plot_mean_and_CI(mean, low, hi, color_mean='blue', color_shading='blue')
datas = datas.to(device)
pivot = args.way * args.shot
shot, query = datas[:pivot], datas[pivot:]
labels = torch.arange(args.way).repeat(args.query).to(device)
# one_hot_labels = Variable(torch.zeros(args.way*args.query, args.way).scatter_(1, labels.view(-1, 1), 1)).to(device)
pred = model(shot, query)
# calculate loss
loss = F.cross_entropy(pred, labels).item()
test_loss.append(loss)
total_loss = sum(test_loss) / len(test_loss)
# calculate accuracy
acc = 100 * (pred.argmax(1) == labels).type(
torch.cuda.FloatTensor if torch.cuda.is_available(
) else torch.FloatTensor).mean().item()
test_acc.append(acc)
total_acc = sum(test_acc) / len(test_acc)
printer("test", e, args.num_epochs, i + 1, len(test_loader), loss,
total_loss, acc, total_acc)
print("")
# get mean confidence interval and mean
m, h = mean_confidence_interval(test_acc, confidence=0.95)
total_mean.append(m)
total_interval.append(h)
print("Result: {:.2f}+-{:.2f}".format(
sum(total_mean) / len(total_mean),
sum(total_interval) / len(total_interval)))
reward = np.array([[0, -1, -1], \
[-1, 0, -1], \
[-1, -1, 0]])
regrets = np.zeros(num_run)
accuracy_scores = np.zeros(num_run)
for i in range(num_run):
print("Processing Trial # %d out of %d." % (i+1, num_run))
SLR = SupLinRel(K, reward, delta=delta)
SLR.data_load(X, y)
SLR.run()
regrets[i] = SLR.total_regret()
accuracy_scores[i] = SLR.accuracy_score()
regrets_mean, regrets_width = mean_confidence_interval(regrets)
acc_mean, acc_width = mean_confidence_interval(accuracy_scores)
plt.scatter(np.zeros_like(regrets), regrets)
plt.xlim(-0.2, 0.2)
plt.fill_between(np.array([-0.1, 0.1]), np.full(2, regrets_mean-regrets_width), \
np.full(2, regrets_mean+regrets_width), color='b', alpha=.1)
plt.hlines(regrets_mean, -0.1, 0.1)
plt.text(0.1, regrets_mean-regrets_width, np.round(regrets_mean-regrets_width, 2))
plt.text(0.1, regrets_mean, np.round(regrets_mean, 2))
plt.text(0.1, regrets_mean+regrets_width, np.round(regrets_mean+regrets_width, 2))
plt.tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=False)
plt.title("Regret Confidence Interval with delta = "+str(delta))
plt.savefig("output/SupLinRel_Regret_confidence_interval")
plt.clf()
ql_no = 'ql_tr_det_online_no' + num + '.npy'
ai = 'trwp_det.csv'
ai_no = 'trwop_det.csv'
BRL_tr_online = np.load(br)
QL_tr_online = np.load(ql)
QL_tr_online_no = np.load(ql_no)
trwp = pd.read_csv(ai, header=None)
trwp_rewards = np.array(np.where(trwp == 1, 100, 0))
trwop = pd.read_csv(ai_no, header=None)
trwop_rewards = np.where(trwop == 1, 100, 0)
# Average:
ci = np.zeros([5, 3])
ql = np.mean(QL_tr_online, axis=1)
ci[0, 0], ci[0, 1], ci[0, 2] = ut.mean_confidence_interval(ql)
brl = np.mean(BRL_tr_online, axis=1)
ci[1, 0], ci[1, 1], ci[1, 2] = ut.mean_confidence_interval(brl)
qlno = np.mean(QL_tr_online_no, axis=1)
ci[2, 0], ci[2, 1], ci[2, 2] = ut.mean_confidence_interval(qlno)
trwp = np.mean(trwp_rewards, axis=1)
ci[3, 0], ci[3, 1], ci[3, 2] = ut.mean_confidence_interval(trwp)
trwop = np.mean(trwop_rewards, axis=1)
ci[4, 0], ci[4, 1], ci[4, 2] = ut.mean_confidence_interval(trwop)
if det:
ut.plot_subresultsci('deterministic_plot', QL_tr_online_no, QL_tr_online,
BRL_tr_online, trwp_rewards, trwop_rewards, 500,
False)
if sto:
ut.plot_subresultsci('stochastic_plot', QL_tr_online_no, QL_tr_online,
def loop_best(opt):
models = ['GAT'] #'AGNN']#,'GAT']
# layers = [1,2,4,8,16]
# layers = [1, 2, 4, 8, 16, 24, 32]
layers = [12, 20]
att_type_AGNN = ['cosine', 'scaled_dot', 'pearson', 'spearman']
att_type_GAT = ['GAT', 'cosine', 'scaled_dot', 'pearson', 'spearman']
for model in models:
for layer in layers:
for att_type in att_type_AGNN if model == 'AGNN' else att_type_GAT:
best_params_dir = get_best_specific_params_dir(
opt, layer, model, att_type)
with open(best_params_dir + '/params.json') as f:
best_params = json.loads(f.read())
# allow params specified at the cmd line to override
best_params_ret = {**best_params, **opt}
# the exception is number of epochs as we want to use more here than we would for hyperparameter tuning.
best_params_ret['epoch'] = opt['epoch']
print("Running with parameters {}".format(best_params_ret))
data_dir = os.path.abspath("../data")
reporter = CLIReporter(metric_columns=[
"accuracy", "loss", "test_acc", "train_acc", "best_time",
"best_epoch", "training_iteration"
])
if opt['name'] is None:
name = opt['folder'] + '_test'
else:
name = opt['name']
result = tune.run(
partial(train_ray_int, data_dir=data_dir),
name=name,
resources_per_trial={
"cpu": opt['cpus'],
"gpu": opt['gpus']
},
search_alg=None,
keep_checkpoints_num=3,
checkpoint_score_attr='accuracy',
config=best_params_ret,
num_samples=opt['reps'] if opt["num_splits"] == 0 else
opt["num_splits"] * opt["reps"],
scheduler=None,
max_failures=
1, # early stop solver can't recover from failure as it doesn't own m2.
local_dir='../ray_tune',
progress_reporter=reporter,
raise_on_failed_trial=False)
df = result.dataframe(metric=opt['metric'],
mode="max").sort_values(opt['metric'],
ascending=False)
filename = f"../ray_results/{opt['name']}.csv"
df.to_csv(filename,
mode='a',
header=(not os.path.exists(filename)))
print(df[[
'accuracy', 'test_acc', 'train_acc', 'best_time',
'best_epoch'
]])
test_accs = df['test_acc'].values
print("test accuracy {}".format(test_accs))
log = "mean test {:04f}, test std {:04f}, test sem {:04f}, test 95% conf {:04f}"
print(
log.format(test_accs.mean(), np.std(test_accs),
get_sem(test_accs),
mean_confidence_interval(test_accs)))
def main(config):
random.seed(0)
np.random.seed(0)
torch.manual_seed(0)
torch.cuda.manual_seed(0)
# torch.backends.cudnn.deterministic = True
# torch.backends.cudnn.benchmark = False
##### Dataset #####
dataset = datasets.make(config['dataset'], **config['test'])
utils.log('meta-test set: {} (x{}), {}'.format(dataset[0][0].shape,
len(dataset),
dataset.n_classes))
loader = DataLoader(dataset,
config['test']['n_episode'],
collate_fn=datasets.collate_fn,
num_workers=1,
pin_memory=True)
##### Model #####
ckpt = torch.load(config['load'])
inner_args = utils.config_inner_args(config.get('inner_args'))
model = models.load(ckpt, load_clf=(not inner_args['reset_classifier']))
if args.efficient:
model.go_efficient()
if config.get('_parallel'):
model = nn.DataParallel(model)
utils.log('num params: {}'.format(utils.compute_n_params(model)))
##### Evaluation #####
model.eval()
aves_va = utils.AverageMeter()
va_lst = []
for epoch in range(1, config['epoch'] + 1):
for data in tqdm(loader, leave=False):
x_shot, x_query, y_shot, y_query = data
x_shot, y_shot = x_shot.cuda(), y_shot.cuda()
x_query, y_query = x_query.cuda(), y_query.cuda()
if inner_args['reset_classifier']:
if config.get('_parallel'):
model.module.reset_classifier()
else:
model.reset_classifier()
logits = model(x_shot,
x_query,
y_shot,
inner_args,
meta_train=False)
logits = logits.view(-1, config['test']['n_way'])
labels = y_query.view(-1)
pred = torch.argmax(logits, dim=1)
acc = utils.compute_acc(pred, labels)
aves_va.update(acc, 1)
va_lst.append(acc)
print('test epoch {}: acc={:.2f} +- {:.2f} (%)'.format(
epoch,
aves_va.item() * 100,
utils.mean_confidence_interval(va_lst) * 100))
pred = model(datas)
# calculate loss
loss = F.cross_entropy(pred, labels).item()
test_loss.append(loss)
total_loss = sum(test_loss) / len(test_loss)
# calculate accuracy
acc = (pred.argmax(1) == labels
).type(torch.cuda.FloatTensor if torch.cuda.is_available(
) else torch.FloatTensor).mean().item()
test_acc.append(acc)
total_acc = sum(test_acc) / len(test_acc)
printer("test", e, args.num_epochs, i + 1, len(test_loader), loss,
total_loss, acc * 100, total_acc * 100)
# tensorboard
writer.add_scalar("Loss/test", loss, n_iter_test)
writer.add_scalar("Accuracy/test", acc, n_iter_test)
n_iter_test += 1
# not saving
if total_acc > best:
best = total_acc
m, h = mean_confidence_interval(test_acc)
print("Best: {:.2f}% ({:.2f}+-{:.2f}) ".format(best * 100, m * 100, h))
lr_scheduler.step()
