def load(opt):
model_opt = filter_opt(opt, 'model')
model_name = model_opt['model_name']
del model_opt['model_name']
return get_model(model_name, model_opt)
def load(opt):
"""
Load the model for the options
"""
model_opt = filter_opt(opt, 'model')
model_name = model_opt['model_name']
del model_opt['model_name']
return get_model(model_name, model_opt)
def main(opt):
# load model
model = model_utils.load(opt)
state_dict = torch.load(opt['model.model_path'])
model.load_state_dict(state_dict)
model.eval()
# load opts
model_opt_file = os.path.join(os.path.dirname(opt['model.model_path']), 'opt.json')
with open(model_opt_file, 'r') as f:
model_opt = json.load(f)
# Postprocess arguments
model_opt['model.x_dim'] = map(int, model_opt['model.x_dim'].split(','))
model_opt['log.fields'] = model_opt['log.fields'].split(',')
# construct data
data_opt = { 'data.' + k: v for k,v in filter_opt(model_opt, 'data').items() }
episode_fields = {
'data.test_way': 'data.way',
'data.test_shot': 'data.shot',
'data.test_query': 'data.query',
'data.test_episodes': 'data.train_episodes'
}
for k,v in episode_fields.items():
if opt[k] != 0:
data_opt[k] = opt[k]
elif model_opt[k] != 0:
data_opt[k] = model_opt[k]
else:
data_opt[k] = model_opt[v]
print("Evaluating {:d}-way, {:d}-shot with {:d} query examples/class over {:d} episodes".format(
data_opt['data.test_way'], data_opt['data.test_shot'],
data_opt['data.test_query'], data_opt['data.test_episodes']))
torch.manual_seed(1234)
if data_opt['data.cuda']:
torch.cuda.manual_seed(1234)
data = data_utils.load(data_opt, ['test'])
if data_opt['data.cuda']:
model.cuda()
meters = { field: tnt.meter.AverageValueMeter() for field in model_opt['log.fields'] }
model_utils.evaluate(model, data['test'], meters, desc="test")
for field,meter in meters.items():
mean, std = meter.value()
print("test {:s}: {:0.6f} +/- {:0.6f}".format(field, mean, 1.96 * std / math.sqrt(data_opt['data.test_episodes'])))
def loader(opt, splits):
ret = {}
for split in splits:
if split in ['val', 'test'] and opt['data.test_way'] != 0:
n_way = opt['data.test_way']
else:
n_way = opt['data.way']
if split in ['val', 'test'] and opt['data.test_shot'] != 0:
n_support = opt['data.test_shot']
else:
n_support = opt['data.shot']
if split in ['val', 'test'] and opt['data.test_query'] != 0:
n_query = opt['data.test_query']
else:
n_query = opt['data.query']
if split in ['val', 'test']:
n_episodes = opt['data.test_episodes']
else:
n_episodes = opt['data.train_episodes']
if opt['data.dataset'] == 'googlespeech':
speech_args = filter_opt(opt, 'speech')
data_dir = os.path.join(os.path.dirname(__file__),
'../../data/speech_commands/core')
class_file = os.path.join(os.path.dirname(__file__),
'../../data/speech_commands/core',
split + '.txt')
ds = FewShotSpeechDataset(data_dir, class_file, n_support, n_query,
opt['data.cuda'], speech_args)
if opt['data.sequential']:
sampler = SequentialBatchSampler(len(ds))
else:
sampler = EpisodicSpeechBatchSampler(
len(ds),
n_way,
n_episodes,
include_silence=opt['speech.include_silence'],
include_unknown=opt['speech.include_unknown'])
# use num_workers=0, otherwise may receive duplicate episodes
ret[split] = torch.utils.data.DataLoader(ds,
batch_sampler=sampler,
num_workers=0)
return ret
def main(opt):
# load model
model = torch.load(opt['model.model_path'])
# def get_image_input_hook(self, input, output):
# # print(len(input))
# # print(type(input))
# # print(len(output))
# # print(type(output))
# print("in:",input[0].data.cpu().numpy()[0].shape)
# print("out:",output.data.cpu().numpy()[0].shape)
# viz.image(input[0].data.cpu().numpy()[0])
# for i in output.data.cpu().numpy()[0]:
# viz.image(i)
# model.encoder[0][0].register_forward_hook(get_image_input_hook)
model.eval()
# load opts
model_opt_file = os.path.join(os.path.dirname(opt['model.model_path']),
'opt.json')
with open(model_opt_file, 'r') as f:
model_opt = json.load(f)
# Postprocess arguments
model_opt['model.x_dim'] = map(int, model_opt['model.x_dim'].split(','))
model_opt['log.fields'] = model_opt['log.fields'].split(',')
# construct data
data_opt = {
'data.' + k: v
for k, v in filter_opt(model_opt, 'data').items()
}
episode_fields = {
'data.test_way': 'data.way',
'data.test_shot': 'data.shot',
'data.test_query': 'data.query',
'data.test_episodes': 'data.train_episodes'
}
for k, v in episode_fields.items():
if opt[k] != 0:
data_opt[k] = opt[k]
elif model_opt[k] != 0:
data_opt[k] = model_opt[k]
else:
data_opt[k] = model_opt[v]
print(
"Evaluating {:d}-way, {:d}-shot with {:d} query examples/class over {:d} episodes"
.format(data_opt['data.test_way'], data_opt['data.test_shot'],
data_opt['data.test_query'], data_opt['data.test_episodes']))
torch.manual_seed(1234)
if data_opt['data.cuda']:
torch.cuda.manual_seed(1234)
print(data_opt)
data = data_utils.load(data_opt, ['test'])
if data_opt['data.cuda']:
model.cuda()
meters = {
field: tnt.meter.AverageValueMeter()
for field in model_opt['log.fields']
}
model_utils.evaluate(model, data['test'], meters, desc="test")
for field, meter in meters.items():
mean, std = meter.value()
print("test {:s}: {:0.6f} +/- {:0.6f}".format(
field, mean,
1.96 * std / math.sqrt(data_opt['data.test_episodes'])))
def load(opt, key='model_name'):
model_opt = filter_opt(opt, 'model')
model_name = model_opt[key]
del model_opt[key]
return get_model(model_name, model_opt)
def main(opt):
# load model
model = torch.load(opt['model.model_path'])
model.eval()
# load opts
model_opt_file = os.path.join(os.path.dirname(opt['model.model_path']),
'opt.json')
with open(model_opt_file, 'r') as f:
model_opt = json.load(f)
# Postprocess arguments
model_opt['model.x_dim'] = map(int, model_opt['model.x_dim'].split(','))
model_opt['log.fields'] = model_opt['log.fields'].split(',')
# construct data
data_opt = {
'data.' + k: v
for k, v in filter_opt(model_opt, 'data').items()
}
episode_fields = {
'data.test_way': 'data.way',
'data.test_shot': 'data.shot',
'data.test_query': 'data.query',
'data.test_episodes': 'data.train_episodes'
}
for k, v in episode_fields.items():
if opt[k] != 0:
data_opt[k] = opt[k]
elif model_opt[k] != 0:
data_opt[k] = model_opt[k]
else:
data_opt[k] = model_opt[v]
print(
"Evaluating {:d}-way, {:d}-shot with {:d} query examples/class over {:d} episodes"
.format(data_opt['data.test_way'], data_opt['data.test_shot'],
data_opt['data.test_query'], data_opt['data.test_episodes']))
torch.manual_seed(1234)
if data_opt['data.cuda']:
torch.cuda.manual_seed(1234)
data = data_utils.load(data_opt, ['test'])
if data_opt['data.cuda']:
model.cuda()
meters = {
field: tnt.meter.AverageValueMeter()
for field in model_opt['log.fields']
}
if opt['stage'] == 'protonet':
_, class_acc, class_prec, prec_micro = model_utils.evaluate(
model,
data['test'],
meters,
stage='protonet',
desc="test",
evaluation=True)
else:
_, class_acc, class_prec, prec_micro = model_utils.evaluate(
model,
data['test'],
meters,
stage='feat',
desc='test',
evaluation=True)
for field, meter in meters.items():
mean, std = meter.value()
print("test {:s}: {:0.6f} +/- {:0.6f}".format(
field, mean,
1.96 * std / math.sqrt(data_opt['data.test_episodes'])))
mean_prec = 0
n = 0
for k in class_acc.keys():
print('class {} acc: {:0.4f}'.format(k, class_acc[k]))
for k in class_prec.keys():
mean_prec += class_prec[k]
n += 1
print('class {} prec: {:0.4f}'.format(k, class_prec[k]))
mean_prec = mean_prec / n
print('Average prec(macro): {:0.4f}; Average prec(micro): {:0.4f}'.format(
mean_prec, prec_micro))
def main(opt, augment_stn):
# load model
model = torch.load(opt['model.model_path'])
model.eval()
# load opts
model_opt_file = os.path.join(os.path.dirname(opt['model.model_path']),
'opt.json')
with open(model_opt_file, 'r') as f:
model_opt = json.load(f)
# Postprocess arguments
model_opt['model.x_dim'] = map(int, model_opt['model.x_dim'].split(','))
model_opt['log.fields'] = model_opt['log.fields'].split(',')
if model_opt['run_stn']:
print("Loading STN here")
try:
stn_model = torch.load(opt['model.model_path'].replace(
'.pt', '_stn.pt'))
stn_model.eval()
except:
stn_model = None
else:
stn_model = None
# Augment overrides it anyway
if not augment_stn:
stn_model = None
# construct data
data_opt = {
'data.' + k: v
for k, v in filter_opt(model_opt, 'data').items()
}
episode_fields = {
'data.test_way': 'data.way',
'data.test_shot': 'data.shot',
'data.test_query': 'data.query',
'data.test_episodes': 'data.train_episodes'
}
for k, v in episode_fields.items():
if opt[k] != 0:
data_opt[k] = opt[k]
elif model_opt[k] != 0:
data_opt[k] = model_opt[k]
else:
data_opt[k] = model_opt[v]
print(
"Evaluating {:d}-way, {:d}-shot with {:d} query examples/class over {:d} episodes"
.format(data_opt['data.test_way'], data_opt['data.test_shot'],
data_opt['data.test_query'], data_opt['data.test_episodes']))
torch.manual_seed(opt['seed'])
if data_opt['data.cuda']:
torch.cuda.manual_seed(opt['seed'])
data = data_utils.load(data_opt, ['test'])
if data_opt['data.cuda']:
model.cuda()
if stn_model is not None:
stn_model.cuda()
meters = {
field: tnt.meter.AverageValueMeter()
for field in model_opt['log.fields']
}
model_utils.evaluate(model, stn_model, data['test'], meters, desc="test")
for field, meter in meters.items():
mean, std = meter.value()
print("test {:s}: {:0.6f} +/- {:0.6f}".format(
field, mean,
1.96 * std / math.sqrt(data_opt['data.test_episodes'])))
def main(opt):
# load model
#model = torch.load(opt['model.model_path'])
print('opt:', opt)
model = model_utils.load(opt)
model.eval()
# load opts
model_opt_file = os.path.join(os.path.dirname(opt['model.model_path']),
'opt.json')
with open(model_opt_file, 'r') as f:
model_opt = json.load(f)
# Postprocess arguments
model_opt['model.x_dim'] = map(int, model_opt['model.x_dim'].split(','))
model_opt['log.fields'] = model_opt['log.fields'].split(',')
# construct data
data_opt = {
'data.' + k: v
for k, v in filter_opt(model_opt, 'data').items()
}
episode_fields = {
'data.test_way': 'data.way',
'data.test_shot': 'data.shot',
'data.test_query': 'data.query',
'data.test_episodes': 'data.train_episodes'
}
for k, v in episode_fields.items():
if opt[k] != 0:
data_opt[k] = opt[k]
elif model_opt[k] != 0:
data_opt[k] = model_opt[k]
else:
data_opt[k] = model_opt[v]
# added for birds dataset
data_opt['data.test_way'] = 5
data_opt['data.test_shot'] = 5
data_opt['data.test_query'] = (41 - 5) # max amount possible
data_opt[
'data.test_episodes'] = 1000 # average over 1000 randomly generated episodes
print(
"Evaluating {:d}-way, {:d}-shot with {:d} query examples/class over {:d} episodes"
.format(data_opt['data.test_way'], data_opt['data.test_shot'],
data_opt['data.test_query'], data_opt['data.test_episodes']))
torch.manual_seed(1234)
if data_opt['data.cuda']:
torch.cuda.manual_seed(1234)
#data = data_utils.load(data_opt, ['test'])
test_way = data_opt['data.test_way']
test_shot = data_opt['data.test_shot']
test_query = data_opt['data.test_query']
test_episodes = data_opt['data.test_episodes']
print('test way:', test_way, 'test shot:', test_shot, 'test query:',
test_query, 'test_episodes', test_episodes)
# figure out how to go through all the data with 5 shot
data = PytorchBirdsDataLoader(n_episodes=test_episodes,
n_way=test_way,
n_query=test_query,
n_support=test_shot)
if data_opt['data.cuda']:
model.cuda()
meters = {
field: tnt.meter.AverageValueMeter()
for field in model_opt['log.fields']
}
model_utils.evaluate(model, data, meters, desc="test")
for field, meter in meters.items():
mean, std = meter.value()
print("test {:s}: {:0.6f} +/- {:0.6f}".format(
field, mean,
1.96 * std / math.sqrt(data_opt['data.test_episodes'])))