def step_three():
data_loader = DataLoader(args)
Checkpoint.CHECKPOINT_DIR_NAME = args.checkpoint_dir_name
checkpoint_path = os.path.join("./experiment",
Checkpoint.CHECKPOINT_DIR_NAME, "best")
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
if args.cuda_use:
seq2seq = seq2seq.cuda()
seq2seq.eval()
evaluator = Evaluator(vocab_dict=data_loader.vocab_dict,
vocab_list=data_loader.vocab_list,
decode_classes_dict=data_loader.decode_classes_dict,
decode_classes_list=data_loader.decode_classes_list,
loss=NLLLoss(),
cuda_use=args.cuda_use)
test_temp_acc, test_ans_acc = evaluator.evaluate(
model=seq2seq,
data_loader=data_loader,
data_list=data_loader.math57k_data_list,
template_flag=False,
batch_size=64,
evaluate_type=0,
use_rule=True,
mode=args.mode)
print(test_temp_acc, test_ans_acc)
def step_one_test():
data_loader = DataLoader(args)
#Checkpoint.CHECKPOINT_DIR_NAME = "0120_0030"
Checkpoint.CHECKPOINT_DIR_NAME = args.checkpoint_dir_name
checkpoint_path = os.path.join("./experiment",
Checkpoint.CHECKPOINT_DIR_NAME, "best")
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
if args.cuda_use:
seq2seq = seq2seq.cuda()
seq2seq.eval()
evaluator = Evaluator(vocab_dict=data_loader.vocab_dict,
vocab_list=data_loader.vocab_list,
decode_classes_dict=data_loader.decode_classes_dict,
decode_classes_list=data_loader.decode_classes_list,
loss=NLLLoss(),
cuda_use=args.cuda_use)
name = args.run_flag
if name == 'test_23k':
test_temp_acc, test_ans_acc = evaluator.evaluate(
model=seq2seq,
data_loader=data_loader,
data_list=data_loader.math23k_test_list,
template_flag=True,
batch_size=64,
evaluate_type=0,
use_rule=False,
mode=args.mode,
post_flag=args.post_flag,
name_save=name)
print(test_temp_acc, test_ans_acc)
def test(args, test_dataset="test"):
seq2seq, data_loader = create_model(args)
resume_checkpoint = Checkpoint.load(model_only=True)
seq2seq.load_state_dict(resume_checkpoint.model)
if args.use_cuda:
seq2seq = seq2seq.cuda()
evaluator = Evaluator(
class_dict=data_loader.class_dict,
class_list=data_loader.class_list,
use_cuda=args.use_cuda
)
if test_dataset == "test":
test_dataset = data_loader.test_list
elif test_dataset == "train":
test_dataset = data_loader.train_list
seq2seq.eval()
with torch.no_grad():
test_temp_acc, test_ans_acc = evaluator.evaluate(
model=seq2seq,
data_loader=data_loader,
data_list=test_dataset,
template_flag=True,
template_len=False,
batch_size=1,
beam_width=args.beam,
test_log=args.test_log,
print_probability=True
)
logging.info(f"temp_acc: {test_temp_acc}, ans_acc: {test_ans_acc}")
return
def test(net, test_loader, epoch):
net.eval()
test_step = len(test_loader)
print('\nEvaluating...')
with torch.no_grad():
evaluator = Evaluator()
for i, sample in enumerate(test_loader):
for key in sample:
sample[key] = sample[key].cuda()
output = net(sample)
evaluator.add(output, sample)
if (i + 1) % 100 == 0:
print('Val Step [{}/{}]'.format(i + 1, test_step))
results = evaluator.evaluate()
print('Epoch {}/{}'.format(epoch + 1, args.epoch))
print(
'| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |'
)
print(
'| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |'
.format(results['lm_dist'][0], results['lm_dist'][1],
results['lm_dist'][2], results['lm_dist'][3],
results['lm_dist'][4], results['lm_dist'][5],
results['lm_dist'][6], results['lm_dist'][7],
results['lm_dist_all']))
def evaluate(split='valid'):
y_true = eval(split + '_data')
# Ttensor = eval(split + '_tensor')
model.eval()
y_score, _, _ = model(train_tensor)
y_score.detach_()
y_score = y_score.squeeze(0)
y_score[train_data.row, train_data.col] = 0
_, rec_items = torch.topk(y_score, args.N, dim=1)
# y_pred = torch.gather(Ttensor, 1, rec_items).cpu().numpy()
run = sort2query(rec_items[:, 0:args.N])
test = csr2test(y_true.tocsr())
evaluator = Evaluator({'recall', 'map_cut'})
evaluator.evaluate(run, test)
result = evaluator.show([
'recall_5', 'recall_10', 'recall_15', 'recall_20', 'map_cut_5',
'map_cut_10', 'map_cut_15', 'map_cut_20'
])
print(result)
def test(net, test_loader, epoch):
net.eval()
test_step = len(test_loader)
print('\nEvaluating...')
with torch.no_grad():
evaluator = Evaluator()
for i, sample in enumerate(test_loader):
for key in sample:
if key[0:2] != 'D_':
sample[key] = sample[key].cuda()
output = net(sample)
evaluator.add(output, sample)
if (i + 1) % 100 == 0:
print('Val Step [{}/{}]'.format(i + 1, test_step)) # j + 1 ?
results = evaluator.evaluate()
print('Epoch {}/{}'.format(epoch + 1, args.epoch))
print(
'| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |'
)
print(
'| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |'
.format(results['lm_dist'][0], results['lm_dist'][1],
results['lm_dist'][2], results['lm_dist'][3],
results['lm_dist'][4], results['lm_dist'][5],
results['lm_dist'][6], results['lm_dist'][7],
results['lm_dist_all']))
file = open('results_lr_%.4f_base_%d_de_%d_g_%.2f.txt'% \
(args.learning_rate, \
args.base_epoch, \
args.decay_epoch, \
args.gamma), 'a')
file.write('Epoch {}\n'.format(args.base_epoch + epoch + 1))
file.write(
'| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |\n'
)
file.write(
'| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |\n\n'
.format(results['lm_dist'][0], results['lm_dist'][1],
results['lm_dist'][2], results['lm_dist'][3],
results['lm_dist'][4], results['lm_dist'][5],
results['lm_dist'][6], results['lm_dist'][7],
results['lm_dist_all']))
file.close()
def test(net, test_loader, epoch):
net.eval()
test_step = len(test_loader)
print('\nEvaluating...')
with torch.no_grad():
evaluator = Evaluator()
for i, (sample, img) in enumerate(test_loader):
#import matplotlib.pyplot as plt
#print(len(sample['image_original']))
#plt.imshow(sample['image_original'].numpy().swapaxes(0, 2))
#plt.show()
#image_name = sample['image_name']
#print(image_name)
for key in sample:
if key != 'image_name':
sample[key] = sample[key].cuda()
output = net(sample)
#print(type(output))
pos_map = output['lm_pos_map']
#print(pos_map.size())
#resh = pos_map.reshape(50, 8, -1)
#print(resh.size())
evaluator.add(output, sample, img)
if (i + 1) % 100 == 0:
print('Val Step [{}/{}]'.format(i + 1, test_step))
results = evaluator.evaluate()
print('Epoch {}/{}'.format(epoch + 1, args.epoch))
print(
'| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |'
)
print(
'| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |'
.format(results['lm_dist'][0], results['lm_dist'][1],
results['lm_dist'][2], results['lm_dist'][3],
results['lm_dist'][4], results['lm_dist'][5],
results['lm_dist'][6], results['lm_dist'][7],
results['lm_dist_all']))
def test(net, test_loader, epoch):
net.eval()
test_step = len(test_loader)
print('\nEvaluating...')
with torch.no_grad():
evaluator = Evaluator()
pbar2 = tqdm(test_loader)
for i, sample in enumerate(pbar2):
iter_start_time = time.time()
for key in sample:
sample[key] = sample[key].cuda()
output = net(sample)
evaluator.add(output, sample)
t = time.time() - iter_start_time
if (i + 1) % 100 == 0:
tqdm.write('Val Step [{}/{}], Time:{:.3f}'.format(
i + 1, test_step, t))
results = evaluator.evaluate()
print('Epoch {}/{}'.format(epoch + 1, args.epoch))
print('lm_dist_all: {:.5f} '.format(results['lm_dist_all']))
if __name__ == "__main__":
norm_layer = torch.nn.modules.instancenorm.InstanceNorm2d
model = G(3, 3, 64,norm_layer=norm_layer , n_blocks=9)
device = 0
weights = "./weights/cyclegan/stb_net_G.pth"
cpm2d = './weights/cpm/stb_hpm2d.pth'
cpm3d = './weights/cpm/stb_hpm3d.pth'
evaluate = Evaluator(model, weights, cpm2d, cpm3d, device)
opt = edict()
opt.dataroot = "./dataset/stb-dataset/test"
opt.isTrain = False
dataset = STBdataset(opt)
for i in tqdm(range(len(dataset))):
sample = dataset[i]
evaluate.feed(sample['xyz'], sample['A']*255.0)
results = evaluate.evaluate()
print(f"STB results: {results}")
opt.dataroot = "./dataset/rhd-dataset/test"
weights = "./weights/cyclegan/rhd_net_G.pth"
cpm2d = "./weights/cpm/rhd_hpm2d.pth"
dataset = RHDdataset(opt)
evaluate = Evaluator(model, weights, cpm2d, cpm3d, device)
for i in tqdm(range(len(dataset))):
sample = dataset[i]
evaluate.feed(sample['xyz'], sample['A']*255.0)
results = evaluate.evaluate()
print(f"rhd results: {results}")
optimizer = optim.Adam(model.parameters(), lr=args.lr)
for epoch in range(1, args.maxiter + 1):
train(epoch)
# test()
model.eval()
score, _, _ = model(Rtensor)
score = score.squeeze(0)
score[R.row, R.col] = 0
_, idx = torch.sort(score, 1, True)
run = sort2query(idx[:, 0:args.N])
test = csr2test(T.tocsr())
evaluator = Evaluator({'recall'})
evaluator.evaluate(run, test)
result = evaluator.show(
['recall_5', 'recall_10', 'recall_15', 'recall_20'])
print(result)
line = 'cVAE\t{}\t{}\t{}\t{}\t0'.format(args.data, args.alpha, args.beta, len(args.layer))
for _, value in result.items():
line += '\t{:.5f}'.format(value)
line += '\r\n'
file = open('result', 'a')
file.write(line)
file.close()
if args.save:
name = 'cvae' if args.rating else 'fvae'
path = directory + '/model/' + name
for l in args.layer:
# For parallel evaluation
print('[+] Initializing evaluators')
evaluators = Evaluator(config)
# [Evaluator(config) for _ in range(2 * N)]
# First evaluation (sequential)
# delayed_functions = []
# for i in range(2 * N):
# sf = skill_factor[i]
# delayed_functions.append(delayed(evaluators[i].evaluate)(population[i], sf))
# fitnesses = Parallel(n_jobs=cpu_count())(delayed_functions)
# for i in range(2 * N):
# sf = skill_factor[i]
# factorial_cost[i, sf] = fitnesses[i]
# scalar_fitness = calculate_scalar_fitness(factorial_cost)
fitnesses = [evaluators.evaluate(x, 0) for x in population]
# Evolve
# iterator = trange(T)
iterator = trange(10)
for t in iterator:
# permute current population
permutation_index = np.random.permutation(N)
population[:N] = population[:N][permutation_index]
# skill_factor[:N] = skill_factor[:N][permutation_index]
# factorial_cost[:N] = factorial_cost[:N][permutation_index]
# factorial_cost[N:] = np.inf
# As we consider all the population as parents, we don't samplt P^{s}
if t % TrInt == 0:
class SupervisedTrainer(object):
def __init__(self, class_dict, class_list, use_cuda):
self.test_train_every = 10
self.print_every = 30
self.use_cuda = use_cuda
self.pad_idx_in_class = class_dict['PAD_token']
loss_weight = torch.ones(len(class_dict))
loss_weight[self.pad_idx_in_class] = 0
self.loss = nn.NLLLoss(weight=loss_weight, reduction="sum")
if use_cuda:
self.loss = self.loss.cuda()
self.evaluator = Evaluator(class_dict=class_dict,
class_list=class_list,
use_cuda=use_cuda)
return
def _train_batch(self, input_variables, num_pos, input_lengths,
span_length, target_variables, tree, model, batch_size):
decoder_outputs, _, _ = model(input_variable=input_variables,
num_pos=num_pos,
input_lengths=input_lengths,
span_length=span_length,
target_variable=target_variables,
tree=tree)
batch_size = span_length.size(0)
# loss
loss = 0
for step, step_output in enumerate(decoder_outputs):
loss += self.loss(step_output.contiguous().view(batch_size, -1),
target_variables[:, step].view(-1))
total_target_length = (target_variables !=
self.pad_idx_in_class).sum().item()
loss = loss / total_target_length
model.zero_grad()
loss.backward()
self.optimizer.step()
return loss.item()
def _train_epoches(self, data_loader, model, batch_size, start_epoch,
start_step, max_acc, n_epoch):
train_list = data_loader.train_list
test_list = data_loader.test_list
step = start_step
print_loss_total = 0
max_ans_acc = max_acc
for epoch_index, epoch in enumerate(range(start_epoch, n_epoch + 1)):
model.train()
batch_generator = data_loader.get_batch(train_list,
batch_size,
template_flag=True)
for batch_data_dict in batch_generator:
step += 1
input_variables = batch_data_dict['batch_span_encode_idx']
input_lengths = batch_data_dict['batch_span_encode_len']
span_length = batch_data_dict['batch_span_len']
tree = batch_data_dict["batch_tree"]
input_variables = [
torch.LongTensor(input_variable)
for input_variable in input_variables
]
input_lengths = [
torch.LongTensor(input_length)
for input_length in input_lengths
]
span_length = torch.LongTensor(span_length)
if self.use_cuda:
input_variables = [
input_variable.cuda()
for input_variable in input_variables
]
input_lengths = [
input_length.cuda() for input_length in input_lengths
]
span_length = span_length.cuda()
span_num_pos = batch_data_dict["batch_span_num_pos"]
word_num_poses = batch_data_dict["batch_word_num_poses"]
span_num_pos = torch.LongTensor(span_num_pos)
word_num_poses = [
torch.LongTensor(word_num_pos)
for word_num_pos in word_num_poses
]
if self.use_cuda:
span_num_pos = span_num_pos.cuda()
word_num_poses = [
word_num_pose.cuda()
for word_num_pose in word_num_poses
]
num_pos = (span_num_pos, word_num_poses)
target_variables = batch_data_dict['batch_decode_idx']
target_variables = torch.LongTensor(target_variables)
if self.use_cuda:
target_variables = target_variables.cuda()
loss = self._train_batch(input_variables=input_variables,
num_pos=num_pos,
input_lengths=input_lengths,
span_length=span_length,
target_variables=target_variables,
tree=tree,
model=model,
batch_size=batch_size)
print_loss_total += loss
if step % self.print_every == 0:
print_loss_avg = print_loss_total / self.print_every
print_loss_total = 0
logging.info(
f'step: {step}, Train loss: {print_loss_avg:.4f}')
if self.use_cuda:
torch.cuda.empty_cache()
self.scheduler.step()
model.eval()
with torch.no_grad():
test_temp_acc, test_ans_acc = self.evaluator.evaluate(
model=model,
data_loader=data_loader,
data_list=test_list,
template_flag=True,
template_len=True,
batch_size=batch_size,
)
if epoch_index % self.test_train_every == 0:
train_temp_acc, train_ans_acc = self.evaluator.evaluate(
model=model,
data_loader=data_loader,
data_list=train_list,
template_flag=True,
template_len=True,
batch_size=batch_size,
)
logging.info(
f"Epoch: {epoch}, Step: {step}, test_acc: {test_temp_acc:.3f}, {test_ans_acc:.3f}, train_acc: {train_temp_acc:.3f}, {train_ans_acc:.3f}"
)
else:
logging.info(
f"Epoch: {epoch}, Step: {step}, test_acc: {test_temp_acc:.3f}, {test_ans_acc:.3f}"
)
if test_ans_acc > max_ans_acc:
max_ans_acc = test_ans_acc
logging.info("saving checkpoint ...")
Checkpoint.save(epoch=epoch,
step=step,
max_acc=max_ans_acc,
model=model,
optimizer=self.optimizer,
scheduler=self.scheduler,
best=True)
else:
Checkpoint.save(epoch=epoch,
step=step,
max_acc=max_ans_acc,
model=model,
optimizer=self.optimizer,
scheduler=self.scheduler,
best=False)
return
def train(self,
model,
data_loader,
batch_size,
n_epoch,
resume=False,
optim_lr=1e-3,
optim_weight_decay=1e-5,
scheduler_step_size=60,
scheduler_gamma=0.6):
start_epoch = 1
start_step = 0
max_acc = 0
self.optimizer = optim.Adam(model.parameters(),
lr=optim_lr,
weight_decay=optim_weight_decay)
self.scheduler = optim.lr_scheduler.StepLR(
self.optimizer,
step_size=scheduler_step_size,
gamma=scheduler_gamma)
if resume:
resume_checkpoint = Checkpoint.load(model_only=False)
model.load_state_dict(resume_checkpoint.model)
resume_optimizer = resume_checkpoint.optimizer
resume_scheduler = resume_checkpoint.scheduler
if resume_optimizer is not None:
start_epoch = resume_checkpoint.epoch
start_step = resume_checkpoint.step
max_acc = resume_checkpoint.max_acc
self.optimizer.load_state_dict(resume_optimizer)
self.scheduler.load_state_dict(resume_scheduler)
self._train_epoches(data_loader=data_loader,
model=model,
batch_size=batch_size,
start_epoch=start_epoch,
start_step=start_step,
max_acc=max_acc,
n_epoch=n_epoch)
return
