def gen_best_23_error():
data_loader = DataLoader()
Checkpoint.CHECKPOINT_DIR_NAME = args.checkpoint_dir_name
checkpoint_path = os.path.join("./experiment", Checkpoint.CHECKPOINT_DIR_NAME, 'best')
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
if args.cuda_use:
seq2seq = seq2seq.cuda()
seq2seq.eval()
emb_model = seq2seq.encoder.embedding
emb_np = emb_model.weight.cpu().data.numpy()
np.save("./data/rl_train_data/emb.npy", emb_np)
evaluator = Evaluator(vocab_dict = data_loader.vocab_dict,
vocab_list = data_loader.vocab_list,
decode_classes_dict = data_loader.decode_classes_dict,
decode_classes_list = data_loader.decode_classes_list,
loss = NLLLoss(),
cuda_use = args.cuda_use)
evaluator.gen_rl_data(model = seq2seq,
data_loader = data_loader,
data_list = data_loader.math23k_train_list,
template_flag = False,
batch_size = 16,
evaluate_type = 0,
use_rule = False,
mode = args.mode,
filename = args.load_name)
class TestEvaluator(TestCase):
def setUp(self):
self.info = {0: {'amount_correct': 400,
'amount_total': 600},
1: {'amount_correct': 100,
'amount_total': 1000}}
self.evaluator = Evaluator(self.info)
def test_inits_correctly(self):
self.assertEqual(self.info, self.evaluator.class_info)
def test_update_accuracy(self):
self.evaluator.update_accuracy(0, 200, 800)
self.assertEqual(self.evaluator.class_info[0].amount_correct, 200)
self.assertEqual(self.evaluator.class_info[0].amount_total, 800)
def test_percent_correct(self):
class_info = self.evaluator.class_info[1]
self.assertEqual(class_info.amount_correct / class_info.amount_total, .1)
def test_total_percent_correct(self):
self.assertEqual(self.evaluator.total_percent_correct(), 500/1600)
def evaluate_fn(agent_dir, output_dir, seeds, port):
agent = agent_dir.split('/')[-1]
if not check_dir(agent_dir):
logging.error('Evaluation: %s does not exist!' % agent)
return
# load config file for env
config_dir = find_file(agent_dir + '/data/')
if not config_dir:
return
config = configparser.ConfigParser()
config.read(config_dir)
# init env
env, greedy_policy = init_env(config['ENV_CONFIG'],
port=port,
naive_policy=True)
env.init_test_seeds(seeds)
# load model for agent
if agent != 'greedy':
# init centralized or multi agent
model = init_agent(env, config['MODEL_CONFIG'], 0, 0)
if model is None:
return
if not model.load(agent_dir + '/model/'):
return
else:
model = greedy_policy
# collect evaluation data
evaluator = Evaluator(env, model, output_dir)
evaluator.run()
def test(net, test_loader, epoch):
net.eval()
test_step = len(test_loader)
print('\nEvaluating...')
with torch.no_grad():
evaluator = Evaluator()
for i, sample in enumerate(test_loader):
for key in sample:
sample[key] = sample[key].cuda()
output = net(sample)
evaluator.add(output, sample)
if (i + 1) % 100 == 0:
print('Val Step [{}/{}]'.format(i + 1, test_step))
results = evaluator.evaluate()
print('Epoch {}/{}'.format(epoch + 1, args.epoch))
print(
'| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |'
)
print(
'| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |'
.format(results['lm_dist'][0], results['lm_dist'][1],
results['lm_dist'][2], results['lm_dist'][3],
results['lm_dist'][4], results['lm_dist'][5],
results['lm_dist'][6], results['lm_dist'][7],
results['lm_dist_all']))
def test(args, test_dataset="test"):
seq2seq, data_loader = create_model(args)
resume_checkpoint = Checkpoint.load(model_only=True)
seq2seq.load_state_dict(resume_checkpoint.model)
if args.use_cuda:
seq2seq = seq2seq.cuda()
evaluator = Evaluator(
class_dict=data_loader.class_dict,
class_list=data_loader.class_list,
use_cuda=args.use_cuda
)
if test_dataset == "test":
test_dataset = data_loader.test_list
elif test_dataset == "train":
test_dataset = data_loader.train_list
seq2seq.eval()
with torch.no_grad():
test_temp_acc, test_ans_acc = evaluator.evaluate(
model=seq2seq,
data_loader=data_loader,
data_list=test_dataset,
template_flag=True,
template_len=False,
batch_size=1,
beam_width=args.beam,
test_log=args.test_log,
print_probability=True
)
logging.info(f"temp_acc: {test_temp_acc}, ans_acc: {test_ans_acc}")
return
def step_three():
data_loader = DataLoader(args)
Checkpoint.CHECKPOINT_DIR_NAME = args.checkpoint_dir_name
checkpoint_path = os.path.join("./experiment",
Checkpoint.CHECKPOINT_DIR_NAME, "best")
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
if args.cuda_use:
seq2seq = seq2seq.cuda()
seq2seq.eval()
evaluator = Evaluator(vocab_dict=data_loader.vocab_dict,
vocab_list=data_loader.vocab_list,
decode_classes_dict=data_loader.decode_classes_dict,
decode_classes_list=data_loader.decode_classes_list,
loss=NLLLoss(),
cuda_use=args.cuda_use)
test_temp_acc, test_ans_acc = evaluator.evaluate(
model=seq2seq,
data_loader=data_loader,
data_list=data_loader.math57k_data_list,
template_flag=False,
batch_size=64,
evaluate_type=0,
use_rule=True,
mode=args.mode)
print(test_temp_acc, test_ans_acc)
def step_one_test():
data_loader = DataLoader(args)
#Checkpoint.CHECKPOINT_DIR_NAME = "0120_0030"
Checkpoint.CHECKPOINT_DIR_NAME = args.checkpoint_dir_name
checkpoint_path = os.path.join("./experiment",
Checkpoint.CHECKPOINT_DIR_NAME, "best")
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
if args.cuda_use:
seq2seq = seq2seq.cuda()
seq2seq.eval()
evaluator = Evaluator(vocab_dict=data_loader.vocab_dict,
vocab_list=data_loader.vocab_list,
decode_classes_dict=data_loader.decode_classes_dict,
decode_classes_list=data_loader.decode_classes_list,
loss=NLLLoss(),
cuda_use=args.cuda_use)
name = args.run_flag
if name == 'test_23k':
test_temp_acc, test_ans_acc = evaluator.evaluate(
model=seq2seq,
data_loader=data_loader,
data_list=data_loader.math23k_test_list,
template_flag=True,
batch_size=64,
evaluate_type=0,
use_rule=False,
mode=args.mode,
post_flag=args.post_flag,
name_save=name)
print(test_temp_acc, test_ans_acc)
def evaluate_fn(agent_dir, output_dir, seeds, port, demo, policy_type):
agent = agent_dir.split('/')[-1]
if not check_dir(agent_dir):
logging.error('Evaluation: %s does not exist!' % agent)
return
# load config file for env
config_dir = find_file(agent_dir + '/data/')
if not config_dir:
return
config = configparser.ConfigParser()
config.read(config_dir)
# init env
env, greedy_policy = init_env(config['ENV_CONFIG'],
port=port,
naive_policy=True)
logging.info(
'Evaluation: s dim: %d, a dim %d, s dim ls: %r, a dim ls: %r' %
(env.n_s, env.n_a, env.n_s_ls, env.n_a_ls))
env.init_test_seeds(seeds)
# load model for agent
if agent != 'greedy':
# init centralized or multi agent
if agent == 'a2c':
model = A2C(env.n_s, env.n_a, 0, config['MODEL_CONFIG'])
elif agent == 'ia2c':
model = IA2C(env.n_s_ls, env.n_a_ls, env.n_w_ls, 0,
config['MODEL_CONFIG'])
elif agent == 'ma2c':
model = MA2C(env.n_s_ls, env.n_a_ls, env.n_w_ls, env.n_f_ls, 0,
config['MODEL_CONFIG'])
elif agent == 'iqld':
model = IQL(env.n_s_ls,
env.n_a_ls,
env.n_w_ls,
0,
config['MODEL_CONFIG'],
seed=0,
model_type='dqn')
else:
model = IQL(env.n_s_ls,
env.n_a_ls,
env.n_w_ls,
0,
config['MODEL_CONFIG'],
seed=0,
model_type='lr')
if not model.load(agent_dir + '/model/'):
return
else:
model = greedy_policy
env.agent = agent
# collect evaluation data
evaluator = Evaluator(env,
model,
output_dir,
demo=demo,
policy_type=policy_type)
evaluator.run()
def gen_math57k_error():
data_loader = DataLoader()
Checkpoint.CHECKPOINT_DIR_NAME = args.checkpoint_dir_name
checkpoint_path = os.path.join("./experiment", Checkpoint.CHECKPOINT_DIR_NAME, args.load_name)
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
if args.cuda_use:
seq2seq = seq2seq.cuda()
seq2seq.eval()
evaluator = Evaluator(vocab_dict = data_loader.vocab_dict,
vocab_list = data_loader.vocab_list,
decode_classes_dict = data_loader.decode_classes_dict,
decode_classes_list = data_loader.decode_classes_list,
loss = NLLLoss(),
cuda_use = args.cuda_use)
evaluator.gen_rl_data(model = seq2seq,
data_loader = data_loader,
data_list = data_loader.math57k_data_list,
template_flag = False,
batch_size = 16,
evaluate_type = 0,
use_rule = True,
mode = args.mode,
filename = args.load_name)
def evaluate_fn(agent_dir, output_dir, seeds, port, demo):
agent = agent_dir.split('/')[-1]
if not check_dir(agent_dir):
logging.error('Evaluation: %s does not exist!' % agent)
return
# load config file
config_dir = find_file(agent_dir + '/data/')
if not config_dir:
return
config = configparser.ConfigParser()
config.read(config_dir)
# init env
env = init_env(config['ENV_CONFIG'], port=port)
env.init_test_seeds(seeds)
# load model for agent
model = init_agent(env, config['MODEL_CONFIG'], 0, 0)
if model is None:
return
model_dir = agent_dir + '/model/'
if not model.load(model_dir):
return
# collect evaluation data
evaluator = Evaluator(env, model, output_dir, gui=demo)
evaluator.run()
def setUp(self):
self.info = {0: {'amount_correct': 400,
'amount_total': 600},
1: {'amount_correct': 100,
'amount_total': 1000}}
self.evaluator = Evaluator(self.info)
def run(self):
# checkpoint
self.model = self.model.eval()
self.dataset, test_gallery = get_initial_test(
self.config, test=True) # return dataset instance
print("data set len is :", len(self.dataset))
data_gallery, vID_gallery, label_gallery = test_gallery[
0], test_gallery[1], test_gallery[2]
print("sample leve ----------->", self.config.test.sampler)
# dataloader define
self.data_loader = DataLoader(dataset=self.dataset,
batch_size=1,
sampler=SequentialSampler(self.dataset),
collate_fn=self.collate_fn,
num_workers=self.num_workers)
len_gallery = len(label_gallery)
feature_gallery = self.extract_gallery_feature(data_gallery,
len_gallery)
probe_feature = list()
probe_vID = list()
for seq, vID, label, _ in tqdm(self.data_loader):
seq = torch.from_numpy(seq).float().cuda()
# print(seq.size())
fc, out = self.model(seq)
n, num_bin = fc.size()
feat = fc.view(n, -1).data.cpu().numpy()
for ii in range(n):
feat[ii] = feat[ii] / np.linalg.norm(feat[ii])
probe_feature.append(feat)
probe_vID += vID
test_gallery = feature_gallery, vID_gallery, label_gallery
feature_probe = np.concatenate(probe_feature, 0)
test_probe = feature_probe, probe_vID
self.save_npy(feature_gallery, "feature_gallery.npy")
self.save_npy(vID_gallery, "vID_gallery.npy")
self.save_npy(label_gallery, "label_gallery.npy")
self.save_npy(feature_probe, "feature_probe.npy")
self.save_npy(probe_vID, "probe_vID.npy")
evaluation = Evaluator(test_gallery, test_probe, self.config)
evaluation.run()
def train(config, device, train_loader, epoch):
"""
train an epoch
Args:
config (EasyDict): configurations for training
device (torch.device): the GPU or CPU used for training
train_loader (torch.utils.data.DataLoader): a DataLoader instance object for training set
epoch (int): current epoch
Returns:
err (float): error rate
"""
losses = AverageMeter()
evaluator = Evaluator(config.num_classes)
model.train()
with tqdm(train_loader) as pbar:
pbar.set_description('Train Epoch {}'.format(epoch))
for step, (input_, target) in enumerate(train_loader):
# move data to device
input_ = torch.tensor(input_, device=device, dtype=torch.float32)
target = torch.tensor(target, device=device, dtype=torch.long)
# forward and compute loss
output = model(input_)
loss = criterion(output, target)
# backward and update params
optimizer.zero_grad()
loss.backward()
optimizer.step()
# record loss and show it in the pbar
losses.update(loss.item(), input_.size(0))
postfix = OrderedDict({
'batch_loss': f'{losses.val:6.4f}',
'running_loss': f'{losses.avg:6.4f}'
})
pbar.set_postfix(ordered_dict=postfix)
pbar.update()
# visualization with TensorBoard
total_iter = (epoch - 1) * len(train_loader) + step + 1
writer.add_scalar('training_loss', losses.val, total_iter)
# update confusion matrix
true = target.cpu().numpy()
pred = output.max(dim=1)[1].cpu().numpy()
evaluator.update_matrix(true, pred)
return evaluator.error()
def get_callbacks(output_subdir, training_model, prediction_model,
val_generator):
training_model_checkpoint = MultiGPUModelCheckpoint(
os.path.join(output_subdir, cfg.training_model_cp_filename),
training_model,
save_best_only=cfg.save_best_only,
monitor='loss',
mode='min')
prediction_model_checkpoint = PredictionModelCheckpoint(
os.path.join(output_subdir, cfg.prediction_model_cp_filename),
prediction_model,
save_best_only=cfg.save_best_only,
monitor='loss',
mode='min')
evaluator = Evaluator(prediction_model,
val_generator,
cfg.label_len,
cfg.characters,
cfg.optimizer,
period=cfg.val_iter_period)
lr_reducer = ReduceLROnPlateau(factor=cfg.lr_reduction_factor,
patience=3,
verbose=1,
min_lr=0.00001)
os.makedirs(cfg.tb_log, exist_ok=True)
tensorboard = TensorBoard(log_dir=cfg.tb_log)
return [
training_model_checkpoint, prediction_model_checkpoint, evaluator,
lr_reducer, tensorboard
]
def train(self, epoch, train_loader):
"""
使用训练集训练一个epoch
Args:
epoch: (int) 第几代训练
train_loader: (torch.utils.data.DataLoader) 训练数据加载器
Returns:
err: (float) error rate
"""
losses = AverageMeter()
evaluator = Evaluator(self.cfgs.num_classes)
self.model.train()
with tqdm(train_loader) as pbar:
pbar.set_description('Train Epoch {}'.format(epoch))
for step, (input_, target) in enumerate(train_loader):
# move data to device
input_ = torch.tensor(input_, device=self.device, dtype=torch.float32)
target = torch.tensor(target, device=self.device, dtype=torch.long)
# forward and compute loss
output = self.model(input_)
loss = self.criterion(output, target)
# backward and update params
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# record loss and show it in the pbar
losses.update(loss.item(), input_.size(0))
postfix = OrderedDict({'batch_loss': f'{losses.val:6.4f}', 'running_loss': f'{losses.avg:6.4f}'})
pbar.set_postfix(ordered_dict=postfix)
pbar.update()
# visualization with TensorBoard
total_iter = (epoch - 1) * len(train_loader) + step + 1
self.writer.add_scalar('training_loss', losses.val, total_iter)
# update confusion matrix
true = target.cpu().numpy()
pred = output.max(dim=1)[1].cpu().numpy()
evaluator.update_matrix(true, pred)
return evaluator.error()
def __init__(self, argv, preprocessor, model_api):
self.argv = argv
self.emb_loader = EmbeddingLoader(argv)
self.evaluator = Evaluator(argv)
self.loader = CoNLL09Loader(argv)
self.saver = CoNLL09Saver(argv)
self.preprocessor = preprocessor(argv)
self.model_api = model_api(argv)
def __init__(self, class_dict, class_list, use_cuda):
self.test_train_every = 10
self.print_every = 30
self.use_cuda = use_cuda
self.pad_idx_in_class = class_dict['PAD_token']
loss_weight = torch.ones(len(class_dict))
loss_weight[self.pad_idx_in_class] = 0
self.loss = nn.NLLLoss(weight=loss_weight, reduction="sum")
if use_cuda:
self.loss = self.loss.cuda()
self.evaluator = Evaluator(class_dict=class_dict,
class_list=class_list,
use_cuda=use_cuda)
return
def train(args):
base_dir = args.base_dir
dirs = init_dir(base_dir)
init_log(dirs['log'])
config_dir = args.config_dir
copy_file(config_dir, dirs['data'])
config = configparser.ConfigParser()
config.read(config_dir)
in_test, post_test = init_test_flag(args.test_mode)
# init env
env = init_env(config['ENV_CONFIG'])
logging.info('Training: a dim %d, agent dim: %d' % (env.n_a, env.n_agent))
# init step counter
total_step = int(config.getfloat('TRAIN_CONFIG', 'total_step'))
test_step = int(config.getfloat('TRAIN_CONFIG', 'test_interval'))
log_step = int(config.getfloat('TRAIN_CONFIG', 'log_interval'))
global_counter = Counter(total_step, test_step, log_step)
# init centralized or multi agent
seed = config.getint('ENV_CONFIG', 'seed')
model = init_agent(env, config['MODEL_CONFIG'], total_step, seed)
# disable multi-threading for safe SUMO implementation
summary_writer = tf.summary.FileWriter(dirs['log'])
trainer = Trainer(env,
model,
global_counter,
summary_writer,
in_test,
output_path=dirs['data'])
trainer.run()
# save model
final_step = global_counter.cur_step
logging.info('Training: save final model at step %d ...' % final_step)
model.save(dirs['model'], final_step)
# post-training test
if post_test:
test_dirs = init_dir(base_dir, pathes=['eva_data'])
evaluator = Evaluator(env, model, test_dirs['eva_data'])
evaluator.run()
def test(net, test_loader, epoch):
net.eval()
test_step = len(test_loader)
print('\nEvaluating...')
with torch.no_grad():
evaluator = Evaluator()
for i, sample in enumerate(test_loader):
for key in sample:
if key[0:2] != 'D_':
sample[key] = sample[key].cuda()
output = net(sample)
evaluator.add(output, sample)
if (i + 1) % 100 == 0:
print('Val Step [{}/{}]'.format(i + 1, test_step)) # j + 1 ?
results = evaluator.evaluate()
print('Epoch {}/{}'.format(epoch + 1, args.epoch))
print(
'| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |'
)
print(
'| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |'
.format(results['lm_dist'][0], results['lm_dist'][1],
results['lm_dist'][2], results['lm_dist'][3],
results['lm_dist'][4], results['lm_dist'][5],
results['lm_dist'][6], results['lm_dist'][7],
results['lm_dist_all']))
file = open('results_lr_%.4f_base_%d_de_%d_g_%.2f.txt'% \
(args.learning_rate, \
args.base_epoch, \
args.decay_epoch, \
args.gamma), 'a')
file.write('Epoch {}\n'.format(args.base_epoch + epoch + 1))
file.write(
'| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |\n'
)
file.write(
'| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |\n\n'
.format(results['lm_dist'][0], results['lm_dist'][1],
results['lm_dist'][2], results['lm_dist'][3],
results['lm_dist'][4], results['lm_dist'][5],
results['lm_dist'][6], results['lm_dist'][7],
results['lm_dist_all']))
file.close()
def validate(config, device, val_loader, epoch):
"""
validate the model
Args:
config (EasyDict): configurations for training
device (torch.device): the GPU or CPU used for training
val_loader (torch.utils.data.DataLoader): a DataLoader instance object for validation set
epoch (int): current epoch
Returns:
err: (float) error rate
"""
losses = AverageMeter()
evaluator = Evaluator(config.num_classes)
model.eval()
with tqdm(val_loader) as pbar:
pbar.set_description('Valid Epoch {}'.format(epoch))
for i, (input_, target) in enumerate(val_loader):
# move data to GPU
input_ = torch.tensor(input_, device=device, dtype=torch.float32)
target = torch.tensor(target, device=device, dtype=torch.long)
with torch.no_grad():
# compute output and loss
output = model(input_)
loss = criterion(output, target)
# record loss and show it in the pbar
losses.update(loss.item(), input_.size(0))
postfix = OrderedDict({
'batch_loss': f'{losses.val:6.4f}',
'running_loss': f'{losses.avg:6.4f}'
})
pbar.set_postfix(ordered_dict=postfix)
pbar.update()
# update confusion matrix
true = target.cpu().numpy()
pred = output.max(dim=1)[1].cpu().numpy()
evaluator.update_matrix(true, pred)
return evaluator.error()
def evaluate(split='valid'):
y_true = eval(split + '_data')
# Ttensor = eval(split + '_tensor')
model.eval()
y_score, _, _ = model(train_tensor)
y_score.detach_()
y_score = y_score.squeeze(0)
y_score[train_data.row, train_data.col] = 0
_, rec_items = torch.topk(y_score, args.N, dim=1)
# y_pred = torch.gather(Ttensor, 1, rec_items).cpu().numpy()
run = sort2query(rec_items[:, 0:args.N])
test = csr2test(y_true.tocsr())
evaluator = Evaluator({'recall', 'map_cut'})
evaluator.evaluate(run, test)
result = evaluator.show([
'recall_5', 'recall_10', 'recall_15', 'recall_20', 'map_cut_5',
'map_cut_10', 'map_cut_15', 'map_cut_20'
])
print(result)
def evaluate_fn(agent_dir, output_dir, seeds, port):
agent = agent_dir.split('/')[-1]
if not check_dir(agent_dir):
logging.error('Evaluation: %s does not exist!' % agent)
return
# load config file for env
config_dir = find_file(agent_dir)
if not config_dir:
return
config = configparser.ConfigParser()
config.read(config_dir)
# init env
env, greedy_policy = init_env(config['ENV_CONFIG'],
port=port,
naive_policy=True)
env.init_test_seeds(seeds)
# load model for agent
if agent != 'greedy':
# init centralized or multi agent
if env.agent == 'ia2c':
model = IA2C(env.n_s_ls, env.n_a, env.neighbor_mask,
env.distance_mask, env.coop_gamma, 0,
config['MODEL_CONFIG'])
elif env.agent == 'ia2c_fp':
model = IA2C_FP(env.n_s_ls, env.n_a, env.neighbor_mask,
env.distance_mask, env.coop_gamma, 0,
config['MODEL_CONFIG'])
elif env.agent == 'ma2c_nc':
model = MA2C_NC(env.n_s, env.n_a, env.neighbor_mask,
env.distance_mask, env.coop_gamma, 0,
config['MODEL_CONFIG'])
else:
return
if not model.load(agent_dir + '/'):
return
else:
model = greedy_policy
env.agent = agent
# collect evaluation data
evaluator = Evaluator(env, model, output_dir)
evaluator.run()
def test(segmentation_module, args=None):
label_num_ = args.num_class
segmentation_module.eval()
evaluator = Evaluator(label_num_)
print('validation')
with open(os.path.join(args.dataroot, 'val.txt'), 'r') as f:
lines = f.readlines()
videolists = [line[:-1] for line in lines]
for video in videolists:
test_dataset = TestDataset_clip(args.dataroot,
video,
args,
is_train=True)
loader = torch.utils.data.DataLoader(test_dataset,
batch_size=args.batchsize,
shuffle=False,
num_workers=args.workers,
drop_last=False)
for i, data in enumerate(loader):
# process data
print('[{}]/[{}]'.format(i, len(loader)))
imgs, gts, clip_imgs, _, _ = data
imgs = imgs.cuda(args.start_gpu)
gts = gts.cuda(args.start_gpu)
clip_imgs = [img.cuda(args.start_gpu) for img in clip_imgs]
batch_data = {}
batch_data['img_data'] = imgs
batch_data['seg_label'] = gts
batch_data['clipimgs_data'] = clip_imgs
segSize = (imgs.size(2), imgs.size(3))
with torch.no_grad():
scores = segmentation_module(batch_data, segSize=segSize)
pred = torch.argmax(scores, dim=1)
pred = pred.data.cpu().numpy()
target = gts.squeeze(1).cpu().numpy()
# Add batch sample into evaluator
evaluator.add_batch(target, pred)
Acc = evaluator.Pixel_Accuracy()
Acc_class = evaluator.Pixel_Accuracy_Class()
mIoU = evaluator.Mean_Intersection_over_Union()
FWIoU = evaluator.Frequency_Weighted_Intersection_over_Union()
print('Validation:')
print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format(
Acc, Acc_class, mIoU, FWIoU))
def init_trainer(self, model, epochs, lr, eval_steps=1, metric='acc'):
self.epochs = epochs
self.lr = lr
self.eval_steps = eval_steps
self.metric = metric
self.model = model
self.optimizer = torch.optim.Adam(model.parameters(),
lr=self.lr,
weight_decay=5e-4)
self.evaluator = Evaluator(metric=self.metric)
self.best_model_parameters = None
def train(self, model, data_loader, batch_size, n_epoch, template_flag, \
resume=False, optimizer=None, mode=0, teacher_forcing_ratio=0, post_flag=False):
self.evaluator = Evaluator(
vocab_dict=self.vocab_dict,
vocab_list=self.vocab_list,
decode_classes_dict=self.decode_classes_dict,
decode_classes_list=self.decode_classes_list,
loss=NLLLoss(),
cuda_use=self.cuda_use)
if resume:
checkpoint_path = Checkpoint.get_certain_checkpoint(
"./experiment", "best")
resume_checkpoint = Checkpoint.load(checkpoint_path)
model = resume_checkpoint.model
self.optimizer = resume_checkpoint.optimizer
resume_optim = self.optimizer.optimizer
defaults = resume_optim.param_groups[0]
defaults.pop('params', None)
self.optimizer.optimizer = resume_optim.__class__(
model.parameters(), **defaults)
start_epoch = resume_checkpoint.epoch
start_step = resume_checkpoint.step
self.train_acc_list = resume_checkpoint.train_acc_list
self.test_acc_list = resume_checkpoint.test_acc_list
self.loss_list = resume_checkpoint.loss_list
else:
start_epoch = 1
start_step = 0
self.train_acc_list = []
self.test_acc_list = []
self.loss_list = []
model_opt = NoamOpt(
512, 1, 2000,
torch.optim.Adam(model.parameters(),
lr=0,
betas=(0.9, 0.98),
eps=1e-9))
if optimizer is None:
optimizer = Optimizer(optim.Adam(model.parameters()),
max_grad_norm=0)
self.optimizer = model_opt
self._train_epoches(data_loader=data_loader,
model=model,
batch_size=batch_size,
start_epoch=start_epoch,
start_step=start_step,
n_epoch=n_epoch,
mode=mode,
template_flag=template_flag,
teacher_forcing_ratio=teacher_forcing_ratio,
post_flag=post_flag)
def test(net, test_loader, epoch):
net.eval()
test_step = len(test_loader)
print('\nEvaluating...')
with torch.no_grad():
evaluator = Evaluator()
for i, (sample, img) in enumerate(test_loader):
#import matplotlib.pyplot as plt
#print(len(sample['image_original']))
#plt.imshow(sample['image_original'].numpy().swapaxes(0, 2))
#plt.show()
#image_name = sample['image_name']
#print(image_name)
for key in sample:
if key != 'image_name':
sample[key] = sample[key].cuda()
output = net(sample)
#print(type(output))
pos_map = output['lm_pos_map']
#print(pos_map.size())
#resh = pos_map.reshape(50, 8, -1)
#print(resh.size())
evaluator.add(output, sample, img)
if (i + 1) % 100 == 0:
print('Val Step [{}/{}]'.format(i + 1, test_step))
results = evaluator.evaluate()
print('Epoch {}/{}'.format(epoch + 1, args.epoch))
print(
'| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |'
)
print(
'| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |'
.format(results['lm_dist'][0], results['lm_dist'][1],
results['lm_dist'][2], results['lm_dist'][3],
results['lm_dist'][4], results['lm_dist'][5],
results['lm_dist'][6], results['lm_dist'][7],
results['lm_dist_all']))
def predict():
net = UNet(n_channels=3, n_classes=2)
# net = ULeNet(n_channels=3, n_classes=6)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
net.to(device=device)
net.load_state_dict(torch.load(model, map_location=device))
evaluator = Evaluator(num_class=2)
evaluator.reset()
for img in os.listdir(dir_img):
image = cv2.imread(os.path.join(dir_img, img))
# mask = cv2.imread(os.path.join(dir_mask, img))
# mask = cv2.cvtColor(mask, cv2.COLOR_RGB2GRAY)
mask = Image.open(os.path.join(dir_mask, img))
mask = mask.convert('L')
mask = np.array(mask)
mask_class = np.unique(mask)
for i in range(len(mask_class)):
mask[mask == mask_class[i]] = i
predict = np.zeros((image.shape[0], image.shape[1]))
p_size = 128
for i in range(0, image.shape[0] - p_size, p_size):
for j in range(0, image.shape[1] - p_size, p_size):
patch = image[i:i + p_size, j:j + p_size, :]
patch = Normalization(patch)
predict[i:i + p_size, j:j + p_size] = getPredict(net=net,
img=patch,
device=device)
evaluator.add_batch(mask, predict)
# mIoU = evaluator.Mean_Intersection_over_Union()
predict = Image.fromarray((predict).astype(np.uint8))
predict.save(
os.path.join(dir_predict,
os.path.splitext(img)[0] + '.tif'))
show_confMat(evaluator.conf_mat, [str(c) for c in range(2)],
re.split('[/.]', model)[1], dir_confmat)
def eval_net(net, data_loader, device):
net.eval()
val_batch_num = len(data_loader)
eval_loss = 0
e = Evaluator(num_class=8)
pixel_acc_avg = 0
mean_iou_avg = 0
fw_iou_avg = 0
with tqdm(total=val_batch_num,
desc='Validation round',
unit='batch',
leave=False) as pbar:
for idx, batch_samples in enumerate(data_loader):
batch_image, batch_mask = batch_samples["image"], batch_samples[
"mask"]
batch_image = batch_image.to(device=device, dtype=torch.float32)
mask_true = batch_mask.to(device=device, dtype=torch.long)
with torch.no_grad():
mask_pred = net(batch_image)
probs = F.softmax(mask_pred, dim=1).squeeze(0) # [8, 256, 256]
pre = torch.argmax(probs, dim=1) # [256,256]
#????
e.add_batch(mask_true.cpu().data.numpy(), pre.cpu().data.numpy())
pixel_acc = e.Pixel_Accuracy()
pixel_acc_avg += pixel_acc
mean_iou = e.Mean_Intersection_over_Union()
mean_iou_avg += mean_iou
fw_iou = e.Frequency_Weighted_Intersection_over_Union()
fw_iou_avg += fw_iou
eval_loss += F.cross_entropy(mask_pred, mask_true).item()
pbar.set_postfix({'eval_loss': eval_loss / (idx + 1)})
pbar.update()
e.reset()
print("pixel_acc_avg:" + str(pixel_acc_avg / val_batch_num))
print("mean_iou_avg:" + str(mean_iou_avg / val_batch_num))
print("fw_iou_avg:" + str(fw_iou_avg / val_batch_num))
net.train()
return eval_loss / val_batch_num, pixel_acc_avg / val_batch_num, mean_iou_avg / val_batch_num, fw_iou_avg / val_batch_num
def fit(self, iteration=1000):
""" generate a style-transfered image.
The overall process is as follows:
1. initiating the initial canvas
2. setting Evaluator
3. optimizing using L-BFGS method
Args:
iteration (int): the total iteration number of optimization
"""
if self.initial_canvas == 'random':
generated_img = utils.preproc(np.random.uniform(0, 255, size=self.img_shape)\
.astype(K.floatx()))
elif self.initial_canvas == 'content':
generated_img = self.content_img.copy()
else: # style
generated_img = self.style_img.copy()
evaluator = Evaluator(self.eval_fn, self.img_shape)
self.step = 0
print('Starting optimization with L-BFGS-B')
for i in range(1, iteration + 1):
if self.verbose:
print('Starting iteration %d' % (i))
start_time = time.time()
generated_img, min_loss, _ = fmin_l_bfgs_b(evaluator.loss,
generated_img.flatten(),
fprime=evaluator.grads,
callback=self._callback,
maxfun=20)
generated_img = np.clip(generated_img, -127, 127)
end_time = time.time()
if self.verbose:
print('Total_Loss: %g, Content Loss: %g, Style Loss: %g' % \
(min_loss,
evaluator.content_loss,
evaluator.style_loss))
print('Iteration %d completed in %d s' %
(i, end_time - start_time))
if i == 1 or (self.save_every_n_iters != 0
and i % self.save_every_n_iters == 0):
utils.save_image(utils.deproc(generated_img, self.img_shape),
self.output_path,
'generated_%d' % (i) + '.jpg')
def test(net, test_loader, epoch):
net.eval()
test_step = len(test_loader)
print('\nEvaluating...')
with torch.no_grad():
evaluator = Evaluator()
pbar2 = tqdm(test_loader)
for i, sample in enumerate(pbar2):
iter_start_time = time.time()
for key in sample:
sample[key] = sample[key].cuda()
output = net(sample)
evaluator.add(output, sample)
t = time.time() - iter_start_time
if (i + 1) % 100 == 0:
tqdm.write('Val Step [{}/{}], Time:{:.3f}'.format(
i + 1, test_step, t))
results = evaluator.evaluate()
print('Epoch {}/{}'.format(epoch + 1, args.epoch))
print('lm_dist_all: {:.5f} '.format(results['lm_dist_all']))
