def evaluate(self):
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
pred_labels = []
gt_labels = []
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
with function.no_backprop_mode():
if isinstance(in_arrays, tuple):
eval_func(*in_arrays)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays)
else:
eval_func(in_arrays)
if eval_func.predictions is not None:
pred_labels.extend(cuda.to_cpu(eval_func.predictions))
gt_labels.extend(cuda.to_cpu(eval_func.gt))
summary.add(observation)
observation = summary.compute_mean()
if self.label_names is not None and len(pred_labels) > 0:
pred_labels = np.array(pred_labels)
gt_labels = np.array(gt_labels)
result = eval_semantic_segmentation(pred_labels, gt_labels)
report = {
'miou': result['miou'],
'pixel_acc': result['pixel_accuracy'],
'mean_class_acc': result['mean_class_accuracy']
}
for l, label_name in enumerate(self.label_names):
try:
report['iou/{:s}'.format(label_name)] = result['iou'][l]
report['class_acc/{:s}'.format(
label_name)] = result['class_accuracy'][l]
except IndexError:
report['iou/{:s}'.format(label_name)] = np.nan
report['class_acc/{:s}'.format(label_name)] = np.nan
with reporter_module.report_scope(observation):
reporter_module.report(report, eval_func)
return observation
def evaluate_roc(self, trainer):
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
y_total = []
t_total = []
length = it.dataset.__len__()
batchsize = it.batch_size
length = length // batchsize
from tqdm import tqdm
pbar = tqdm(total=length)
for batch in it:
in_arrays = self.converter(batch, self.device)
with chainer.no_backprop_mode(), chainer.using_config('train', False):
y = eval_func(*in_arrays[:-1])
t = in_arrays[-1]
y_data = cuda.to_cpu(y.data)
t_data = cuda.to_cpu(t)
y_total.extend(y_data)
t_total.extend(t_data)
pbar.update(1)
y_total = numpy.concatenate(y_total).ravel()
t_total = numpy.concatenate(t_total).ravel()
index = numpy.where(t_total != -1)[0]
y_total = y_total[index]
t_total = t_total[index]
d = {'label': t_total, 'score': y_total}
from pathlib import Path
out_path = Path('./valid_result') / str(self.epoch)
out_path.mkdir(exist_ok=True)
np.savez(out_path / ('validation_' + str(self.rank)), **d)
observation = {}
with reporter.report_scope(observation):
roc_auc = metrics.roc_auc_score(t_total, F.sigmoid(y_total).data)
with reporter.report_scope(observation):
reporter.report({'roc_auc_': roc_auc}, self._targets['main'])
reporter.report({'loss': F.sigmoid_cross_entropy(y_total, t_total).data},
self._targets['main'])
reporter.report({'accuracy': F.binary_accuracy(y_total, t_total).data}, self._targets['main'])
return observation
def evaluate(self):
val_iter = self.get_iterator('main')
model = self.get_target('main')
it = copy.copy(val_iter)
summary = reporter.DictSummary()
res = []
for i, batch in enumerate(it):
observation = {}
with reporter.report_scope(observation):
imgs, pafs, heatmaps, ignore_mask = self.converter(batch, self.device)
with function.no_backprop_mode():
x_data = preprocess(imgs)
pafs_ys, heatmaps_ys = model(x_data)
loss, paf_loss_log, heatmap_loss_log = compute_loss(
imgs, pafs_ys, heatmaps_ys, pafs, heatmaps, ignore_mask)
observation['val/loss'] = cuda.to_cpu(loss.data)
observation['val/paf'] = sum(paf_loss_log)
observation['val/heat'] = sum(heatmap_loss_log)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
annType = ['segm', 'bbox', 'keypoints']
annType = annType[1]
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_prediction_to_iterator(
target.predict, it)
# delete unused iterators explicitly
del in_values
pred_bboxes, _, pred_labels, pred_scores, pred_masks = out_values
if len(rest_values) == 3:
gt_bboxes, gt_labels, gt_difficults = rest_values
elif len(rest_values) == 2:
gt_bboxes, gt_labels = rest_values
gt_difficults = None
elif len(rest_values) == 5:
gt_bboxes, gt_labels, _, _, i = rest_values
gt_difficults = None
pred_bboxes = iter(list(pred_bboxes))
pred_labels = iter(list(pred_labels))
pred_scores = iter(list(pred_scores))
gt_bboxes = iter(list(gt_bboxes))
gt_labels = iter(list(gt_labels))
data_dict = []
for i, (pred_bbox, pred_label, pred_score) in \
enumerate(zip(pred_bboxes, pred_labels, pred_scores)):
for bbox, label, score in zip(pred_bbox, pred_label, pred_score):
A = {
"image_id": int(self.ids[i]),
"category_id": int(label),
"bbox": bbox.tolist(),
"score": float(score)
}
data_dict.append(A)
if len(data_dict) > 0:
cocoGt = self.cocoanns
cocoDt = cocoGt.loadRes(data_dict)
cocoEval = COCOeval(self.cocoanns, cocoDt, annType)
cocoEval.params.imgIds = [int(id_) for id_ in self.ids]
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
report = {
'map': cocoEval.stats[0]
} # report COCO AP (IoU=0.5:0:95)
else:
report = {'map': 0}
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def test_report_key(self, metrics_fun, compute_metrics):
repo = chainer.Reporter()
link = Classifier(predictor=DummyPredictor(),
metrics_fun=metrics_fun)
link.compute_metrics = compute_metrics
repo.add_observer('target', link)
with repo:
observation = {}
with reporter.report_scope(observation):
link(self.x, self.t)
# print('observation ', observation)
actual_keys = set(observation.keys())
if compute_metrics:
if metrics_fun is None:
assert set(['target/loss']) == actual_keys
elif isinstance(metrics_fun, dict):
assert set(['target/loss', 'target/user_key']) == actual_keys
elif callable(metrics_fun):
assert set(['target/loss', 'target/accuracy']) == actual_keys
else:
raise TypeError()
else:
assert set(['target/loss']) == actual_keys
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(
target.predict,
it,
n_input=2,
)
# delete unused iterators explicitly
del in_values
pred_labels, = out_values
gt_labels, = rest_values
report = eval_sigmoid_segmentation(pred_labels,
gt_labels,
channel_names=self.channel_names)
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def evaluate(self):
"""Evaluate the model."""
val_iter = self.get_iterator('main')
loss = 0
nll = 0
count = 0
self.model.eval()
with torch.no_grad():
for batch in copy.copy(val_iter):
x, t = concat_examples(batch,
device=self.device[0],
padding=(0, -100))
if self.device[0] == -1:
l, n, c = self.model(x, t)
else:
# apex does not support torch.nn.DataParallel
l, n, c = data_parallel(self.model, (x, t), self.device)
loss += float(l.sum())
nll += float(n.sum())
count += int(c.sum())
self.model.train()
# report validation loss
observation = {}
with reporter.report_scope(observation):
reporter.report({'loss': loss}, self.model.reporter)
reporter.report({'nll': nll}, self.model.reporter)
reporter.report({'count': count}, self.model.reporter)
return observation
def evaluate(self):
iterator = self._iterators['main']
unet = self._targets['unet']
#eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
ground_truth, data = self.converter(batch, self.device)
with chainer.using_config("train", False):
with chainer.no_backprop_mode():
predict = unet(data)
ground_truth = ground_truth[:, :, 20:72, 20:72, 20:72]
#observation['vali/unet/loss'] = self.loss_softmax_cross_entropy(predict,ground_truth)
observation['vali/unet/dice'] = self.dice_coefficent(
predict, ground_truth)
summary.add(observation)
#print(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(
self.predict_func, it)
# delete unused iterators explicitly
del in_values
pred_labels, pred_scores = out_values
gt_labels, = rest_values
result = eval_multi_label_classification(pred_labels, pred_scores,
gt_labels)
report = {'map': result['map']}
if self.label_names is not None:
for l, label_name in enumerate(self.label_names):
try:
report['ap/{:s}'.format(label_name)] = result['ap'][l]
except IndexError:
report['ap/{:s}'.format(label_name)] = np.nan
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def evaluate(self):
val_iter = self.get_iterator('main')
model = self.get_target('main')
it = copy.copy(val_iter)
summary = reporter.DictSummary()
res = []
for i, batch in enumerate(it):
observation = {}
with reporter.report_scope(observation):
imgs, pafs, heatmaps, ignore_mask = self.converter(
batch, self.device)
with function.no_backprop_mode():
x_data = preprocess(imgs)
pafs_ys, heatmaps_ys = model(x_data)
loss, paf_loss_log, heatmap_loss_log = compute_loss(
imgs, pafs_ys, heatmaps_ys, pafs, heatmaps,
ignore_mask)
observation['val/loss'] = cuda.to_cpu(loss.data)
observation['val/paf'] = sum(paf_loss_log)
observation['val/heat'] = sum(heatmap_loss_log)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
pred_bboxes, pred_labels, pred_scores, gt_values =\
apply_detection_link(target, it)
if len(gt_values) == 3:
gt_bboxes, gt_labels, gt_difficults = gt_values
elif len(gt_values) == 2:
gt_bboxes, gt_labels = gt_values
gt_difficults = None
eval_ = eval_detection_voc(pred_bboxes,
pred_labels,
pred_scores,
gt_bboxes,
gt_labels,
gt_difficults,
use_07_metric=self.use_07_metric)
observation = {}
with reporter.report_scope(observation):
reporter.report({'map': eval_['map']}, target)
return observation
def evaluate(self):
from chainer import reporter
import copy
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
it = copy.copy(iterator)
summary = reporter.DictSummary()
for batch in it:
observation = {}
with reporter.report_scope(observation):
in_arrays = self.converter(batch, self.device)
if isinstance(in_arrays, tuple):
eval_func(*in_arrays)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays)
else:
eval_func(in_arrays)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
"""Main evaluate routine for CustomEvaluator."""
iterator = self._iterators['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
self.model.eval()
with torch.no_grad():
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
x = self.converter(batch, self.device)
if self.ngpu == 0:
self.model(*x)
else:
# apex does not support torch.nn.DataParallel
data_parallel(self.model, x, range(self.ngpu))
summary.add(observation)
self.model.train()
return summary.compute_mean()
def evaluate(self):
bt = time.time()
with chainer.no_backprop_mode():
references = []
hypotheses = []
observation = {}
with reporter.report_scope(observation):
for i in range(0, len(self.test_data), self.batch):
src, trg = zip(*self.test_data[i:i + self.batch])
references.extend([[t.tolist()] for t in trg])
src = [chainer.dataset.to_device(self.device, x)
for x in src]
if self.comm.rank == 0:
self.model.translate(src, self.max_length)
elif self.comm.rank == 1:
ys = [y.tolist()
for y in self.model.translate(
src, self.max_length)]
hypotheses.extend(ys)
if self.comm.rank == 1:
bleu = bleu_score.corpus_bleu(
references, hypotheses, smoothing_function=bleu_score.
SmoothingFunction().method1)
reporter.report({'bleu': bleu}, self.model)
et = time.time()
if self.comm.rank == 1:
print('BleuEvaluator(single)::evaluate(): '
'took {:.3f} [s]'.format(et - bt))
sys.stdout.flush()
return observation
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if self.eval_hook:
self.eval_hook(self)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
acc = 0
count = 0
observation = {}
with reporter_module.report_scope(observation):
for batch in it:
target.predictor.reset_state()
w, l = batch[0]
xp = cuda.cupy if self.device >= 0 else np
xp_words = [xp.array([_w], xp.int32) for _w in w]
predict = np.argmax(target.predictor(xp_words).data)
if predict == l:
acc += 1
count += 1
summary.add({'main/validation/accuracy': acc / float(count)})
return summary.compute_mean()
def evaluate(self):
#target = self._targets['main']
summary = reporter_module.DictSummary()
for name, target in six.iteritems(self._targets):
iterator = self._iterators['main']
#target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
#summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
with function.no_backprop_mode():
if isinstance(in_arrays, tuple):
eval_func(*in_arrays)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays)
else:
eval_func(in_arrays)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
self.model.eval()
with torch.no_grad():
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
x = self.converter(batch, self.device)
self.model(*x)
summary.add(observation)
self.model.train()
return summary.compute_mean()
def test_report_key(self, metrics_fun, compute_metrics):
repo = chainer.Reporter()
link = Regressor(predictor=DummyPredictor(),
metrics_fun=metrics_fun)
link.compute_metrics = compute_metrics
repo.add_observer('target', link)
with repo:
observation = {}
with reporter.report_scope(observation):
link(self.x, self.t)
# print('observation ', observation)
actual_keys = set(observation.keys())
if compute_metrics:
if metrics_fun is None:
assert set(['target/loss']) == actual_keys
elif isinstance(metrics_fun, dict):
assert set(['target/loss', 'target/user_key']) == actual_keys
elif callable(metrics_fun):
assert set(['target/loss', 'target/metrics']) == actual_keys
else:
raise TypeError()
else:
assert set(['target/loss']) == actual_keys
def evaluate(self):
"""Evaluate the model."""
val_iter = self.get_iterator("main")
loss = 0
nll = 0
count = 0
self.model.eval()
with torch.no_grad():
for batch in copy.copy(val_iter):
x, t = concat_examples(batch[0],
device=self.device[0],
padding=(0, -100))
aver_mask = concat_examples_pad_last_2dim(
batch[1], device=self.device[0], padding=0)
if self.device[0] == -1:
l, n, c = self.model(x, t, aver_mask)
else:
# apex does not support torch.nn.DataParallel
l, n, c = data_parallel(self.model, (x, t, aver_mask),
self.device)
loss += float(l.sum())
nll += float(n.sum())
count += int(c.sum())
self.model.train()
# report validation loss
observation = {}
with reporter.report_scope(observation):
reporter.report({"loss": loss}, self.model.reporter)
reporter.report({"nll": nll}, self.model.reporter)
reporter.report({"count": count}, self.model.reporter)
return observation
def evaluate(self):
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
while True:
batch = it.next()
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
with function.no_backprop_mode():
if isinstance(in_arrays, tuple):
eval_func(*in_arrays)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays)
else:
eval_func(in_arrays)
summary.add(observation)
if it.is_new_epoch:
break
out = summary.compute_mean()
print('#############################################', out)
return out
def evaluate(self):
iterator = self._iterators["main"]
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, "reset"):
iterator.reset()
it = iterator
else:
warnings.warn('This iterator does not have the reset method. Evaluator '
'copies the iterator instead of resetting. This behavior is '
'deprecated. Please implement the reset method.',
DeprecationWarning)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
xs, adjs = self.converter(batch, self.device)
xp = cuda.get_array_module(xs)
with chainer.no_backprop_mode(), chainer.using_config("train", False):
hs = self.model(xs, adjs)
hs = F.reshape(hs, (-1, hs.shape[-1]))
xs = xs.reshape(-1)
loss = F.softmax_cross_entropy(hs, xs)
acc = F.accuracy(hs, xs)
reporter_module.report({"accuracy": acc, "ce_loss": loss})
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(
target.predict, it)
# delete unused iterators explicitly
del in_values
points, labels, scores = out_values
gt_points, gt_labels = rest_values
result = eval_projected_3d_bbox_single(
points, scores, gt_points,
self.vertex, self.intrinsics, diam=self.diam)
report = result
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(
target.predict, it)
# delete unused iterators explicitly
del in_values
points, labels, scores = out_values
gt_points, gt_labels = rest_values
result = eval_projected_3d_bbox_single(points,
scores,
gt_points,
self.vertex,
self.intrinsics,
diam=self.diam)
report = result
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def evaluate(self):
iterator = self._iterators['main']
eval_func = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
warnings.warn(
'This iterator does not have the reset method. Evaluator '
'copies the iterator instead of resetting. This behavior is '
'deprecated. Please implement the reset method.',
DeprecationWarning)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for x_batch, t_batch in it:
observation = {}
with reporter_module.report_scope(observation):
with function.no_backprop_mode():
eval_func(x_batch, t_batch)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
batch = next(iterator)
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
if isinstance(in_arrays, tuple):
in_vars = tuple(variable.Variable(x, volatile='on')
for x in in_arrays)
eval_func(*in_vars)
elif isinstance(in_arrays, dict):
in_vars = {key: variable.Variable(x, volatile='on')
for key, x in six.iteritems(in_arrays)}
eval_func(**in_vars)
else:
in_var = variable.Variable(in_arrays, volatile='on')
eval_func(in_var)
return observation
def evaluate(self):
iterator = self._iterators['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
self.model.eval()
if not torch_is_old:
torch.set_grad_enabled(False)
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
x = self.converter(batch)
self.model(x)
delete_feat(x)
summary.add(observation)
self.model.train()
if not torch_is_old:
torch.set_grad_enabled(True)
return summary.compute_mean()
def evaluate(self):
'''evaluate over iterator'''
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
# for multi gpu calculation
chainer.cuda.get_device_from_id(self.device).use()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
# batch only has one minibatch utterance, which is specified by batch[0]
x = converter_kaldi(batch[0], self.reader)
with function.no_backprop_mode():
eval_func(x)
delete_feat(x)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
val_iter = self.get_iterator('main')
model = self.get_target('main')
it = copy.copy(val_iter)
summary = reporter.DictSummary()
res = []
for i, batch in enumerate(it):
observation = {}
with reporter.report_scope(observation):
imgs, pafs, heatmaps, ignore_mask = self.converter(
batch, self.device)
with function.no_backprop_mode():
x_data = imgs.astype(np.float32).transpose(0, 3, 1,
2) / 256 - 0.5
inferenced_pafs, inferenced_heatmaps = model(x_data)
loss, pafs_loss_log, heatmaps_loss_log = compute_loss(
inferenced_pafs, inferenced_heatmaps, pafs, heatmaps,
ignore_mask)
observation['val/loss'] = cuda.to_cpu(loss.data)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
kwargs = {}
kwargs['train'] = False
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
with function.no_backprop_mode():
if isinstance(in_arrays, tuple):
eval_func(*in_arrays, **kwargs)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays, **kwargs)
else:
eval_func(in_arrays, **kwargs)
summary.add(observation)
observation = summary.compute_mean()
return observation
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(
target.predict, it)
# delete unused iterators explicitly
del in_values
pred_imgs, = out_values
gt_imgs, = rest_values
mse_list = []
for pred_img, gt_img in zip(pred_imgs, gt_imgs):
diff = (pred_img - gt_img).ravel()
mse = diff.dot(diff) / diff.size
mse_list.append(mse)
report = {
'loss': np.mean(mse_list),
}
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def evaluate(self):
bt = time.time()
with chainer.no_backprop_mode():
references = []
hypotheses = []
observation = {}
with reporter.report_scope(observation):
for i in range(0, len(self.test_data), self.batch):
src, trg = zip(*self.test_data[i:i + self.batch])
references.extend([[t.tolist()] for t in trg])
src = [chainer.dataset.to_device(self.device, x)
for x in src]
if self.comm.rank == 0:
self.model.translate(src, self.max_length)
elif self.comm.rank == 1:
ys = [y.tolist()
for y in self.model.translate(
src, self.max_length)]
hypotheses.extend(ys)
if self.comm.rank == 1:
bleu = bleu_score.corpus_bleu(
references, hypotheses, smoothing_function=bleu_score.
SmoothingFunction().method1)
reporter.report({'bleu': bleu}, self.model)
et = time.time()
if self.comm.rank == 1:
print("BleuEvaluator(single)::evaluate(): "
"took {:.3f} [s]".format(et - bt))
sys.stdout.flush()
return observation
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(
self.predict_func, it)
# delete unused iterators explicitly
del in_values
pred_labels, pred_scores = out_values
gt_labels, = rest_values
result = eval_multi_label_classification(
pred_labels, pred_scores, gt_labels)
report = {'map': result['map']}
if self.label_names is not None:
for l, label_name in enumerate(self.label_names):
try:
report['ap/{:s}'.format(label_name)] = result['ap'][l]
except IndexError:
report['ap/{:s}'.format(label_name)] = np.nan
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def evaluate(self):
iterator = self._iterators['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
# batch only has one minibatch utterance, which is specified by batch[0]
x = converter_kaldi(batch[0], self.reader)
self.model.eval()
self.model(x)
delete_feat(x)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self.get_iterator("main")
target = self.get_target("main")
if hasattr(iterator, "reset"):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter.DictSummary()
for batch in it:
observation = {}
with reporter.report_scope(observation):
with function.no_backprop_mode():
in_arrays, t_arrays = self.converter(batch, self.device)
p_arrays = target.predict(in_arrays)
_, t_tag_sentences = list(
zip(*self.transform_func(in_arrays[0], t_arrays)))
_, p_tag_sentences = list(
zip(*self.transform_func(in_arrays[0], p_arrays)))
fscore = metrics.f1_score(t_tag_sentences, p_tag_sentences)
reporter.report({"loss": target(in_arrays, t_arrays)},
target)
reporter.report({"fscore": fscore}, target)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for _ in range(min(len(iterator.dataset) // iterator.batch_size, self.num_iterations)):
batch = next(it, None)
if batch is None:
break
observation = {}
with reporter_module.report_scope(observation), chainer.using_config('train', False), chainer.using_config('enable_backprop', False):
in_arrays = self.converter(batch, self.device)
if isinstance(in_arrays, tuple):
eval_func(*in_arrays)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays)
else:
eval_func(in_arrays)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
"""Evaluates the model and returns a result dictionary.
This method runs the evaluation loop over the validation dataset. It
accumulates the reported values to :class:`~chainer.DictSummary` and
returns a dictionary whose values are means computed by the summary.
Note that this function assumes that the main iterator raises
``StopIteration`` or code in the evaluation loop raises an exception.
So, if this assumption is not held, the function could be caught in
an infinite loop.
Users can override this method to customize the evaluation routine.
.. note::
This method encloses :attr:`eval_func` calls with
:func:`function.no_backprop_mode` context, so all calculations
using :class:`~chainer.FunctionNode`\\s inside
:attr:`eval_func` do not make computational graphs. It is for
reducing the memory consumption.
Returns:
dict: Result dictionary. This dictionary is further reported via
:func:`~chainer.report` without specifying any observer.
"""
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
with function.no_backprop_mode():
if isinstance(in_arrays, tuple):
eval_func(*in_arrays)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays)
else:
eval_func(in_arrays)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
"""override method of extensions.Evaluator."""
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
features = None
max_loc = None
pbar = tqdm(total=len(iterator.dataset))
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
if isinstance(in_arrays, tuple):
in_vars = tuple(variable.Variable(x, volatile='off')
for x in in_arrays)
eval_func(*in_vars)
elif isinstance(in_arrays, dict):
in_vars = {key: variable.Variable(x, volatile='off')
for key, x in six.iteritems(in_arrays)}
eval_func(**in_vars)
else:
in_var = variable.Variable(in_arrays, volatile='off')
eval_func(in_var)
pbar.update(len(batch))
# deconv対象の層のVariableを取得
layer_variable = Vutil.get_variable(
observation[self.lastname], self.layer_rank)
if features is None:
features = Vutil.get_features(
layer_variable, self.operation)
else:
xp = cuda.get_array_module(features)
features = xp.vstack((features, Vutil.get_features(
layer_variable, self.operation)))
#self.add_to_confmat(self.confmat, in_vars[1].data, self.getpred(observation[self.lastname]))
summary.add(observation)
pbar.close()
#print(self.confmat)
#print(np.diag(self.confmat))
#print(1.0 * np.diag(self.confmat).sum() / self.confmat.sum())
return summary.compute_mean(), features
def evaluate(self):
"""Evaluates the model and returns a result dictionary.
This method runs the evaluation loop over the validation dataset. It
accumulates the reported values to :class:`~chainer.DictSummary` and
returns a dictionary whose values are means computed by the summary.
Users can override this method to customize the evaluation routine.
Returns:
dict: Result dictionary. This dictionary is further reported via
:func:`~chainer.report` without specifying any observer.
"""
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
if isinstance(in_arrays, tuple):
in_vars = tuple(variable.Variable(x, volatile='on')
for x in in_arrays)
eval_func(*in_vars)
elif isinstance(in_arrays, dict):
in_vars = {key: variable.Variable(x, volatile='on')
for key, x in six.iteritems(in_arrays)}
eval_func(**in_vars)
else:
in_var = variable.Variable(in_arrays, volatile='on')
eval_func(in_var)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
"""override method of extensions.Evaluator."""
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
predictions = []
rankings = []
n_categories = 0
pbar = tqdm(total=len(it.dataset))
self.confmat = None
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
if isinstance(in_arrays, tuple):
in_vars = tuple(variable.Variable(x, volatile='off')
for x in in_arrays)
eval_func(*in_vars)
elif isinstance(in_arrays, dict):
in_vars = {key: variable.Variable(x, volatile='off')
for key, x in six.iteritems(in_arrays)}
eval_func(**in_vars)
else:
in_var = variable.Variable(in_arrays, volatile='off')
eval_func(in_var)
if n_categories == 0:
n_categories = self.get_n_categories(observation[self.lastname])
self.confmat = np.zeros((n_categories, n_categories), dtype=np.int32)
#self.printloss(observation[self.lastname])
#print(self.getpred(observation[self.lastname]))
predictions.append(self.getpred(observation[self.lastname]))
rankings.append(self.getranking(observation[self.lastname], in_vars[1].data))
self.add_to_confmat(self.confmat, in_vars[1].data, self.getpred(observation[self.lastname]))
summary.add(observation)
pbar.update(len(batch))
#print(self.confmat)
#print(np.diag(self.confmat))
#print(1.0 * np.diag(self.confmat).sum() / self.confmat.sum())
return summary.compute_mean(), predictions, rankings
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(
target.predict, it)
# delete unused iterators explicitly
del in_values
pred_bboxes, pred_labels, pred_scores = out_values
if len(rest_values) == 3:
gt_bboxes, gt_labels, gt_difficults = rest_values
elif len(rest_values) == 2:
gt_bboxes, gt_labels = rest_values
gt_difficults = None
result = eval_detection_voc(
pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_difficults,
use_07_metric=self.use_07_metric)
report = {'map': result['map']}
if self.label_names is not None:
for l, label_name in enumerate(self.label_names):
try:
report['ap/{:s}'.format(label_name)] = result['ap'][l]
except IndexError:
report['ap/{:s}'.format(label_name)] = np.nan
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def evaluate(self):
val_iter = self.get_iterator('main')
model = self.get_target('main')
it = copy.copy(val_iter)
summary = reporter.DictSummary()
res = []
for i, batch in enumerate(it):
observation = {}
with reporter.report_scope(observation):
imgs, pafs, heatmaps, ignore_mask = self.converter(batch, self.device)
with function.no_backprop_mode():
x_data = imgs.astype(np.float32).transpose(0, 3, 1, 2) / 256 - 0.5
inferenced_pafs, inferenced_heatmaps = model(x_data)
loss, pafs_loss_log, heatmaps_loss_log = compute_loss(
inferenced_pafs, inferenced_heatmaps, pafs, heatmaps, ignore_mask)
observation['val/loss'] = cuda.to_cpu(loss.data)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(
target.predict, it)
# delete unused iterators explicitly
del in_values
pred_labels, = out_values
gt_labels, = rest_values
result = eval_semantic_segmentation(pred_labels, gt_labels)
report = {'miou': result['miou'],
'pixel_accuracy': result['pixel_accuracy'],
'mean_class_accuracy': result['mean_class_accuracy']}
if self.label_names is not None:
for l, label_name in enumerate(self.label_names):
try:
report['iou/{:s}'.format(label_name)] = result['iou'][l]
report['class_accuracy/{:s}'.format(label_name)] =\
result['class_accuracy'][l]
except IndexError:
report['iou/{:s}'.format(label_name)] = np.nan
report['class_accuracy/{:s}'.format(label_name)] = np.nan
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def evaluate(self):
bt = time.time()
with chainer.no_backprop_mode():
references = []
hypotheses = []
observation = {}
with reporter.report_scope(observation):
for i in range(0, len(self.test_data), self.batch):
src, trg = zip(*self.test_data[i:i + self.batch])
references.extend([[t.tolist()] for t in trg])
src = [chainer.dataset.to_device(self.device, x)
for x in src]
ys = [y.tolist()
for y in self.model.translate(src, self.max_length)]
hypotheses.extend(ys)
bleu = bleu_score.corpus_bleu(
references, hypotheses,
smoothing_function=bleu_score.SmoothingFunction().method1)
reporter.report({'bleu': bleu}, self.model)
et = time.time()
if self.comm is not None:
# This evaluator is called via chainermn.MultiNodeEvaluator
for i in range(0, self.comm.size):
print('BleuEvaluator::evaluate(): '
'took {:.3f} [s]'.format(et - bt))
sys.stdout.flush()
self.comm.mpi_comm.Barrier()
else:
# This evaluator is called from a conventional
# Chainer exntension
print('BleuEvaluator(single)::evaluate(): '
'took {:.3f} [s]'.format(et - bt))
sys.stdout.flush()
return observation
def evaluate(self):
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
y_total = []
t_total = []
for batch in it:
in_arrays = self.converter(batch, self.device)
with chainer.no_backprop_mode(), chainer.using_config('train',
False):
y = eval_func(*in_arrays[:-1])
t = in_arrays[-1]
y_data = _get_1d_numpy_array(y)
t_data = _get_1d_numpy_array(t)
y_total.append(y_data)
t_total.append(t_data)
y_total = numpy.concatenate(y_total).ravel()
t_total = numpy.concatenate(t_total).ravel()
# metrics_value = self.metrics_fun(y_total, t_total)
metrics = {key: metric_fun(y_total, t_total) for key, metric_fun in
self.metrics_fun.items()}
observation = {}
with reporter.report_scope(observation):
reporter.report(metrics, self._targets['main'])
return observation
def evaluate(self):
"""override method of extensions.Evaluator."""
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
max_locs = []
bounds = []
n_processed = 0
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
if isinstance(in_arrays, tuple):
in_vars = tuple(variable.Variable(x, volatile='off')
for x in in_arrays)
eval_func(*in_vars)
elif isinstance(in_arrays, dict):
in_vars = {key: variable.Variable(x, volatile='off')
for key, x in six.iteritems(in_arrays)}
eval_func(**in_vars)
else:
in_var = variable.Variable(in_arrays, volatile='off')
eval_func(in_var)
# deconv対象の層のVariableを取得
layer_variable = Vutil.get_variable(
observation[self.lastname], self.layer_rank)
pred = self.getpred(layer_variable)
truth = in_vars[1].data
xp = cuda.get_array_module(pred)
if xp == cupy:
pred = pred.get()
truth = truth.get()
indices = pred.flatten() if self.target == 'pred' else truth.flatten()
# 最大値の位置の計算に必要な入力層の領域を取得
isfc = Vutil.has_fc_layer(layer_variable)
if isfc:
bounds = Vutil.get_data_bounds(layer_variable)
else:
bounds = Vutil.get_max_bounds(layer_variable, indices)
# deconvを実行
deconv_data = self.get_deconv(
layer_variable, indices)
topk = np.arange(n_processed, n_processed + len(batch)) // self.n_features
for i, (t, p, d, b) in enumerate(zip(truth, indices, deconv_data, bounds)):
#print(dir(d))
# deconvされた入力層に平均画像を足して画像化
img = ioutil.deprocess(d.get(), self.mean)
# 最大値の位置の計算に必要な入力層の領域だけクロッピングして保存
img.crop((b[0], b[2], b[1], b[3])).save(
os.path.join(self.deconv_image_dir,
"{0:0>6}_{1:0>4}_{2:0>4}.png".format(n_processed + i, t, p)))
'''max_locs.extend(self.get_max_locs(
observation[self.lastname], self.layer_rank, indices))
bounds.extend(self.get_max_patch_bounds(
observation[self.lastname], self.layer_rank, indices))'''
n_processed += len(batch)
#self.add_to_confmat(self.confmat, in_vars[1].data, self.getpred(observation[self.lastname]))
summary.add(observation)
#print(self.confmat)
#print(np.diag(self.confmat))
#print(1.0 * np.diag(self.confmat).sum() / self.confmat.sum())
return summary.compute_mean(), max_locs, bounds
def evaluate(self):
"""override method of extensions.Evaluator."""
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
max_locs = []
bounds = []
n_processed = 0
filter_idx = 0
pbar = tqdm(total=len(iterator.dataset))
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
if isinstance(in_arrays, tuple):
in_vars = tuple(variable.Variable(x, volatile='off')
for x in in_arrays)
eval_func(*in_vars)
elif isinstance(in_arrays, dict):
in_vars = {key: variable.Variable(x, volatile='off')
for key, x in six.iteritems(in_arrays)}
eval_func(**in_vars)
else:
in_var = variable.Variable(in_arrays, volatile='off')
eval_func(in_var)
pbar.update(len(batch))
indices = np.arange(filter_idx, filter_idx + len(batch)) % self.n_features
# deconv対象の層のVariableを取得
layer_variable = Vutil.get_variable(
observation[self.lastname], self.layer_rank)
# 最大値の位置の計算に必要な入力層の領域を取得
isfc = Vutil.has_fc_layer(layer_variable)
if isfc:
batch_bounds = Vutil.get_data_bounds(layer_variable)
else:
batch_bounds = Vutil.get_max_bounds(layer_variable, indices)
topk = np.arange(n_processed, n_processed + len(batch)) // self.n_features
input_data = Vutil.get_data_layer(layer_variable).data
for k, f, d, b in zip(topk, indices, input_data, batch_bounds):
#print(dir(d))
# deconvされた入力層に平均画像を足して画像化
img = ioutil.deprocess(d.get(), self.mean)
# 最大値の位置の計算に必要な入力層の領域だけクロッピングして保存
img.crop((b[0], b[2], b[1], b[3])).save(
os.path.join(self.patch_image_dir,
"{0:0>4}_{1:0>2}.png".format(f, k)))
if not isfc:
max_locs.extend(Vutil.get_max_locs(layer_variable, indices))
bounds.extend(batch_bounds)
filter_idx = (filter_idx + len(batch)) % self.n_features
n_processed += len(batch)
#self.add_to_confmat(self.confmat, in_vars[1].data, self.getpred(observation[self.lastname]))
summary.add(observation)
pbar.close()
#print(self.confmat)
#print(np.diag(self.confmat))
#print(1.0 * np.diag(self.confmat).sum() / self.confmat.sum())
return summary.compute_mean(), max_locs, bounds
