def evaluate(self):
from chainer import reporter
import copy
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
it = copy.copy(iterator)
summary = reporter.DictSummary()
for batch in it:
observation = {}
with reporter.report_scope(observation):
in_arrays = self.converter(batch, self.device)
if isinstance(in_arrays, tuple):
eval_func(*in_arrays)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays)
else:
eval_func(in_arrays)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
self.model.eval()
with torch.no_grad():
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
x = self.converter(batch, self.device)
self.model(*x)
summary.add(observation)
self.model.train()
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
while True:
batch = it.next()
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
with function.no_backprop_mode():
if isinstance(in_arrays, tuple):
eval_func(*in_arrays)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays)
else:
eval_func(in_arrays)
summary.add(observation)
if it.is_new_epoch:
break
out = summary.compute_mean()
print('#############################################', out)
return out
def __init__(self,
links,
statistics=default_statistics,
report_params=True,
report_grads=True,
prefix=None,
trigger=(1, 'epoch'),
skip_nan_params=False):
if not isinstance(links, (list, tuple)):
links = links,
self._links = links
if statistics is None:
statistics = {}
self._statistics = statistics
attrs = []
if report_params:
attrs.append('data')
if report_grads:
attrs.append('grad')
self._attrs = attrs
self._prefix = prefix
self._trigger = trigger_module.get_trigger(trigger)
self._summary = reporter.DictSummary()
self._skip_nan_params = skip_nan_params
def evaluate(self):
iterator = self._iterators['main']
it = copy.copy(iterator)
summary = reporter.DictSummary()
observation = {}
iter_times = 0
loss = 0
accuracy = 0
for batch in it:
x_data, true_label = zip(*batch)
x_data = self.xp.asarray(x_data).astype("f")
y_data = self.xp.asarray(true_label).astype("int8")
y_predict = self._targets["main"](x_data).data
loss += F.softmax_cross_entropy(y_predict, y_data)
accuracy += F.accuracy(y_predict, y_data)
iter_times += 1
observation["val/loss"] = loss / iter_times
observation["val/acc"] = accuracy / iter_times
reporter.report(observation)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if self.eval_hook:
self.eval_hook(self)
iterator.reset()
it = iterator
summary = reporter_module.DictSummary()
true_y = []
pred_y = []
for batch in it:
in_arrays = convert._call_converter(self.converter, batch,
self.device)
assert isinstance(in_arrays, tuple)
x, y = in_arrays
true_y.append(y)
pred_y.append(target.predict(x).data)
auc = roc_auc_score(
cuda.to_cpu(target.xp.concatenate(true_y, axis=0)),
cuda.to_cpu(target.xp.concatenate(pred_y, axis=0)),
)
summary.add({f'{self.name}/main/auc': auc})
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
x, y = self.get_batch(batch, target.xp)
observation = {}
with reporter_module.report_scope(observation):
with chainer.no_backprop_mode():
y_pred = target(x)
loss = F.softmax_cross_entropy(y_pred, y)
acc = F.accuracy(y_pred, y)
chainer.reporter.report({'loss': loss}, target)
chainer.reporter.report({'accuracy': acc}, target)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
#target = self._targets['main']
summary = reporter_module.DictSummary()
for name, target in six.iteritems(self._targets):
iterator = self._iterators['main']
#target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
#summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
with function.no_backprop_mode():
if isinstance(in_arrays, tuple):
eval_func(*in_arrays)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays)
else:
eval_func(in_arrays)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators["main"]
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, "reset"):
iterator.reset()
it = iterator
else:
warnings.warn('This iterator does not have the reset method. Evaluator '
'copies the iterator instead of resetting. This behavior is '
'deprecated. Please implement the reset method.',
DeprecationWarning)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
xs, adjs = self.converter(batch, self.device)
xp = cuda.get_array_module(xs)
with chainer.no_backprop_mode(), chainer.using_config("train", False):
hs = self.model(xs, adjs)
hs = F.reshape(hs, (-1, hs.shape[-1]))
xs = xs.reshape(-1)
loss = F.softmax_cross_entropy(hs, xs)
acc = F.accuracy(hs, xs)
reporter_module.report({"accuracy": acc, "ce_loss": loss})
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
# batch only has one minibatch utterance, which is specified by batch[0]
x = converter_kaldi(batch[0], self.reader)
self.model.eval()
self.model(x)
delete_feat(x)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
with function.no_backprop_mode():
loss, acc = eval_func(**in_arrays)
reporter_module.report({
'val/loss': loss.data,
'val/acc': acc.data
})
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
"""Main evaluate routine for CustomEvaluator."""
iterator = self._iterators['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
self.model.eval()
with torch.no_grad():
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
x = self.converter(batch, self.device)
if self.ngpu == 0:
self.model(*x)
else:
# apex does not support torch.nn.DataParallel
data_parallel(self.model, x, range(self.ngpu))
summary.add(observation)
self.model.train()
return summary.compute_mean()
def evaluate(self):
val_iter = self.get_iterator('main')
model = self.get_target('main')
it = copy.copy(val_iter)
summary = reporter.DictSummary()
res = []
for i, batch in enumerate(it):
observation = {}
with reporter.report_scope(observation):
imgs, pafs, heatmaps, ignore_mask = self.converter(
batch, self.device)
with function.no_backprop_mode():
x_data = imgs.astype(np.float32).transpose(0, 3, 1,
2) / 256 - 0.5
inferenced_pafs, inferenced_heatmaps = model(x_data)
loss, pafs_loss_log, heatmaps_loss_log = compute_loss(
inferenced_pafs, inferenced_heatmaps, pafs, heatmaps,
ignore_mask)
observation['val/loss'] = cuda.to_cpu(loss.data)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
eval_func = self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
warnings.warn(
'This iterator does not have the reset method. Evaluator '
'copies the iterator instead of resetting. This behavior is '
'deprecated. Please implement the reset method.',
DeprecationWarning)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for x_batch, t_batch in it:
observation = {}
with reporter_module.report_scope(observation):
with function.no_backprop_mode():
eval_func(x_batch, t_batch)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
warnings.warn(
'This iterator does not have the reset method. Evaluator '
'copies the iterator instead of resetting. This behavior is '
'deprecated. Please implement the reset method.',
DeprecationWarning)
it = copy.copy(iterator)
summary = reporter.DictSummary()
for batch in it:
observation = {}
with reporter.report_scope(observation):
in_arrays = convert._call_converter(self.converter, batch,
self.device)
xp = self.device.xp
X, Y = xp.array(in_arrays[0]), xp.array(in_arrays[1])
with function.no_backprop_mode():
eval_func(X, Y)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self.get_iterator("main")
target = self.get_target("main")
if hasattr(iterator, "reset"):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter.DictSummary()
for batch in it:
observation = {}
with reporter.report_scope(observation):
with function.no_backprop_mode():
in_arrays, t_arrays = self.converter(batch, self.device)
p_arrays = target.predict(in_arrays)
_, t_tag_sentences = list(
zip(*self.transform_func(in_arrays[0], t_arrays)))
_, p_tag_sentences = list(
zip(*self.transform_func(in_arrays[0], p_arrays)))
fscore = metrics.f1_score(t_tag_sentences, p_tag_sentences)
reporter.report({"loss": target(in_arrays, t_arrays)},
target)
reporter.report({"fscore": fscore}, target)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
it = copy.copy(iterator) # これがないと1回しかEvaluationが走らない
summary = reporter.DictSummary()
for batch in it:
observation = {}
with reporter.report_scope(observation):
inds = xp.argsort([-len(x[0]) for x in batch]).astype('i')
xs = [
self.xp.array(batch[i][0], dtype=self.xp.int32)
for i in inds
]
ts = [
self.xp.array(batch[i][2], dtype=self.xp.int32)
for i in inds
]
xxs = [[
self.xp.array(x, dtype=self.xp.int32) for x in batch[i][1]
] for i in inds]
hx = chainer.Variable(
self.xp.zeros((1, len(xs), self.unit + 50),
dtype=self.xp.float32))
cx = chainer.Variable(
self.xp.zeros((1, len(xs), self.unit + 50),
dtype=self.xp.float32))
loss = target(xs, hx, cx, xxs, ts, train=False)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for _ in range(min(len(iterator.dataset) // iterator.batch_size, self.num_iterations)):
batch = next(it, None)
if batch is None:
break
observation = {}
with reporter_module.report_scope(observation), chainer.using_config('train', False), chainer.using_config('enable_backprop', False):
in_arrays = self.converter(batch, self.device)
if isinstance(in_arrays, tuple):
eval_func(*in_arrays)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays)
else:
eval_func(in_arrays)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
self.model.eval()
if not torch_is_old:
torch.set_grad_enabled(False)
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
x = self.converter(batch)
self.model(x)
delete_feat(x)
summary.add(observation)
self.model.train()
if not torch_is_old:
torch.set_grad_enabled(True)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
unet = self._targets['unet']
#eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
ground_truth, data = self.converter(batch, self.device)
with chainer.using_config("train", False):
with chainer.no_backprop_mode():
predict = unet(data)
ground_truth = ground_truth[:, :, 20:72, 20:72, 20:72]
#observation['vali/unet/loss'] = self.loss_softmax_cross_entropy(predict,ground_truth)
observation['vali/unet/dice'] = self.dice_coefficent(
predict, ground_truth)
summary.add(observation)
#print(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
kwargs = {}
kwargs['train'] = False
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
with function.no_backprop_mode():
if isinstance(in_arrays, tuple):
eval_func(*in_arrays, **kwargs)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays, **kwargs)
else:
eval_func(in_arrays, **kwargs)
summary.add(observation)
observation = summary.compute_mean()
return observation
def evaluate(self):
val_iter = self.get_iterator('main')
model = self.get_target('main')
it = copy.copy(val_iter)
summary = reporter.DictSummary()
res = []
for i, batch in enumerate(it):
observation = {}
with reporter.report_scope(observation):
imgs, pafs, heatmaps, ignore_mask = self.converter(
batch, self.device)
with function.no_backprop_mode():
x_data = preprocess(imgs)
pafs_ys, heatmaps_ys = model(x_data)
loss, paf_loss_log, heatmap_loss_log = compute_loss(
imgs, pafs_ys, heatmaps_ys, pafs, heatmaps,
ignore_mask)
observation['val/loss'] = cuda.to_cpu(loss.data)
observation['val/paf'] = sum(paf_loss_log)
observation['val/heat'] = sum(heatmap_loss_log)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
'''evaluate over iterator'''
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
# for multi gpu calculation
chainer.cuda.get_device_from_id(self.device).use()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
# batch only has one minibatch utterance, which is specified by batch[0]
x = converter_kaldi(batch[0], self.reader)
with function.no_backprop_mode():
eval_func(x)
delete_feat(x)
summary.add(observation)
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
if self.eval_hook:
self.eval_hook(self)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
acc = 0
count = 0
observation = {}
with reporter_module.report_scope(observation):
for batch in it:
target.predictor.reset_state()
w, l = batch[0]
xp = cuda.cupy if self.device >= 0 else np
xp_words = [xp.array([_w], xp.int32) for _w in w]
predict = np.argmax(target.predictor(xp_words).data)
if predict == l:
acc += 1
count += 1
summary.add({'main/validation/accuracy': acc / float(count)})
return summary.compute_mean()
def evaluate(self):
"""override method of extensions.Evaluator."""
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func or target
if self.eval_hook:
self.eval_hook(self)
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
max_locs = []
bounds = []
filter_idx = 0
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
if isinstance(in_arrays, tuple):
in_vars = tuple(
variable.Variable(x, volatile='off')
for x in in_arrays)
eval_func(*in_vars)
elif isinstance(in_arrays, dict):
in_vars = {
key: variable.Variable(x, volatile='off')
for key, x in six.iteritems(in_arrays)
}
eval_func(**in_vars)
else:
in_var = variable.Variable(in_arrays, volatile='off')
eval_func(in_var)
indices = np.arange(filter_idx,
filter_idx + len(batch)) % self.n_features
if len(max_locs) == 0: print(indices)
max_locs.extend(
self.get_max_locs(observation[self.lastname], self.layer_rank,
indices))
bounds.extend(
self.get_max_patch_bounds(observation[self.lastname],
self.layer_rank, indices))
filter_idx = (filter_idx + len(batch)) % self.n_features
'''if max_loc is None:
max_loc = self.get_features(
observation[self.lastname], self.layer_rank, 'argmax')
#print(batch)
print(max_loc)
else:
xp = cuda.get_array_module(max_loc)
max_loc = xp.vstack((max_loc, self.get_features(
observation[self.lastname], self.layer_rank, 'argmax')))'''
#self.add_to_confmat(self.confmat, in_vars[1].data, self.getpred(observation[self.lastname]))
summary.add(observation)
#print(self.confmat)
#print(np.diag(self.confmat))
#print(1.0 * np.diag(self.confmat).sum() / self.confmat.sum())
return summary.compute_mean(), max_locs, bounds
def evaluate(self):
train_X, train_y = self.collect_prediction_for_train_data()
model = LinearSVC()
try:
model.fit(X=train_X, y=train_y)
except:
summary_dict = {}
summary_dict["validation/main/svm_map"] = 0
summary_dict["validation/main/svm_mrr"] = 0
summary_dict["validation/main/map"] = 0
summary_dict["validation/main/mrr"] = 0
return summary_dict
else:
iterator = self._iterators['dev']
target = self._targets['main']
# this is necessary for more-than-once-evaluation
it = copy.copy(iterator)
label_scores = []
svm_label_scores = []
summary = reporter.DictSummary()
for n, batch in enumerate(it):
observation = {}
with reporter.report_scope(observation):
padded_batch = self.converter(batch, device=self.device)
x1s = padded_batch['x1s']
x2s = padded_batch['x2s']
wordcnt = padded_batch['wordcnt']
wgt_wordcnt = padded_batch['wgt_wordcnt']
x1s_len = padded_batch['x1s_len']
x2s_len = padded_batch['x2s_len']
y = padded_batch['y']
y_score, sim_scores = target(x1s, x2s, wordcnt, wgt_wordcnt, x1s_len, x2s_len)
# compute loss
loss = F.sigmoid_cross_entropy(x=y_score, t=y).data
reporter.report({'loss': loss}, target)
# We evaluate WikiQA by MAP and MRR
# for direct evaluation
label_score = np.c_[y, y_score.data]
label_scores.append(label_score)
# for SVM/LR
x = np.concatenate([x.data for x in sim_scores] + [wordcnt, wgt_wordcnt, x1s_len, x2s_len], axis=1)
y_score = model.decision_function(x)
svm_label_score = np.c_[y, y_score]
svm_label_scores.append(svm_label_score)
summary.add(observation)
stats = compute_map_mrr(label_scores)
svm_stats = compute_map_mrr(svm_label_scores)
summary_dict = summary.compute_mean()
summary_dict["validation/main/svm_map"] = svm_stats.map
summary_dict["validation/main/svm_mrr"] = svm_stats.mrr
summary_dict["validation/main/map"] = stats.map
summary_dict["validation/main/mrr"] = stats.mrr
return summary_dict
def evaluate(self):
"""Evaluates the model and returns a result dictionary.
This method runs the evaluation loop over the validation dataset. It
accumulates the reported values to :class:`~chainer.DictSummary` and
returns a dictionary whose values are means computed by the summary.
Note that this function assumes that the main iterator raises
``StopIteration`` or code in the evaluation loop raises an exception.
So, if this assumption is not held, the function could be caught in
an infinite loop.
Users can override this method to customize the evaluation routine.
.. note::
This method encloses :attr:`eval_func` calls with
:func:`function.no_backprop_mode` context, so all calculations
using :class:`~chainer.FunctionNode`\\s inside
:attr:`eval_func` do not make computational graphs. It is for
reducing the memory consumption.
Returns:
dict: Result dictionary. This dictionary is further reported via
:func:`~chainer.report` without specifying any observer.
"""
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
if self.max_num_iterations is not None:
it = self.fixed_num_iterations_iterator(it)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
with function.no_backprop_mode():
if isinstance(in_arrays, tuple):
eval_func(*in_arrays)
elif isinstance(in_arrays, dict):
eval_func(**in_arrays)
else:
eval_func(in_arrays)
summary.add(observation)
return self.calculate_mean_of_summary(summary)
def evaluate(self):
iterator = self._iterators['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter.DictSummary()
xp = self.model.xp
with chainer.no_backprop_mode():
for batch in it:
observation = {}
data = self.converter(batch, self.device)
batch_size = len(batch)
pixel_log_sigma_test = xp.full(
(batch_size, 3) + self.model.hyperparams.image_size,
math.log(self.variance_scheduler.standard_deviation),
dtype="float32")
images = data['image']
viewpoints = data['viewpoint']
with reporter.report_scope(observation):
# Scene encoder
representation, query_images, query_viewpoints = encode_scene(
images, viewpoints, self.model, self.device)
# Compute empirical ELBO
(z_t_param_array, pixel_mean
) = self.model.sample_z_and_x_params_from_posterior(
query_images, query_viewpoints, representation)
(ELBO, bits_per_pixel, NLL,
KLD) = estimate_ELBO(xp, query_images, z_t_param_array,
pixel_mean, pixel_log_sigma_test,
batch_size)
MSE = cf.mean_squared_error(query_images, pixel_mean)
reporter.report(
{
'ELBO': float(ELBO.data),
'bits_per_pixel': float(bits_per_pixel.data),
'NLL': float(NLL.data),
'KLD': float(KLD.data),
'MSE': float(MSE.data)
}, self.model)
summary.add(observation)
# reporter.report({
# 'ELBO':float(ELBO.data),
# 'bits_per_pixel':float(bits_per_pixel.data),
# 'NLL':float(negative_log_likelihood.data),
# 'KLD':float(kl_divergence.data),
# 'MSE':float(mean_squared_error.data)
# })
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
eval_func = self.eval_func
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
count = 0
flag = 0
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
in_arrays = self.converter(batch, self.device)
with function.no_backprop_mode():
if not isinstance(in_arrays, tuple):
raise TypeError
images, labels = in_arrays
res = Queue(self.extract_func(target, images))
""" VGGのテストのフロ(ー)チャ(ート)
1. activationsが空になるまで繰り返す
2. pool_indexより、pool_activationsの空き領域にactivations
の先頭から入れる。pool_indexが0の場合、pool_labelを
同じlabelsで設定する。
3. pool_activationsの状態をチェック
4. 満タンなら5、そうでなければ1に移動する。
5. eval_funcを実行、pool_activationsを空にし、1へ移動する。
"""
# activations_index = 0
while not res.is_full:
if self.pool.is_empty:
label = labels[res.index]
set_remain(self.pool, res)
if self.pool.is_full:
eval_func(label)
self.pool.index = 0 # reset
summary.add(observation)
# TODO: print_reportで書き直すことは、無理そう
# evaluateはiterやepochの概念外。なのでロガーで書こう。
flag += 1
count += len(batch)
if flag % 100 == 0:
sys.stdout.write("\r{}/{} : {}".format(count, len(it.dataset),
summary.compute_mean()))
return summary.compute_mean()
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
it = copy.copy(iterator)
summary = reporter.DictSummary()
ys_final, ts_final, raw_xs = [], [], []
for batch in it:
# Read batch data and sort sentences in descending order for CRF layer
observation = {}
raw_words = [x['str_words'] for x in batch]
words = [self.xp.array(x['words']).astype('i') for x in batch]
chars = [
self.xp.array(y, dtype=self.xp.int32) for x in batch
for y in x['chars']
]
tags = self.xp.vstack(
[self.xp.array(x['tags']).astype('i') for x in batch])
# Init index to keep track of words
index_start = self.xp.arange(F.hstack(words).shape[0])
index_end = index_start + 1
index = self.xp.column_stack((index_start, index_end))
# Nest level + 1
max_depth = len(batch[0]['tags'][0])
sentence_len = xp.array([x.shape[0] for x in words])
section = xp.cumsum(sentence_len[:-1])
# Init
predicts_depths = self.xp.empty(
(0, self.xp.sum(sentence_len))).astype('i')
with reporter.report_scope(observation):
for depth in range(max_depth):
accuracy, loss, next, index, extend_predicts, words, chars = target(
chars, words, tags[:, depth], index, False)
predicts_depths = self.xp.vstack(
(predicts_depths, extend_predicts))
if not next:
break
summary.add(observation)
predicts_depths = self.xp.split(predicts_depths, section, axis=1)
ts_depths = self.xp.split(self.xp.transpose(tags), section, axis=1)
ys_final.extend(predicts_depths)
ts_final.extend(ts_depths)
raw_xs.extend(raw_words)
fmeasure = summary.compute_mean()
fmeasure['dev/main/fscore'] = evaluate(target, ys_final, ts_final,
raw_xs)
return fmeasure
