def __call__(self, fp, y):
mean_activation = F.mean(fp, axis=0)
rho = 0.01
zero_array = chainer.Variable(
numpy.zeros(mean_activation.shape, dtype=numpy.float32))
small_array = zero_array + 0.001
cond = (mean_activation.data != 0)
cond = chainer.Variable(cond)
mean_activation = F.where(cond, mean_activation, small_array)
self.kl_div = rho * F.sum(
F.where(
cond,
self.p * F.log(self.p / mean_activation) +
(1 - self.p) * F.log(
(1 - self.p) / (1 - mean_activation)), zero_array))
# sampling z
eps = numpy.random.uniform(0.0, 1.0,
fp.data.shape).astype(numpy.float32)
eps = chainer.Variable(eps)
if self.train == True:
z = self.logistic_func(fp - eps)
#z = fp
else:
z = fp
h = F.relu(self.l1(z))
h = F.relu(self.l2(h))
h = self.l3(h)
self.rec_loss = F.sigmoid_cross_entropy(h, y)
self.accuracy = F.binary_accuracy(h, y)
self.loss = self.rec_loss + self.kl_div
return self.loss, self.accuracy
def __call__(self, xs, labels): # B, T, F, 2048, labels: B, T, F, 12
xs = F.transpose(xs, (0, 2, 1, 3)) # B, F, T, 2048
orig_labels = labels
labels = F.transpose(labels, (0, 2, 1, 3)) # B, F, T, 12
mini_batch, frame_node, T, _ = xs.shape
xs = xs.reshape(xs.shape[0] * xs.shape[1], xs.shape[2], xs.shape[3])
labels = labels.reshape(labels.shape[0] * labels.shape[1],
labels.shape[2], labels.shape[3])
xs = list(F.separate(xs, axis=0)) # list of T, 2048
labels = list(F.separate(labels, axis=0)) # list of T, 12
output = F.stack(self.label_dep_rnn(xs, labels)) # B * F, T, 12
output = output.reshape(mini_batch, frame_node, T, -1)
output = F.transpose(output, (0, 2, 1, 3)) # B, T, F, D
output = output.reshape(-1, self.class_num) # B * T * F, 12
orig_labels = orig_labels.reshape(-1, self.class_num)
assert output.shape == orig_labels.shape
pick_index, accuracy_pick_index = self.get_loss_index(
output, orig_labels)
loss = F.sigmoid_cross_entropy(
output[list(pick_index[0]),
list(pick_index[1])], orig_labels[list(pick_index[0]),
list(pick_index[1])])
accuracy = F.binary_accuracy(
output[list(accuracy_pick_index[0]),
list(accuracy_pick_index[1])], orig_labels[[
list(accuracy_pick_index[0]),
list(accuracy_pick_index[1])
]])
return loss, accuracy
def __call__(self, x, y):
h = F.relu(self.l1(x))
h = F.relu(self.l2(h))
h = self.l3(h)
self.loss = F.sigmoid_cross_entropy(h, y)
self.accuracy = F.binary_accuracy(h, y)
return self.loss, self.accuracy
def __call__(self, ecfp, sequences, n2vc, n2vp, interactions):
z = self.cos_similarity(ecfp, sequences, n2vc, n2vp)
Z = self.fc6(z)
loss = F.sigmoid_cross_entropy(Z, interactions)
accuracy = F.binary_accuracy(Z, interactions)
report({'loss': loss, 'accuracy': accuracy}, self)
return loss
def __call__(self, x, t):
y = self.predictor(x)
if self.lastlayer == 1: # The number of last layer units = 1
loss = F.sigmoid_cross_entropy(y, t.reshape(len(t), 1))
accuracy = F.binary_accuracy(y, t.reshape(len(t), 1))
else: # The number of last layer units = 2
loss = F.softmax_cross_entropy(y, t)
accuracy = F.accuracy(y, t)
summary = F.classification_summary(y, t, beta=1.0)
precision = summary[0]
recall = summary[1]
f_value = summary[2]
reporter = Reporter()
observer = object()
reporter.add_observer('f_value:', observer)
observation = {}
with reporter.scope(observation):
reporter.report({'x': f_value}, observer)
report(
{
'loss': loss,
'accuracy': accuracy,
'precision': precision,
'recall': recall,
'f_value': f_value
}, self)
report(dict(('f_value_%d' % i, val) for i, val in enumerate(f_value)),
self)
return loss
def __call__(self, xs,
labels): # xs shape = B,T,F,C,H,W, labels= (batch, T, F, D)
assert xs.ndim == 6
assert labels.ndim == 4
with chainer.cuda.get_device_from_array(xs.data) as device:
fc_output = self.forward(xs) # B, T, F, 2048
if self.use_label_dependency:
loss, accuracy = self.label_dependency_layer(
fc_output, labels) # B, T, F, 2048
else:
fc_output = F.reshape(fc_output, shape=(-1, self.box_dim))
fc_output = self.score_fc(fc_output) # B * T * F, class_num
labels = self.xp.reshape(labels, (-1, self.class_num))
pick_index, accuracy_pick_index = self.get_loss_index(
fc_output, labels)
loss = F.sigmoid_cross_entropy(
fc_output[list(pick_index[0]),
list(pick_index[1])],
labels[list(pick_index[0]),
list(pick_index[1])])
accuracy = F.binary_accuracy(
fc_output[list(accuracy_pick_index[0]),
list(accuracy_pick_index[1])], labels[[
list(accuracy_pick_index[0]),
list(accuracy_pick_index[1])
]])
return loss, accuracy
def calc_acc(self, dis_out, label=0, plot_name='acc'):
prob = F.sigmoid(dis_out)
label = cp.array([[label] for i in range(len(dis_out))])
label = Variable(self.converter(label, self.device))
label = Variable(label.data.astype(int))
acc_real = F.binary_accuracy(prob - 0.5, label)
chainer.report({plot_name: acc_real}, self.dis)
def accuracy(y, t):
xp = backend.get_array_module(y)
acc_0, acc_1 = tupled_binary_accuracy(y, t)
reporter.report({
"accuracy_false": acc_0,
"accuracy_true": acc_1
}, classifier)
return F.binary_accuracy(y, xp.array(t))
def forward(self, inp, target=None):
out = self.calc(inp)
if not target is None:
loss = F.sigmoid_cross_entropy(out, target)
acc = F.binary_accuracy(out, target)
return loss, acc
else:
out = F.sigmoid(out)
return out.reshape(-1)
def evaluate(self, s1, s2, t):
# 0.5は-1にして評価対象から外す
t = (t * 2).astype(np.int32)
t = np.array(t)
t[t == 1] = -1
t = t // 2
o = s1 - s2
acc = F.binary_accuracy(o, t)
chainer.report({"accuracy": acc}, self)
return acc
def evaluate_roc(self, trainer):
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
y_total = np.array([]).reshape([0, len(self.label_name)])
t_total = np.array([]).reshape([0, len(self.label_name)])
for batch in it:
in_arrays = self.converter(batch, self.device)
with chainer.no_backprop_mode(), chainer.using_config('train', False):
y = eval_func(*in_arrays[:-1])
t = in_arrays[-1]
# y = F.sigmoid(y)
y_data = cuda.to_cpu(y.data)
t_data = cuda.to_cpu(t)
y_total = np.vstack([y_total, y_data])
t_total = np.vstack([t_total, t_data])
updater = trainer.updater
epoch = updater.iteration
out_dir = trainer.out
observation = {}
for label_index, label in enumerate(self.label_name):
y = y_total[:, label_index]
t = t_total[:, label_index]
index = numpy.where(t != -1)[0]
y = y[index]
t = t[index]
out_name = os.path.join(out_dir, str(epoch) + 'iteration_' + label + '_roc.pdf')
gather_data = self.comm.gather(np.vstack([t, y]))
if self.rank == 0:
gather_data = reduce(lambda x, y: np.hstack([x, y]), gather_data)
gather_t = np.array(gather_data[0], dtype=np.int)
gather_y = np.array(gather_data[1], dtype=np.float32)
plot_roc(y_true=gather_t, y_score=F.sigmoid(gather_y).data, out_name=out_name)
roc_auc = metrics.roc_auc_score(gather_t, F.sigmoid(gather_y).data)
with reporter.report_scope(observation):
reporter.report({'roc_auc_' + label: roc_auc}, self._targets['main'])
reporter.report({'loss': F.sigmoid_cross_entropy(gather_y, gather_t).data},
self._targets['main'])
reporter.report({'accuracy': F.binary_accuracy(gather_y, gather_t).data}, self._targets['main'])
return observation
def evaluate_roc(self, trainer):
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
y_total = []
t_total = []
length = it.dataset.__len__()
batchsize = it.batch_size
length = length // batchsize
from tqdm import tqdm
pbar = tqdm(total=length)
for batch in it:
in_arrays = self.converter(batch, self.device)
with chainer.no_backprop_mode(), chainer.using_config('train', False):
y = eval_func(*in_arrays[:-1])
t = in_arrays[-1]
y_data = cuda.to_cpu(y.data)
t_data = cuda.to_cpu(t)
y_total.extend(y_data)
t_total.extend(t_data)
pbar.update(1)
y_total = numpy.concatenate(y_total).ravel()
t_total = numpy.concatenate(t_total).ravel()
index = numpy.where(t_total != -1)[0]
y_total = y_total[index]
t_total = t_total[index]
d = {'label': t_total, 'score': y_total}
from pathlib import Path
out_path = Path('./valid_result') / str(self.epoch)
out_path.mkdir(exist_ok=True)
np.savez(out_path / ('validation_' + str(self.rank)), **d)
observation = {}
with reporter.report_scope(observation):
roc_auc = metrics.roc_auc_score(t_total, F.sigmoid(y_total).data)
with reporter.report_scope(observation):
reporter.report({'roc_auc_': roc_auc}, self._targets['main'])
reporter.report({'loss': F.sigmoid_cross_entropy(y_total, t_total).data},
self._targets['main'])
reporter.report({'accuracy': F.binary_accuracy(y_total, t_total).data}, self._targets['main'])
return observation
def tupled_binary_accuracy(y, t):
"""
Compte binary classification accuracy
Attributes
----------
y
The output predictions
t
The ground truth label
"""
xp = backend.get_array_module(y)
t_0 = cuda.to_cpu(t.copy())
t_1 = cuda.to_cpu(t.copy())
t_0[t_0 == 1] = -1
t_1[t_0 == 0] = -1
t_0 = xp.array(t_0)
t_1 = xp.array(t_1)
acc_0 = F.binary_accuracy(y, t_0)
acc_1 = F.binary_accuracy(y, t_1)
return acc_0, acc_1
def __call__(self, xs, bboxes, labels): # all shape is (B, T, F, D)
node_out = self.forward(xs) # node_out B,T,F,D
node_out = F.reshape(node_out, (-1, self.out_size))
node_labels = self.xp.reshape(labels, (-1, self.out_size))
pick_index, accuracy_pick_index = self.get_loss_index(node_out, node_labels)
loss = F.sigmoid_cross_entropy(node_out,
node_labels)
accuracy = F.binary_accuracy(node_out[list(accuracy_pick_index[0]), list(accuracy_pick_index[1])],
node_labels[[list(accuracy_pick_index[0]), list(accuracy_pick_index[1])]])
report_dict = {'loss': loss, "accuracy":accuracy}
chainer.reporter.report(report_dict,
self)
return loss
def __call__(self, roi_feature, labels):
# labels shape = B, T, F(9 or 8), 12
# roi_feature shape = B, T, F, D, where F is box number in one frame image
with chainer.cuda.get_device_from_array(roi_feature.data) as device:
node_out = self.forward(roi_feature, labels) # node_out B,T,F,D
node_out = F.reshape(node_out, (-1, self.out_size))
node_labels = self.xp.reshape(labels, (-1, self.out_size))
pick_index, accuracy_pick_index = self.get_loss_index(node_out, node_labels)
loss = F.sigmoid_cross_entropy(node_out[list(pick_index[0]), list(pick_index[1])],
node_labels[list(pick_index[0]), list(pick_index[1])])
accuracy = F.binary_accuracy(node_out[list(accuracy_pick_index[0]), list(accuracy_pick_index[1])],
node_labels[[list(accuracy_pick_index[0]), list(accuracy_pick_index[1])]])
return loss, accuracy
def forward(self, xin, targets):
"""Compute total loss to train."""
vctx, vq, va, supps = xin # (B, R, P, C), (B, Q), (B,), (B, I)
# ---------------------------
# Compute main loss
predictions = self.predictor(xin) # (B,)
mainloss = F.sigmoid_cross_entropy(predictions, targets) # ()
acc = F.binary_accuracy(predictions, targets) # ()
# ---------------------------
# Compute aux losses
oattloss = F.stack(self.predictor.log['raw_att'], 1) # (B, I, R)
oattloss = F.reshape(oattloss, (-1, vctx.shape[1])) # (B*I, R)
oattloss = F.softmax_cross_entropy(oattloss, supps.flatten()) # ()
# ---
C.report({'loss': mainloss, 'oatt': oattloss, 'acc': acc}, self)
return mainloss + STRONG * oattloss # ()
def __call__(self, xs, crf_pact_structures):
loss = 0.0
xp = chainer.cuda.cupy.get_array_module(xs.data)
accuracy = 0.0
for idx, X in enumerate(xs):
crf_pact_structure = crf_pact_structures[idx]
gt_label = self.get_gt_label_one_graph(xp,
crf_pact_structure,
is_bin=True) # N x Y
A = crf_pact_structure.A
for i in range(self.layers_num):
X = getattr(self, "attn_{}".format(i))(X, A)
loss += F.sigmoid_cross_entropy(X, gt_label)
accuracy = F.binary_accuracy(X, gt_label)
chainer.reporter.report({"loss": loss, "accuracy": accuracy})
return loss
def __call__(self, roi_score, gt_roi_label): # shape = B, T, F, D (D can be 22(label_number) or 2048)
with chainer.cuda.get_device_from_array(roi_score.data) as device:
roi_score = roi_score.reshape(-1, roi_score.shape[-1])
gt_roi_label = gt_roi_label.reshape(-1, gt_roi_label.shape[-1])
assert roi_score.shape == gt_roi_label.shape
union_gt = set() # union of prediction positive and ground truth positive
cpu_gt_roi_label = chainer.cuda.to_cpu(gt_roi_label)
gt_pos_index = np.nonzero(cpu_gt_roi_label)
cpu_pred_score = (chainer.cuda.to_cpu(roi_score.data) > 0).astype(np.int32)
pred_pos_index = np.nonzero(cpu_pred_score)
len_gt_pos = len(gt_pos_index[0]) if len(gt_pos_index[0]) > 0 else 1
neg_pick_count = self.neg_pos_ratio * len_gt_pos
gt_pos_index_set = set(list(zip(*gt_pos_index)))
pred_pos_index_set = set(list(zip(*pred_pos_index)))
union_gt.update(gt_pos_index_set)
union_gt.update(pred_pos_index_set)
false_positive_index = np.asarray(list(pred_pos_index_set - gt_pos_index_set)) # shape = n x 2
gt_pos_index_lst = list(gt_pos_index_set)
if neg_pick_count <= len(false_positive_index):
choice_fp = np.random.choice(np.arange(len(false_positive_index)), size=neg_pick_count, replace=False)
gt_pos_index_lst.extend(list(map(tuple, false_positive_index[choice_fp].tolist())))
else:
gt_pos_index_lst.extend(list(map(tuple, false_positive_index.tolist())))
rest_pick_count = neg_pick_count - len(false_positive_index)
gt_neg_index = np.where(cpu_gt_roi_label == 0)
gt_neg_index_set = set(list(zip(*gt_neg_index)))
gt_neg_index_set = gt_neg_index_set - set(gt_pos_index_lst) # remove already picked
gt_neg_index_array = np.asarray(list(gt_neg_index_set))
rest_pick_count = len(gt_neg_index_array) if len(gt_neg_index_array) < rest_pick_count else rest_pick_count
choice_rest = np.random.choice(np.arange(len(gt_neg_index_array)), size=rest_pick_count, replace=False)
gt_pos_index_lst.extend(list(map(tuple, gt_neg_index_array[choice_rest].tolist())))
# TODO need class imbalance? NO
pick_index = list(zip(*gt_pos_index_lst))
if len(union_gt) == 0:
accuracy_pick_index = np.where(cpu_gt_roi_label)
else:
accuracy_pick_index = list(zip(*union_gt))
accuracy = F.binary_accuracy(roi_score[list(accuracy_pick_index[0]), list(accuracy_pick_index[1])],
gt_roi_label[list(accuracy_pick_index[0]), list(accuracy_pick_index[1])])
loss = F.sigmoid_cross_entropy(roi_score[list(pick_index[0]), list(pick_index[1])],
gt_roi_label[list(pick_index[0]), list(pick_index[1])]) # 支持多label
chainer.reporter.report({
'loss': loss, "accuracy": accuracy},
self)
return loss, accuracy
def __call__(self, xs, crf_pact_structures): # xs is chainer.Variable
'''
only support batch_size = 1
some example of NStepLSTM : https://github.com/kei-s/chainer-ptb-nsteplstm/blob/master/train_ptb_nstep.py#L24
:return : chainer.Variable shape= B * N * D , B is batch_size, N is one video all nodes count, D is each node output vector
'''
xp = chainer.cuda.get_array_module(xs.data)
hs = self.get_output(xs) # B x N x D, B = 1
ts = self.get_gt_labels(xp, crf_pact_structures=crf_pact_structures)
ts = chainer.Variable(ts)
hs = hs.reshape(-1, hs.shape[-1])
ts = ts.reshape(-1, ts.shape[-1])
loss = F.sigmoid_cross_entropy(hs, ts)
accuracy = F.binary_accuracy(hs, ts)
chainer.reporter.report({'loss': loss, 'accuracy': accuracy}, self)
return loss
def __call__(self, x, t, train=False):
y = self.predictor(x)
if self.lastlayer == 1: # The number of last layer units = 1
loss = F.sigmoid_cross_entropy(y, t.reshape(len(t), 1))
accuracy = F.binary_accuracy(y, t.reshape(len(t), 1))
f1 = F.f1_score(y, t)
else: # The number of last layer units = 2
loss = F.softmax_cross_entropy(y, t)
accuracy = F.accuracy(y, t).data
f1 = F.f1_score(y, t)[0].data
# f1 = F.f1_score(y, t)
summary = F.classification_summary(y, t, beta=1.0)
precision = summary[0]
recall = summary[1]
f_value = summary[2]
return accuracy.min(), f1[0]
def forward(self, texts, labels):
"""Compute total loss to train."""
# texts [(L1,), (L2,), (L3,)]
# labels (B,)
report = dict()
r = self.predictor(texts)
# ---------------------------
# Compute loss and accs
labels = labels[:, None] # (B, 1)
ilabels = self.predictor.inv_examples[1][:, None] # (I, 1)
for k, t in [('o', labels), ('u', labels), ('ig', ilabels)]:
report[k + 'loss'] = F.sigmoid_cross_entropy(r[k + 'pred'], t)
report[k + 'acc'] = F.binary_accuracy(r[k + 'pred'], t)
# ---------------------------
# Aux lossess
vloss = F.mean(r['vmap']) # ()
report['vloss'] = vloss
# ---------------------------
C.report(report, self)
return self.uniparam * (report['uloss'] +
0.1 * report['vloss']) + report['oloss']
def __call__(self, xs,
labels): # xs shape = B,T,F,C,H,W, labels= (batch, T, F, D)
assert xs.ndim == 4
assert labels.ndim == 4
with chainer.cuda.get_device_from_array(xs.data) as device:
fc_output = self.forward(xs) # B, T, F, 2048
fc_output = F.reshape(fc_output, (-1, self.class_num))
labels = self.xp.reshape(labels, (-1, self.class_num))
pick_index, accuracy_pick_index = self.get_loss_index(
fc_output, labels)
loss = F.sigmoid_cross_entropy(
fc_output[list(pick_index[0]),
list(pick_index[1])], labels[list(pick_index[0]),
list(pick_index[1])])
accuracy = F.binary_accuracy(
fc_output[list(accuracy_pick_index[0]),
list(accuracy_pick_index[1])], labels[[
list(accuracy_pick_index[0]),
list(accuracy_pick_index[1])
]])
return loss, accuracy
def __call__(self, xs,
labels): # xs shape = B,T,F,C,H,W, labels= (batch, T, F, D)
assert xs.ndim == 6
assert labels.ndim == 4
fc_output = self.forward(xs) # B, T, F, 2048
fc_output = F.reshape(fc_output, shape=(-1, self.box_dim))
fc_output = self.score_fc(fc_output) # B * T * F, class_num
labels = self.xp.reshape(labels, (-1, self.class_num))
pick_index, accuracy_pick_index = self.get_loss_index(
fc_output, labels)
loss = F.sigmoid_cross_entropy(
fc_output[list(pick_index[0]),
list(pick_index[1])], labels[list(pick_index[0]),
list(pick_index[1])])
accuracy = F.binary_accuracy(
fc_output[list(accuracy_pick_index[0]),
list(accuracy_pick_index[1])], labels[[
list(accuracy_pick_index[0]),
list(accuracy_pick_index[1])
]])
return loss, accuracy
def forward(self, inputs, device):
x, t = inputs
return functions.binary_accuracy(x, t),
def forward(self, inputs, device):
x, t = inputs
return functions.binary_accuracy(x, t),
from __future__ import print_function, unicode_literals
import numpy as np
import chainer
import chainer.functions as F
if __name__ == '__main__':
a = np.array([[0.0, -0.5], [0.0, 0.5]])
print(a)
t = np.array([0, 0], 'i')
tb = np.array([[1, 0], [0, 1]], 'i')
print(t)
print(F.accuracy(a, t).data)
print(F.binary_accuracy(a, tb).data)
tc = np.array([[1, 0], [1, 0], [0, 1], [0, 1]])
print(np.where(tc))
print(np.where(tc)[1])
x = np.random.random((10, 4))
b1 = (0, 2)
b2 = (2, 4)
print(x)
print(x[:, slice(*b1)])
print(x[:, slice(*b2)])
def __call__(self, imgs, label):
"""Forward backbone and sigmoid cross entropy loss.
support batch_size > 1 train
Here are notations used.
* :math:`N` is the batch size.
Args:
imgs (~chainer.Variable): A variable with a batch of images. shape is (N C H W)
Returns:
chainer.Variable:
Scalar loss variable.
This is the sum of losses for Region Proposal Network and
the head module.
"""
xp = cuda.get_array_module(imgs)
with cuda.get_device_from_array(imgs) as device:
_, _, H, W = imgs.shape
score = self.backbone(imgs)
# 算sigmoid的时候，从gt中挑选=1的元素，然后从pred=1 但gt=0中挑选元素，如果还不够再从剩下的随机挑选凑够x 3倍的=0元素，最后算sigmoid cross entropy
union_gt = set() # union of prediction positive and ground truth positive
cpu_gt_roi_label = chainer.cuda.to_cpu(label)
gt_pos_index = np.nonzero(cpu_gt_roi_label)
cpu_pred_score = (chainer.cuda.to_cpu(score.data) > 0).astype(np.int32)
pred_pos_index = np.nonzero(cpu_pred_score)
len_gt_pos = len(gt_pos_index[0]) if len(gt_pos_index[0]) > 0 else 1
neg_pick_count = self.neg_pos_ratio * len_gt_pos
gt_pos_index_set = set(list(zip(*gt_pos_index)))
pred_pos_index_set = set(list(zip(*pred_pos_index)))
union_gt.update(gt_pos_index_set)
union_gt.update(pred_pos_index_set)
false_positive_index = np.asarray(list(pred_pos_index_set - gt_pos_index_set)) # shape = n x 2
gt_pos_index_lst = list(gt_pos_index_set)
if neg_pick_count <= len(false_positive_index):
choice_fp = np.random.choice(np.arange(len(false_positive_index)), size=neg_pick_count, replace=False)
gt_pos_index_lst.extend(list(map(tuple,false_positive_index[choice_fp].tolist())))
else:
gt_pos_index_lst.extend(list(map(tuple, false_positive_index.tolist())))
rest_pick_count = neg_pick_count - len(false_positive_index)
gt_neg_index = np.where(cpu_gt_roi_label == 0)
gt_neg_index_set = set(list(zip(*gt_neg_index)))
gt_neg_index_set = gt_neg_index_set - set(gt_pos_index_lst) # remove already picked
gt_neg_index_array = np.asarray(list(gt_neg_index_set))
rest_pick_count = min(rest_pick_count, len(gt_neg_index_array))
choice_rest = np.random.choice(np.arange(len(gt_neg_index_array)), size=rest_pick_count, replace=False)
gt_pos_index_lst.extend(list(map(tuple,gt_neg_index_array[choice_rest].tolist())))
pick_index = list(zip(*gt_pos_index_lst))
if len(union_gt) == 0:
accuracy_pick_index = np.where(cpu_gt_roi_label)
else:
accuracy_pick_index = list(zip(*union_gt))
accuracy = F.binary_accuracy(score[list(accuracy_pick_index[0]), list(accuracy_pick_index[1])],
label[list(accuracy_pick_index[0]), list(accuracy_pick_index[1])])
loss = F.sigmoid_cross_entropy(score,
label)
chainer.reporter.report({
'loss': loss, "accuracy": accuracy},
self)
return loss
itr += 1
sum_loss += loss.data
itr_epoch += 1
sum_loss_epoch += loss.data
# print train loss and test accuracy
if optimizer.t % eval_interval == 0:
x1, x2, t = mini_batch(np.random.choice(test_data, 640))
with chainer.no_backprop_mode():
with chainer.using_config('train', False):
y = model(x1, x2)
loss = F.sigmoid_cross_entropy(y, t)
logging.info('epoch = {}, iteration = {}, train loss = {}, test loss = {}, test accuracy = {}'.format(
optimizer.epoch + 1, optimizer.t, sum_loss / itr,
loss.data,
F.binary_accuracy(y, t).data))
itr = 0
sum_loss = 0
# validate test data
itr_test = 0
sum_test_loss = 0
sum_test_accuracy = 0
for i in range(0, len(test_data) - args.batchsize, args.batchsize):
x1, x2, t = mini_batch(test_data[i:i+args.batchsize])
with chainer.no_backprop_mode():
with chainer.using_config('train', False):
y = model(x1, x2)
itr_test += 1
sum_test_loss += F.sigmoid_cross_entropy(y, t).data
sum_test_accuracy += F.binary_accuracy(y, t).data
itr += 1
sum_loss += loss.data
itr_epoch += 1
sum_loss_epoch += loss.data
# print train loss and test accuracy
if optimizer.t % args.eval_interval == 0:
x, t1, t2 = mini_batch_for_test(positions_test,
args.test_batchsize)
y1, y2 = model(x)
logging.info(
'epoch = {}, iteration = {}, loss = {}, accuracy = {}, {}'.
format(optimizer.epoch + 1, optimizer.t, sum_loss / itr,
F.accuracy(y1, t1).data,
F.binary_accuracy(y2, t2).data))
itr = 0
sum_loss = 0
# validate test data
logging.info('validate test data')
itr_test = 0
sum_test_accuracy1 = 0
sum_test_accuracy2 = 0
for i in range(0, len(positions_test) - args.batchsize, args.batchsize):
x, t1, t2 = mini_batch(positions_test, i, args.batchsize)
y1, y2 = model(x)
itr_test += 1
sum_test_accuracy1 += F.accuracy(y1, t1).data
sum_test_accuracy2 += F.binary_accuracy(y2, t2).data
logging.info(
dataset = LogicDataset(batch_size, seq_len, max_ops, min_ops)
dim_vector = dataset.dim_vector
model = ACTNet(dim_vector, state_size, 1)
if use_gpu:
model.to_gpu()
optimizer = chainer.optimizers.Adam(learning_rate)
optimizer.setup(model)
optimizer.use_cleargrads(True)
loss_log = []
for i in range(1000000):
print('{}:'.format(i), end=' ')
x, t = next(dataset)
y, ponder_cost = model(x)
loss = F.sigmoid_cross_entropy(y, t) + time_penalty * ponder_cost
model.cleargrads()
loss.backward()
loss_log.append(chainer.cuda.to_cpu(loss.data))
optimizer.update()
accuracy = F.binary_accuracy(y, t)
print('acc:', accuracy.data)
if i % 50 == 0:
plt.plot(loss_log, '.', markersize=1)
plt.grid()
plt.show()
def __call__(self, imgs, bboxes, labels):
"""Forward Faster R-CNN and calculate losses.
support batch_size > 1 train
Here are notations used.
* :math:`N` is the batch size.
* :math:`R` is the number of bounding boxes per image.
* :math:`L` is the number of Action Unit Set(in config.py define)
Currently, only :math:`N=1` is supported.
Args:
imgs (~chainer.Variable): A variable with a batch of images. shape is (N C H W)
bboxes (~chainer.Variable): A batch of ground truth bounding boxes.
Its shape is :math:`(N, R, 4)`.
labels (~chainer.Variable): A batch of labels.
Its shape is :math:`(N, R, L)`. this is for the multi-label region,
The background is excluded from the definition, which means that the range of the value
is :math:`[-1,0,1]`.0 means this AU index does not occur. -1 means ignore_label
classes.
Returns:
chainer.Variable:
Scalar loss variable.
This is the sum of losses for Region Proposal Network and
the head module.
"""
xp = cuda.get_array_module(imgs)
with cuda.get_device_from_array(imgs) as device:
if isinstance(bboxes, chainer.Variable):
bboxes = bboxes.data
if isinstance(labels, chainer.Variable):
labels = labels.data
batch_size = bboxes.shape[0]
_, _, H, W = imgs.shape
# Since batch size is one, convert variables to singular form
if batch_size > 1:
rois = []
roi_batch_indices = []
gt_roi_label = []
for n in range(batch_size):
bbox = bboxes[n] # bbox仍然是一个list，表示一个图内部的bbox
label = labels[n] # label仍然是一个list，表示一个图内部的label
# 若其中的一个label为-99 表示是padding的值，此时该bbox是[-99,-99,-99,-99]
bbox = bbox[bbox != xp.array(-99)].reshape(-1, 4)
label = label[label != xp.array(-99)].reshape(
-1, len(config.AU_SQUEEZE))
assert label.shape[0] == bbox.shape[0] and bbox.shape[0] > 0
# Sample RoIs and forward
sample_roi = bbox
sample_roi_index = n * np.ones(
(len(sample_roi), ), dtype=xp.int32)
gt_roi_label.extend(label) # list 可以extend ndarray
rois.extend(sample_roi)
roi_batch_indices.extend(sample_roi_index)
rois = xp.stack(rois).astype(dtype=xp.float32)
gt_roi_label = xp.stack(gt_roi_label).astype(dtype=xp.int32)
roi_batch_indices = xp.asarray(roi_batch_indices).astype(
dtype=xp.int32)
elif batch_size == 1: # batch_size = 1
bbox = bboxes[0]
label = labels[0]
rois, gt_roi_label = bbox, label
roi_batch_indices = xp.zeros((len(rois), ), dtype=xp.int32)
roi_indices = roi_batch_indices.astype(np.float32)
indices_and_rois = self.xp.concatenate(
(roi_indices[:, None], rois),
axis=1) # None means np.newaxis, concat along column
roi_score = self.fpn(imgs, indices_and_rois)
roi_score = roi_score.reshape(-1, self.fpn.classes)
assert gt_roi_label.shape[-1] == self.fpn.classes
union_gt = set(
) # union of prediction positive and ground truth positive
cpu_gt_roi_label = chainer.cuda.to_cpu(gt_roi_label)
gt_pos_index = np.nonzero(cpu_gt_roi_label)
cpu_pred_score = (chainer.cuda.to_cpu(roi_score.data) > 0).astype(
np.int32)
pred_pos_index = np.nonzero(cpu_pred_score)
len_gt_pos = len(
gt_pos_index[0]) if len(gt_pos_index[0]) > 0 else 1
neg_pick_count = self.neg_pos_ratio * len_gt_pos
gt_pos_index_set = set(list(zip(*gt_pos_index)))
pred_pos_index_set = set(list(zip(*pred_pos_index)))
union_gt.update(gt_pos_index_set)
union_gt.update(pred_pos_index_set)
false_positive_index = np.asarray(
list(pred_pos_index_set - gt_pos_index_set)) # shape = n x 2
gt_pos_index_lst = list(gt_pos_index_set)
if neg_pick_count <= len(false_positive_index):
choice_fp = np.random.choice(np.arange(
len(false_positive_index)),
size=neg_pick_count,
replace=False)
gt_pos_index_lst.extend(
list(map(tuple, false_positive_index[choice_fp].tolist())))
else:
gt_pos_index_lst.extend(
list(map(tuple, false_positive_index.tolist())))
rest_pick_count = neg_pick_count - len(false_positive_index)
gt_neg_index = np.where(cpu_gt_roi_label == 0)
gt_neg_index_set = set(list(zip(*gt_neg_index)))
gt_neg_index_set = gt_neg_index_set - set(
gt_pos_index_lst) # remove already picked
gt_neg_index_array = np.asarray(list(gt_neg_index_set))
choice_rest = np.random.choice(np.arange(
len(gt_neg_index_array)),
size=rest_pick_count,
replace=False)
gt_pos_index_lst.extend(
list(map(tuple, gt_neg_index_array[choice_rest].tolist())))
pick_index = list(zip(*gt_pos_index_lst))
if len(union_gt) == 0:
accuracy_pick_index = np.where(cpu_gt_roi_label)
else:
accuracy_pick_index = list(zip(*union_gt))
accuracy = F.binary_accuracy(
roi_score[list(accuracy_pick_index[0]),
list(accuracy_pick_index[1])],
gt_roi_label[list(accuracy_pick_index[0]),
list(accuracy_pick_index[1])])
loss = F.sigmoid_cross_entropy(
roi_score[list(pick_index[0]),
list(pick_index[1])],
gt_roi_label[list(pick_index[0]),
list(pick_index[1])]) # 支持多label
chainer.reporter.report({'loss': loss, "accuracy": accuracy}, self)
return loss