def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
it = copy.copy(iterator)
all_gt_label = []
all_score = []
for idx, batch in enumerate(it):
print("processing idx: {}".format(idx))
batch = self.converter(batch, device=self.device)
imgs, bbox, labels = batch
if imgs is None:
continue
xp = chainer.cuda.get_array_module(imgs)
imgs = chainer.Variable(imgs)
bbox = chainer.Variable(bbox)
if bbox.shape[1] != config.BOX_NUM[self.database]:
print("error box num {0} != {1}".format(
bbox.shape[1], config.BOX_NUM[self.database]))
continue
scores = target.predict(imgs, bbox) # R', class_num
scores = scores.reshape(labels.shape[0], labels.shape[1],
labels.shape[2]) # shape = B,F,Y
labels = chainer.cuda.to_cpu(labels) # B, F, Y
scores = chainer.cuda.to_cpu(scores) # B, F, Y
labels = np.maximum.reduce(labels, axis=1) # B, Y
scores = np.maximum.reduce(scores, axis=1) # B, Y
all_gt_label.extend(labels)
all_score.extend(scores)
all_gt_label = np.asarray(all_gt_label) # shape = (N, 5)
all_score = np.asarray(all_score) # shape = (N, 5)
all_gt_label = np.transpose(all_gt_label) # 5, N
all_score = np.transpose(all_score) # 5, N
report = defaultdict(dict)
reporter = Reporter()
reporter.add_observer("main", target)
summary = DictSummary()
for idx, score in enumerate(all_score):
AU = config.AU_INTENSITY_DICT[idx]
gt_label = all_gt_label[idx]
error = mean_squared_error(gt_label, score)
pearson_correlation, _ = pearsonr(gt_label, score)
report["mean_squared_error"][AU] = error
report["pearson_correlation"][AU] = pearson_correlation
summary.add({"pearson_correlation_avg": pearson_correlation})
summary.add({"mean_squared_error_avg": error})
observation = {}
with reporter.scope(observation):
reporter.report(report, target)
reporter.report(summary.compute_mean(), target)
return observation
def evaluate(self):
iterator = self._iterators['main']
_target = self._targets["main"]
if hasattr(iterator, 'reset'):
iterator.reset_for_train_mode()
it = iterator
else:
it = copy.copy(iterator)
reporter = Reporter()
reporter.add_observer("main", _target)
summary = DictSummary()
model = _target
pred_labels_array = []
gt_labels_array = []
for idx, batch in enumerate(it):
print("processing :{}".format(idx))
batch = self.converter(batch, self.device)
feature, boxes, labels = batch # each is B,T,F,D
if not isinstance(feature, chainer.Variable):
feature = chainer.Variable(feature)
labels = np.bitwise_or.reduce(
chainer.cuda.to_cpu(labels)[:, -1, :, :],
axis=1) # B, class_number
pred_labels = model.predict(
feature) # B, T, F, D -> B, class_number
pred_labels_array.extend(pred_labels)
gt_labels_array.extend(labels)
gt_labels_array = np.stack(gt_labels_array)
pred_labels_array = np.stack(
pred_labels_array) # shape = all_N, out_size
gt_labels = np.transpose(gt_labels_array) # shape = Y x frame
pred_labels = np.transpose(pred_labels_array) #shape = Y x frame
report_dict = dict()
AU_id_convert_dict = self.AU_convert if self.AU_convert else config.AU_SQUEEZE
for new_AU_idx, frame_pred in enumerate(pred_labels):
if AU_id_convert_dict[new_AU_idx] in self.paper_use_AU:
AU = AU_id_convert_dict[new_AU_idx]
frame_gt = gt_labels[new_AU_idx]
F1 = f1_score(y_true=frame_gt, y_pred=frame_pred)
report_dict[AU] = F1
summary.add({"f1_frame_avg": F1})
observation = {}
with reporter.scope(observation):
reporter.report(report_dict, model)
reporter.report(summary.compute_mean(), model)
return observation
def evaluate(self):
iterator = self._iterators['main']
_target = self._targets["main"]
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
reporter = Reporter()
reporter.add_observer("main", _target)
summary = DictSummary()
model = _target
pred_labels_array = []
gt_labels_array = []
for idx, batch in enumerate(it):
batch = self.converter(batch, self.device)
images, bboxes, labels = batch # images shape = B*T, C, H, W; bboxes shape = B*T, F, 4; labels shape = B*T, F, 12
if not isinstance(images, chainer.Variable):
images = chainer.Variable(images.astype('f'))
bboxes = chainer.Variable(bboxes.astype('f'))
roi_feature, labels = model.get_roi_feature(images,, bboxes, labels
print("evaluate: Idx:{}".format(idx))
pred_labels = model.loss_head_module.predict(roi_feature) # B, T, F, 12
pred_labels = pred_labels[:, self.T-1, :, :] # B, F, D
pred_labels = np.bitwise_or.reduce(pred_labels, axis=1) # B, class_number
labels = labels[:, self.T-1, :, :] # B, F, D
labels = np.bitwise_or.reduce(labels, axis=1) # B, class_number
assert labels.shape == pred_labels.shape
pred_labels_array.extend(pred_labels)
gt_labels_array.extend(labels)
if idx > 100:
break
gt_labels_array = np.stack(gt_labels_array)
pred_labels_array = np.stack(pred_labels_array) # shape = all_N, out_size
gt_labels = np.transpose(gt_labels_array) # shape = Y x frame
pred_labels = np.transpose(pred_labels_array) #shape = Y x frame
AU_id_convert_dict = self.AU_convert if self.AU_convert else config.AU_SQUEEZE
for new_AU_idx, frame_pred in enumerate(pred_labels):
if AU_id_convert_dict[new_AU_idx] in self.paper_use_AU:
# AU = AU_id_convert_dict[new_AU_idx]
frame_gt = gt_labels[new_AU_idx]
F1 = f1_score(y_true=frame_gt, y_pred=frame_pred)
summary.add({"f1_frame_avg": F1})
observation = {}
with reporter.scope(observation):
reporter.report(summary.compute_mean(), model)
return observation
def evaluate(self):
iterator = self._iterators['main']
_target = self._targets["main"]
if hasattr(iterator, 'reset'):
iterator.reset_for_train_mode()
it = iterator
else:
it = copy.copy(iterator)
reporter = Reporter()
reporter.add_observer("main", _target) # will fail to run?
summary = DictSummary()
video_gt_bin_dict = defaultdict(list) # key = video_id, value = predict bin list
video_pred_bin_dict = defaultdict(list) # key = video_id, value = predict bin list
video_pred_prob_dict = defaultdict(list)
for batch in it:
batch = convert(batch, self.device)
for x, crf_pact_structure in zip(*batch):
sample = crf_pact_structure.sample
file_path = sample.file_path
train_keyword = os.path.basename(os.path.dirname(file_path)) # train_keyword comes from file_path
video_id = os.path.basename(file_path)
if train_keyword not in self.target_dict:
print("error {} not pre-trained".format(train_keyword))
continue
target = self.target_dict[train_keyword] # choose the right predictor
# pred_probs is N x (Y+1)
pred_labels, pred_probs = target.predict(x, crf_pact_structure, is_bin=False) # pred_labels is N x 1, but open-crf predict only produce shape = N
gt_labels = target.get_gt_label_one_graph(np, crf_pact_structure, is_bin=False) # return N x 1
assert pred_labels.ndim == 1
pred_bins = [] # pred_bins is labels in one video sequence
gt_bins = [] # gt_bins is labels in one video sequence
prod_prob_bins = []
for idx, pred_label in enumerate(pred_labels): # N times iterator, N is number of nodes
pred_prob = pred_probs[idx] # probability is len = Y+1
pred_prob_bin = np.zeros(shape=(len(config.AU_SQUEEZE)+1), dtype=np.float32) # Y + 1 because pred=0 also have prob
for pred_idx in range(pred_prob.shape[0]):
if pred_idx == 0:
pred_prob_bin[0] = pred_prob[0] # 第0位置上表示全都是0
else:
AU = train_keyword.split("_")[pred_idx - 1]
AU_idx = config.AU_SQUEEZE.inv[AU]
pred_prob_bin[AU_idx + 1] = pred_prob[pred_idx]
prod_prob_bins.append(pred_prob_bin) # list of Y + 1
pred_bin = np.zeros(shape=len(config.AU_SQUEEZE), dtype=np.int32) # shape = Y
if pred_label > 0:
AU = train_keyword.split("_")[pred_label - 1]
AU_idx = config.AU_SQUEEZE.inv[AU]
pred_bin[AU_idx] = 1 # CRF can only predict one label, translate to AU_squeeze length
pred_bins.append(pred_bin)
for gt_label in gt_labels: # N times iterator, N is number of nodes
gt_bin = np.zeros(shape=len(config.AU_SQUEEZE), dtype=np.int32) # shape = Y
if gt_label > 0:
AU = train_keyword.split("_")[gt_label - 1]
AU_idx = config.AU_SQUEEZE.inv[AU]
gt_bin[AU_idx] = 1
gt_bins.append(gt_bin)
pred_bins = np.asarray(pred_bins) # shape = N x Y (Y is AU_squeeze length)
gt_bins = np.asarray(gt_bins)
prod_prob_bins = np.asarray(prod_prob_bins)
assert len(pred_bins) == len(sample.node_list)
assert len(gt_bins) == len(sample.node_list)
video_pred_bin_dict[video_id].append(pred_bins) # each pred_bins is shape = N x Y. but N of each graph is different
video_gt_bin_dict[video_id].append(gt_bins) # each gt_bins is shape = N x Y. but N of each graph is different
video_pred_prob_dict[video_id].append(prod_prob_bins) # each pred_probs = N x (Y+1)
assert len(video_gt_bin_dict) == len(video_pred_bin_dict)
# predict final is determined by vote
video_pred_final = [] # shape = list of N x Y ,each N is different in each video
video_gt_final = [] # shape = list of N x Y, each N is different
for video_id, prod_prob_bins in video_pred_prob_dict.items():
prod_prob_bins_array = np.asarray(prod_prob_bins) # shape = U x N x (Y+1) , where U is different trainer number, this time N is the same cross diferent video
prod_prob_bins_array = np.transpose(prod_prob_bins_array,(1,0,2)) # shape = N x U x (Y+1)
prod_prob_bins_index = np.argmax(prod_prob_bins_array, axis=2) # shape = N x U choose the biggest Y index in last axis
prod_prob_bins_array = np.max(prod_prob_bins_array, axis=2) # shape = N x U. each element is prob number
choice_trainer_index = np.argmax(prod_prob_bins_array, axis=1) # shape = N, each element is which U is biggest
pred_labels = prod_prob_bins_index[np.arange(len(prod_prob_bins_index)), choice_trainer_index] #shape = N, each element is correct Y
pred_bins_array = np.zeros(shape=(pred_labels.shape[0], len(config.AU_SQUEEZE)),dtype=np.int32)
for pred_idx, pred_label in enumerate(pred_labels):
if pred_label != 0:
pred_bins_array[pred_idx, pred_label - 1] = 1
video_pred_final.append(pred_bins_array) # list of N x Y
# for gt_label part, we don't need vote, we only need element-wise or
gt_bins_array = np.asarray(video_gt_bin_dict[video_id]) # # shape = U x N x Y , where U is different trainer number
gt_bins_array = np.transpose(gt_bins_array, axes=(1,0,2)) # shape = N x U x Y
video_gt_final.append(np.bitwise_or.reduce(gt_bins_array, axis=1)) # list shape = N x Y
video_pred_final = np.concatenate(video_pred_final, axis=0) # shape = N' x Y ,where N' is total nodes of all frames cross videos
video_gt_final = np.concatenate(video_gt_final, axis=0) # shape = N' x Y ,where N' is total nodes of all frames cross videos
box_num = config.BOX_NUM[self.database]
# we suppose that the n nodes order as frame, each frame have 9/8 boxes
pred_labels_batch = video_pred_final.reshape(-1, box_num, len(config.AU_SQUEEZE)) # shape = (V x Frame) x box_num x Y
gt_labels_batch = video_gt_final.reshape(-1, box_num, len(config.AU_SQUEEZE)) # shape = (V x Frame) x box_num x Y
pred_labels_batch = np.bitwise_or.reduce(pred_labels_batch, axis=1) # shape = (V x Frame) x Y
gt_labels_batch = np.bitwise_or.reduce(gt_labels_batch, axis=1) # shape = (V x Frame) x Y
gt_labels_batch = np.transpose(gt_labels_batch, (1,0)) #shape = Y x N'. where N' = (V x Frame)
pred_labels_batch = np.transpose(pred_labels_batch, (1,0)) #shape = Y x N' where N' = (V x Frame)
report = defaultdict(dict)
for gt_idx, gt_label in enumerate(gt_labels_batch):
AU = config.AU_SQUEEZE[gt_idx]
if AU in self.paper_use_AU:
pred_label = pred_labels_batch[gt_idx]
# met_E = get_F1_event(gt_label, pred_label)
F1 = f1_score(y_true=gt_label, y_pred=pred_label)
accuracy = accuracy_score(gt_label, pred_label)
met_F = get_F1_frame(gt_label, pred_label)
# roc = get_ROC(gt_label, pred_label)
report["f1_frame"][AU] = met_F.f1f
# report["AUC"][AU] = roc.auc
report["accuracy"][AU] = accuracy
summary.add({"f1_frame_avg": F1})
# summary.add({"AUC_avg": roc.auc})
summary.add({"accuracy_avg": accuracy})
observation = {}
with reporter.scope(observation):
reporter.report(report, _target)
reporter.report(summary.compute_mean(), _target)
print(observation)
return observation
def evaluate(self):
iterator = self._iterators['main']
target = self._targets['main']
it = copy.copy(iterator)
all_gt_label = []
all_pred_label = []
use_idx = sorted(
filter(lambda idx: config.AU_SQUEEZE[idx] in self.paper_use_AU, list(config.AU_SQUEEZE.keys())))
print(list(config.AU_SQUEEZE[idx] for idx in use_idx))
npz_pred = []
npz_gt = []
npz_pred_score = []
for idx, batch in enumerate(it):
batch = self.converter(batch, device=self.device)
imgs, bbox, labels = batch
imgs = chainer.Variable(imgs)
bbox = chainer.Variable(bbox)
if bbox.shape[1] != config.BOX_NUM[self.database]:
print("error box num {0} != {1}".format(bbox.shape[1], config.BOX_NUM[self.database]))
continue
preds, scores = target.predict(imgs, bbox) # R', class_num
try:
preds = preds.reshape(labels.shape[0], labels.shape[1], labels.shape[2]) # shape = B, F, Y
scores = scores.reshape(labels.shape[0], labels.shape[1], labels.shape[2]) # shape = B,F,Y
except ValueError:
print(preds.shape, labels.shape)
print(scores.shape, labels.shape)
continue
preds = chainer.cuda.to_cpu(preds) # B, F, Y, where B always = 1
labels = chainer.cuda.to_cpu(labels) # B, F, Y
scores = chainer.cuda.to_cpu(scores) # B, F, Y
npz_pred.extend(preds)
npz_gt.extend(labels)
npz_pred_score.extend(scores)
preds = np.bitwise_or.reduce(preds, axis=1) # shape = B, Y
gt_labels = np.bitwise_or.reduce(labels, axis=1) # shape = B, Y
all_gt_index = set()
pos_pred = np.nonzero(preds)
pos_gt_labels = np.nonzero(gt_labels)
all_gt_index.update(list(zip(*pos_pred)))
all_gt_index.update(list(zip(*pos_gt_labels)))
if len(all_gt_index) > 0:
accuracy = np.sum(preds[list(zip(*all_gt_index))[0], list(zip(*all_gt_index))[1]] == gt_labels[list(zip(*all_gt_index))[0], list(zip(*all_gt_index))[1]])/ len(all_gt_index)
print("batch idx:{0} current batch accuracy is :{1}".format(idx, accuracy))
all_gt_label.extend(gt_labels)
all_pred_label.extend(preds)
all_gt_label = np.asarray(all_gt_label) # shape = (N, len(AU_SQUEEZE))
all_pred_label = np.asarray(all_pred_label) # shape = (N, len(AU_SQUEEZE))
AU_gt_label = np.transpose(all_gt_label) # shape = (len(AU_SQUEEZE), N)
AU_pred_label = np.transpose(all_pred_label) # shape= (len(AU_SQUEEZE), N)
report = defaultdict(dict)
reporter = Reporter()
reporter.add_observer("main", target)
summary = DictSummary()
for AU_squeeze_idx, pred_label in enumerate(AU_pred_label):
AU = config.AU_SQUEEZE[AU_squeeze_idx]
if AU in self.paper_use_AU:
gt_label = AU_gt_label[AU_squeeze_idx]
# met_E = get_F1_event(gt_label, pred_label)
met_F = get_F1_frame(gt_label, pred_label)
roc = get_ROC(gt_label, pred_label)
f1 = f1_score(gt_label, pred_label)
# report["f1_frame"][AU] = met_F.f1f
report["f1_score"][AU] = f1
# assert f1 == met_F.f1f
report["AUC"][AU] = roc.auc
report["accuracy"][AU] = met_F.accuracy
# report["f1_event"][AU] = np.median(met_E.f1EventCurve)
summary.add({"f1_score_avg": f1})
summary.add({"AUC_avg": roc.auc})
summary.add({"accuracy_avg": met_F.accuracy})
# summary.add({"f1_event_avg": np.median(met_E.f1EventCurve)})
observation = {}
with reporter.scope(observation):
reporter.report(report, target)
reporter.report(summary.compute_mean(), target)
npz_gt = np.array(npz_gt) # N, F, 12
npz_pred = np.array(npz_pred) # N, F, 12
npz_pred_score = np.array(npz_pred_score)
np.savez(self.npz_out_path, gt=npz_gt, pred=npz_pred, pred_score=npz_pred_score)
return observation
def evaluate(self):
iterator = self._iterators['main']
_target = self._targets["main"]
if hasattr(iterator, 'reset'):
iterator.reset_for_train_mode()
it = iterator
else:
it = copy.copy(iterator)
reporter = Reporter()
reporter.add_observer("main", _target) # will fail to run?
summary = DictSummary()
trainer_result = defaultdict(list)
for batch in it:
batch = convert(batch, self.device)
for x, crf_pact_structure in zip(*batch):
sample = crf_pact_structure.sample
file_path = sample.file_path
print("evaluate file:{0}".format(file_path))
video_id = os.path.basename(file_path)
train_keyword = os.path.basename(os.path.dirname(file_path)) # train_keyword comes from file_path
if train_keyword not in self.target_dict:
print("error {} not pre-trained".format(train_keyword))
continue
target = self.target_dict[train_keyword] # choose the right predictor
# pred_probs is N x (Y'+1)
pred_labels = target.predict(x, crf_pact_structure, is_bin=False) # pred_labels is N x 1
gt_labels = target.get_gt_label_one_graph(np, crf_pact_structure, is_bin=False) # return N x 1
trainer_result[train_keyword].append((video_id, pred_labels, gt_labels))
report_dict = defaultdict(dict) # key = train_keyword
for train_keyword, pred_gt_list in trainer_result.items():
trainer_pred_labels = []
trainer_gt_labels = []
for video_id, pred_labels, gt_labels in sorted(pred_gt_list, key=lambda e:e[0]):
trainer_pred_labels.extend(pred_labels)
trainer_gt_labels.extend(gt_labels)
trainer_pred_labels = np.asarray(trainer_pred_labels,dtype=np.int32)
trainer_gt_labels = np.asarray(trainer_gt_labels,dtype=np.int32)
assert len(trainer_gt_labels) == len(trainer_pred_labels)
node_number = len(trainer_gt_labels)
gt_labels = np.zeros((node_number, len(config.AU_SQUEEZE)), dtype=np.int32) # frame x Y
pred_labels = np.zeros((node_number, len(config.AU_SQUEEZE)), dtype=np.int32)
for node_idx, gt_label in enumerate(trainer_gt_labels):
if gt_label > 0:
AU = train_keyword.split("_")[gt_label - 1]
AU_squeeze_idx = config.AU_SQUEEZE.inv[AU]
gt_labels[node_idx, AU_squeeze_idx] = 1
pred_label = trainer_pred_labels[node_idx]
if pred_label > 0:
AU = train_keyword.split("_")[pred_label - 1]
AU_squeeze_idx = config.AU_SQUEEZE.inv[AU]
pred_labels[node_idx, AU_squeeze_idx] = 1
gt_labels = gt_labels.reshape(-1, config.BOX_NUM[self.database], len(config.AU_SQUEEZE))
pred_labels = pred_labels.reshape(-1, config.BOX_NUM[self.database], len(config.AU_SQUEEZE))
gt_labels = np.bitwise_or.reduce(gt_labels, axis=1) # shape = frame x Y
pred_labels = np.bitwise_or.reduce(pred_labels, axis=1) # shape = frame x Y
gt_labels = np.transpose(gt_labels) # shape = Y x frame
pred_labels = np.transpose(pred_labels) #shape = Y x frame
for AU_idx, frame_pred in enumerate(pred_labels):
if config.AU_SQUEEZE[AU_idx] in self.paper_use_AU:
if config.AU_SQUEEZE[AU_idx] in train_keyword.split("_"):
AU = config.AU_SQUEEZE[AU_idx]
frame_gt = gt_labels[AU_idx]
F1 = f1_score(y_true=frame_gt, y_pred=frame_pred)
accuracy = accuracy_score(frame_gt, frame_pred)
report_dict[train_keyword][AU] = F1
merge_dict = {}
for train_keyword, AU_F1 in report_dict.items():
for AU, F1 in AU_F1.items():
if AU in self.paper_use_AU:
if AU not in merge_dict:
merge_dict[AU] = F1
elif F1 > merge_dict[AU]:
merge_dict[AU] = F1
report_dict["merge_result"] = merge_dict
observation = {}
with reporter.scope(observation):
reporter.report(report_dict, _target)
reporter.report(summary.compute_mean(), _target)
print(report_dict)
return observation
def evaluate(self):
iterator = self._iterators['main']
_target = self._targets["main"]
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
reporter = Reporter()
reporter.add_observer("main", _target)
summary = DictSummary()
model = _target
pred_labels_array = []
gt_labels_array = []
unreduce_pred = []
unreduce_gt = []
last_seq_id = None
one_frame_predict_list = []
one_frame_gt_list = []
for idx, batch in enumerate(it):
print("processing :{}".format(idx))
batch = self.converter(batch, self.device)
feature, gt_seg_rgb, gt_seg_flow, seg_info, seg_labels, gt_labels, npz_file_path = batch # feature shape =(B, C, W); bboxes shape = B,W 4; labels shape B, W, 12
seg_id = os.path.basename(npz_file_path[0])
seg_id = seg_id[:seg_id.rindex("#")]
if last_seq_id is None:
last_seq_id = seg_id
if last_seq_id != seg_id:
one_frame_predict_result = np.stack(one_frame_predict_list, axis=2) # B, W, F, class
unreduce_pred.extend(one_frame_predict_result.reshape(-1,
one_frame_predict_result.shape[-2],one_frame_predict_result.shape[-1])) # list of W, F, class
one_frame_predict_result = np.bitwise_or.reduce(one_frame_predict_result, axis=2) # B, W, class
one_frame_predict_result = one_frame_predict_result.reshape([-1, one_frame_predict_result.shape[-1]]) # B* W, class
pred_labels_array.extend(one_frame_predict_result)
one_frame_gt_result = np.stack(one_frame_gt_list, axis=2) # B, W, F, class
unreduce_gt.extend(one_frame_gt_result.reshape(-1, one_frame_gt_result.shape[-2], one_frame_gt_result.shape[-1])) # list of W, F, class
one_frame_gt_result = np.bitwise_or.reduce(one_frame_gt_result, axis=2) # B, W, class
one_frame_gt_result = one_frame_gt_result.reshape([-1, one_frame_gt_result.shape[-1]]) # B * W, class
gt_labels_array.extend(one_frame_gt_result)
one_frame_predict_list.clear()
one_frame_gt_list.clear()
if not isinstance(feature, chainer.Variable):
feature = chainer.Variable(feature.astype('f'))
# feature = (B, C, W)
predict_labels = model.predict(feature) # (B, W, class)
one_frame_predict_list.append(predict_labels)
gt_labels = chainer.cuda.to_cpu(gt_labels)
one_frame_gt_list.append(gt_labels)
one_frame_predict_result = np.stack(one_frame_predict_list, axis=2) # B, W, F, class
unreduce_pred.extend(one_frame_predict_result.reshape(-1,
one_frame_predict_result.shape[-2],one_frame_predict_result.shape[-1])) # list of W, F, class
one_frame_predict_result = np.bitwise_or.reduce(one_frame_predict_result, axis=2) # B, W, class
assert one_frame_predict_result.shape[-1] == self.class_num
one_frame_predict_result = one_frame_predict_result.reshape(
[-1, one_frame_predict_result.shape[-1]]) # B* W, class
pred_labels_array.extend(one_frame_predict_result)
one_frame_gt_result = np.stack(one_frame_gt_list, axis=2) # B, W, F, class
unreduce_gt.extend(one_frame_gt_result.reshape(-1, one_frame_gt_result.shape[-2], one_frame_gt_result.shape[-1])) # list of W, F, class
one_frame_gt_result = np.bitwise_or.reduce(one_frame_gt_result, axis=2) # B, W, class
one_frame_gt_result = one_frame_gt_result.reshape([-1, one_frame_gt_result.shape[-1]]) # B * W, class
gt_labels_array.extend(one_frame_gt_result)
one_frame_predict_list.clear()
one_frame_gt_list.clear()
# 由于F不一样，因此不能stack
# unreduce_pred = np.stack(unreduce_pred).astype(np.int32) # N, W, F, class
# unreduce_gt = np.stack(unreduce_gt).astype(np.int32) # N, W, F, class
# np.savez(self.output_path , predict=unreduce_pred, gt=unreduce_gt)
gt_labels_array = np.stack(gt_labels_array) # all_N, 12
pred_labels_array = np.stack(pred_labels_array) # shape = all_N, out_size
gt_labels = np.transpose(gt_labels_array) # shape = Y x frame
pred_labels = np.transpose(pred_labels_array) #shape = Y x frame
report_dict = dict()
AU_id_convert_dict = self.AU_convert if self.AU_convert else config.AU_SQUEEZE
for new_AU_idx, frame_pred in enumerate(pred_labels):
if AU_id_convert_dict[new_AU_idx] in self.paper_use_AU:
AU = AU_id_convert_dict[new_AU_idx]
frame_gt = gt_labels[new_AU_idx]
F1 = f1_score(y_true=frame_gt, y_pred=frame_pred)
report_dict[AU] = F1
summary.add({"f1_frame_avg": F1})
observation = {}
with reporter.scope(observation):
reporter.report(report_dict, model)
reporter.report(summary.compute_mean(), model)
return observation
def evaluate(self):
iterator = self._iterators['main']
_target = self._targets["main"]
if hasattr(iterator, 'reset'):
iterator.reset_for_train_mode()
it = iterator
else:
it = copy.copy(iterator)
reporter = Reporter()
reporter.add_observer("main", _target)
summary = DictSummary()
model = _target
pred_labels_array = []
gt_labels_array = []
for batch in it:
batch = convert(batch, self.device)
for x, g, crf_pact_structure in zip(*batch):
sample = crf_pact_structure.sample
file_path = sample.file_path
print("evaluate file:{0}".format(file_path))
pred_labels = model.predict(
x, g, crf_pact_structure) # pred_labels is N x Y
gt_labels = model.get_gt_label_one_graph(
np, crf_pact_structure, is_bin=True) # return N x Y
pred_labels_array.extend(pred_labels)
gt_labels_array.extend(gt_labels)
gt_labels_array = np.stack(gt_labels_array)
pred_labels_array = np.stack(
pred_labels_array) # shape = all_N, out_size
if self.AU_convert:
gt_labels_array = gt_labels_array.reshape(
-1, config.BOX_NUM[self.database],
len(self.paper_report_label)) # T, F, out_size
pred_labels_array = pred_labels_array.reshape(
-1, config.BOX_NUM[self.database],
len(self.paper_report_label)) # T, F, out_size
else:
gt_labels_array = gt_labels_array.reshape(
-1, config.BOX_NUM[self.database],
len(config.AU_SQUEEZE)) # T, F, out_size
pred_labels_array = pred_labels_array.reshape(
-1, config.BOX_NUM[self.database], len(config.AU_SQUEEZE))
gt_labels = np.bitwise_or.reduce(gt_labels_array,
axis=1) # shape = all_frame x Y
pred_labels = np.bitwise_or.reduce(pred_labels_array,
axis=1) # shape = all_frame x Y
gt_labels = np.transpose(gt_labels) # shape = Y x frame
pred_labels = np.transpose(pred_labels) #shape = Y x frame
report_dict = dict()
AU_id_convert_dict = self.AU_convert if self.AU_convert else config.AU_SQUEEZE
for new_AU_idx, frame_pred in enumerate(pred_labels):
if AU_id_convert_dict[new_AU_idx] in self.paper_use_AU:
AU = AU_id_convert_dict[new_AU_idx]
frame_gt = gt_labels[new_AU_idx]
F1 = f1_score(y_true=frame_gt, y_pred=frame_pred)
report_dict[AU] = F1
summary.add({"f1_frame_avg": F1})
observation = {}
with reporter.scope(observation):
reporter.report(report_dict, model)
reporter.report(summary.compute_mean(), model)
return observation
def evaluate(self):
iterator = self._iterators['main']
_target = self._targets["main"]
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
reporter = Reporter()
reporter.add_observer("main", _target)
summary = DictSummary()
model = _target
pred_labels_array = []
gt_labels_array = []
unreduce_pred = []
unreduce_gt = []
for idx, batch in enumerate(it):
print("processing :{}".format(idx))
batch = self.converter(batch, self.device)
rgb_images, flow_images, bboxes, labels = batch # images shape = B*T, C, H, W; bboxes shape = B*T, F, 4; labels shape = B*T, F, 12
if not isinstance(rgb_images, chainer.Variable):
rgb_images = chainer.Variable(rgb_images.astype('f'))
bboxes = chainer.Variable(bboxes.astype('f'))
flow_images = chainer.Variable(flow_images.astype('f'))
roi_feature = model.get_roi_feature(rgb_images, flow_images, bboxes, extract_rgb_flow=False)
pred_labels = model.predict(roi_feature) # B, F, 12
unreduce_pred.extend(pred_labels) # list of F,D
pred_labels = np.bitwise_or.reduce(pred_labels, axis=1) # B, class_number
unreduce_gt.extend(labels) # shape = list of F,D
labels = np.bitwise_or.reduce(labels, axis=1) # B, class_number
assert labels.shape == pred_labels.shape
pred_labels_array.extend(pred_labels)
gt_labels_array.extend(labels)
unreduce_pred = np.stack(unreduce_pred).astype(np.int32)
unreduce_gt = np.stack(unreduce_gt).astype(np.int32)
np.savez(self.output_path , pred=unreduce_pred, gt=unreduce_gt)
gt_labels_array = np.stack(gt_labels_array)
pred_labels_array = np.stack(pred_labels_array) # shape = all_N, out_size
gt_labels = np.transpose(gt_labels_array) # shape = Y x frame
pred_labels = np.transpose(pred_labels_array) #shape = Y x frame
report_dict = dict()
AU_id_convert_dict = self.AU_convert if self.AU_convert else config.AU_SQUEEZE
for new_AU_idx, frame_pred in enumerate(pred_labels):
if AU_id_convert_dict[new_AU_idx] in self.paper_use_AU:
AU = AU_id_convert_dict[new_AU_idx]
frame_gt = gt_labels[new_AU_idx]
F1 = f1_score(y_true=frame_gt, y_pred=frame_pred)
report_dict[AU] = F1
summary.add({"f1_frame_avg": F1})
observation = {}
with reporter.scope(observation):
reporter.report(report_dict, model)
reporter.report(summary.compute_mean(), model)
return observation