def write_csvs(out_dir, exp_name, label_names, labels, predict):
# frame, behavior, behavior ground truth, image
labels = labels.reshape((labels.shape[0], 1, labels.shape[1]))
predict = predict.reshape((predict.shape[0], 1, predict.shape[1]))
frames = [list(range(labels.shape[0]))]
temp = [label.decode() for label in label_names]
sequences_helper.write_predictions2(out_dir, [exp_name],
predict, [labels],
None,
frames,
label_names=temp)
# for ease of use, create a csv file with all the labels.
temp.append("background")
out_filename = os.path.join(out_dir, exp_name.decode(), "all.csv")
with open(out_filename, "w") as fid:
fid.write("%s" % temp[0])
for i in range(1, len(temp)):
fid.write(",%s" % temp[i])
fid.write("\n")
num_rows = min(predict.shape[0], labels.shape[0])
for i in range(num_rows):
fid.write("%f" % predict[i, 0, 0])
for j in range(1, len(temp)):
fid.write(",%f" % predict[i, 0, j])
fid.write("\n")
def write_csvs(out_dir, exp_name, label_names, labels, predict):
# frame, behavior, behavior ground truth, image
labels = labels.reshape((labels.shape[0], 1, labels.shape[1]))
predict = predict.reshape((predict.shape[0], 1, predict.shape[1]))
frames = [list(range(labels.shape[0]))]
temp = [
label.decode() for label in label_names
]
sequences_helper.write_predictions2(
out_dir, [exp_name], predict, [labels], None, frames,
label_names=temp)
def _eval_video(opts, network, inputs, criterion, name, out_dir):
"""Evaluate one video."""
num_frames = inputs[0].size()[0]
chunk_len = 50
num_chunks = int(np.ceil(1.0 * num_frames / chunk_len))
predict = np.zeros((num_frames, 1, 6))
total_loss = 0
# loop over the number of chunks
for j in range(0, num_chunks):
idx1 = j * chunk_len
idx2 = min((j + 1) * chunk_len, num_frames)
# print("\t%d" % idx2)
chunk = [
inputs[0][idx1:idx2, :1, :, :].cuda(),
inputs[1][idx1:idx2, :1, :, :].cuda(),
inputs[2][idx1:idx2, :].cuda()
]
out, _ = network([chunk[0], chunk[1]])
out = torch.exp(out.data)
predict[idx1:idx2, 0, :] = out[:, :6].data.cpu().numpy()
label_idx = _create_labels(opts, chunk[-1])
loss = criterion(out, label_idx)
total_loss += loss.item()
# finished predicting the video.
exp_names = [inputs[3]]
labels = inputs[2].cpu().numpy()
labels = [labels.reshape((labels.shape[0], 1, labels.shape[1]))]
frames = [range(labels[0].shape[0])]
# write the predictions to disk
sequences_helper.write_predictions2(out_dir, exp_names[0], predict, labels,
None, frames)
# process the video
return total_loss
def eval_network(opts, step, network, label_weight, sampler, criterion, name):
"""Evaluate the state of the network."""
out_dir = os.path.join(opts["flags"].out_dir, "predictions", name)
total_loss = 0
for i in range(sampler.num_batch):
inputs = sampler.get_minibatch()
num_frames = inputs[0].size()[0]
chunk_len = 50
num_chunks = int(np.ceil(1.0 * num_frames / chunk_len))
predict = np.zeros((num_frames, 1, 6))
# print(num_frames)
for j in range(0, num_chunks):
idx1 = j * chunk_len
idx2 = min((j + 1) * chunk_len, num_frames)
# print("\t%d" % idx2)
chunk = [
Variable(inputs[0][idx1:idx2, :, :, :],
requires_grad=True).cuda(),
Variable(inputs[1][idx1:idx2, :, :, :],
requires_grad=True).cuda(),
Variable(inputs[2][idx1:idx2, :], requires_grad=False).cuda()
]
out = network([chunk[0], chunk[1]])
predict[idx1:idx2, 0, :] = out.data.cpu().numpy()
loss = criterion(out, chunk[-1])
total_loss += loss.data[0]
exp_names = [inputs[3]]
labels = inputs[2].cpu().numpy()
labels = [labels.reshape((labels.shape[0], 1, labels.shape[1]))]
frames = [range(labels[0].shape[0])]
sequences_helper.write_predictions2(out_dir, exp_names, predict,
labels, None, frames)
print("\teval loss: %f" % (total_loss / sampler.num_batch))
return (total_loss / sampler.num_batch)
def _process_seqs(opts, step, network, sampler, criterion, name):
"""Evaluate the state of the network."""
out_dir = os.path.join(opts["flags"].out_dir, "predictions", name)
total_loss = 0
if sampler.batch_idx.empty():
sampler.reset()
tic = time.time()
num_behav = len(g_label_names)
# init dict
all_matches = []
for i in range(len(g_label_names)):
all_matches.append({
"tps": 0,
"fps": 0,
"fns": 0,
"labels": g_label_names[i]
})
for i in range(sampler.num_batch):
inputs = sampler.get_minibatch()
# num_frames = inputs[0].size()[0]
# ; pdb.set_trace()
num_frames = np.min([opts["flags"].seq_len, inputs[0].size()[0]])
# chunk_len = 4
# print(inputs[3][0].decode("utf-8"))
# print("\t%d" % num_frames)
# num_chunks = int(np.ceil(1.0 * num_frames / chunk_len))
predict = np.zeros((num_frames, 1, num_behav))
# print(num_frames)
for j in range(0, num_frames):
feat1, feat2, labels = _create_chunks(opts, inputs, j)
chunk = [feat1, feat2, labels]
if opts["flags"].cuda_device != -1:
chunk = [feat1.cuda(), feat2.cuda(), labels.cuda()]
# idx1 = j * chunk_len
# idx2 = min((j + 1) * chunk_len, num_frames)
# # print("\t%d" % idx2)
# chunk = [
# Variable(inputs[0][idx1:idx2, :1, :, :], requires_grad=True).cuda(),
# Variable(inputs[1][idx1:idx2, :1, :, :], requires_grad=True).cuda(),
# Variable(inputs[2][idx1:idx2, :], requires_grad=False).cuda()
# ]
out, features = network([chunk[0], chunk[1]])
# import pdb; pdb.set_trace()
out = torch.exp(out.data)
predict[j, 0, :] = out[:, :num_behav].data.cpu().numpy()
# label_idx = _create_labels(opts, chunk[-1])
loss = criterion(out, chunk[-1])
total_loss += loss.item()
exp_names = [inputs[3][0]]
labels = inputs[2].cpu().numpy()
labels = [labels.reshape((labels.shape[0], 1, labels.shape[1]))]
frames = [range(num_frames)]
sequences_helper.write_predictions2(out_dir,
exp_names,
predict,
labels,
None,
frames,
label_names=g_label_names)
# process ouputs
matches = analyze_outs(out_dir, exp_names, predict, labels)
# merge the values
all_matches = _merge_dicts(all_matches, matches)
# # add label names
# for j in range(len(g_label_names)):
# all_matches[j]["label"] = g_label_names[j]
# if len(all_matches[0]["tps"]) > 0:
# import pdb; pdb.set_trace()
print("\t%f" % (time.time() - tic))
print("\teval loss: %f" % (total_loss / sampler.num_batch))
return (total_loss / sampler.num_batch), all_matches
def _predict_write(opts, step, out_dir, network, h5_data, exps, label_weight):
"""Predict and write sequence classifications."""
batch_id = 0
exps.sort()
loss = 0
scores = [0, 0, 0, 0]
batch_count = 0
# t = network["lr_update"]["params"][0]
t = step
while batch_id != -1:
if batch_count % 10 == 0:
print("\t\t%d" % batch_count)
# inputs, org_labels, sample_idx, batch_id = _get_seq_mini_batch(
# opts, batch_id, h5_data, exps)
inputs, labels, mask, org_labels, sample_idx, batch_id =\
_get_seq_mini_batch(opts, batch_id, h5_data, exps)
hidden = _get_hidden(opts)
# img_side = torch.autograd.Variable(torch.Tensor(inputs[0])).cuda()
# img_front = torch.autograd.Variable(torch.Tensor(inputs[1])).cuda()
inputs = [
torch.autograd.Variable(torch.Tensor(feats)).cuda()
for feats in inputs
]
labels = torch.autograd.Variable(torch.Tensor(labels)).cuda()
mask = torch.autograd.Variable(torch.Tensor(mask)).cuda()
predict, update_hid = network(inputs, hidden)
TP_weight, FP_weight, false_neg, false_pos = create_match_array(
opts, predict, org_labels, label_weight[2])
pos_mask, neg_mask = hantman_hungarian.create_pos_neg_masks(labels, label_weight[0], label_weight[1])
perframe_cost = hantman_hungarian.perframe_loss(predict, mask, labels, pos_mask, neg_mask)
tp_cost, fp_cost, fn_cost = hantman_hungarian.structured_loss(
predict, mask, TP_weight, FP_weight, false_neg)
total_cost, struct_cost, perframe_cost, tp_cost, fp_cost, fn_cost =\
hantman_hungarian.combine_losses(opts, step, perframe_cost, tp_cost, fp_cost, fn_cost)
cost = total_cost.mean()
loss += cost.data[0]
# order from past:
# total cost, struct_cost, tp, fp, fn, perframe
scores[0] += tp_cost.data.cpu()[0]
scores[1] += fp_cost.data.cpu()[0]
scores[2] += fn_cost.data.cpu()[0]
scores[3] += perframe_cost.data.cpu()[0]
# scores = [scores[i] + cost[i + 3] for i in range(len(cost[3:]))]
predictions = predict.data.cpu().numpy()
# collect the labels
labels = []
frames = []
for vid in exps[sample_idx]:
labels.append(h5_data["exps"][vid]["labels"].value)
frames.append(list(range(h5_data["exps"][vid]["labels"].shape[0])))
# idx = idx[:valid_idx]
# print feat_idx
# print idx
# print inputs
# import pdb; pdb.set_trace()
sequences_helper.write_predictions2(
out_dir, exps[sample_idx], predictions, labels,
[], frames)
batch_count = batch_count + 1
loss = loss / batch_count
scores = [score / batch_count for score in scores]
return loss, scores
def _process_seqs(opts, step, network, sampler, criterion, name):
"""Evaluate the state of the network."""
out_dir = os.path.join(opts["flags"].out_dir, "predictions", name)
total_loss = 0
if sampler.batch_idx.empty():
sampler.reset()
# init dict
all_matches = []
for i in range(len(g_label_names)):
all_matches.append({
"tps": 0,
"fps": 0,
"fns": 0,
"labels": g_label_names[i]
})
for i in range(sampler.num_batch):
inputs = sampler.get_minibatch()
num_frames = inputs[0].size()[0]
chunk_len = 50
num_chunks = int(np.ceil(1.0 * num_frames / chunk_len))
predict = np.zeros((num_frames, 1, 6))
all_features = np.zeros((num_frames, 1024))
# print(num_frames)
for j in range(0, num_chunks):
idx1 = j * chunk_len
idx2 = min((j + 1) * chunk_len, num_frames)
# print("\t%d" % idx2)
chunk = [
Variable(inputs[0][idx1:idx2, :1, :, :],
requires_grad=False).cuda(),
Variable(inputs[1][idx1:idx2, :1, :, :],
requires_grad=False).cuda(),
Variable(inputs[2][idx1:idx2, :], requires_grad=False).cuda()
]
out, features = network([chunk[0], chunk[1]])
out = torch.exp(out.data)
predict[idx1:idx2, 0, :] = out[:, :6].data.cpu().numpy()
label_idx = _create_labels(opts, chunk[-1])
all_features[idx1:idx2, :] = features.data.cpu().numpy()
loss = criterion(out, label_idx)
total_loss += loss.item()
exp_names = [inputs[3]]
labels = inputs[2].cpu().numpy()
labels = [labels.reshape((labels.shape[0], 1, labels.shape[1]))]
frames = [range(labels[0].shape[0])]
sequences_helper.write_predictions2(out_dir, exp_names[0], predict,
labels, None, frames)
# create the hdf5 file.
curr_exp = exp_names[0][0]
out_file = os.path.join(opts["flags"].feature_dir, curr_exp)
with h5py.File(out_file, "w") as feature_fid:
feature_fid["2dconv"] = all_features
# if "2dconv" in sampler.data["exps"][curr_exp]:
# # import pdb; pdb.set_trace()
# del sampler.data["exps"][curr_exp]["2dconv"]
# # next create an external link
# sampler.data["exps"][curr_exp]["2dconv"] = h5py.ExternalLink(
# os.path.join(os.path.basename(opts["flags"].feature_dir), curr_exp),
# "/2dconv"
# )
# process ouputs
matches = analyze_outs(out_dir, exp_names[0], predict, labels)
# merge the values
all_matches = _merge_dicts(all_matches, matches)
# # add label names
# for j in range(len(g_label_names)):
# all_matches[j]["label"] = g_label_names[j]
# if len(all_matches[0]["tps"]) > 0:
# import pdb; pdb.set_trace()
print("\teval loss: %f" % (total_loss / sampler.num_batch))
return (total_loss / sampler.num_batch), all_matches
