def yield_frames(self, index, every_n=10):
gif, cls_id = self.file_and_clsid[index]
for fn, bgr in yield_frames(gif):
if fn % every_n:
rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
yield fn, rgb
def yield_predictions(m, video_file, show=False):
input_shape = (299, 299, 3)
input_image = Input((299, 299, 3))
input_flow = Input((299, 299, 20))
x_flow = numpy.zeros((299, 299, 20))
old_gray = None
flow_fn = 0
# video_file = "/blender/storage/datasets/vg_smoke/valid/basic_inst/basic-cropped.mp4"
# video_file = "/Volumes/SD128/getty_1525701638930.mp4"
# video_file = "/Volumes/bstorage/datasets/vg_smoke/smoking_videos/mp4/American_History_X_smoke_h_nm_np1_fr_goo_26.avi.mp4"
# video_file = "/Volumes/bstorage/datasets/vg_smoke/smoking_videos/mp4/smoking_again_smoke_h_cm_np1_le_goo_1.avi.mp4"
# video_file = "/Volumes/bstorage/datasets/vg_smoke/smoking_videos/mp4/The_Matrix_4_smoke_h_nm_np1_fr_goo_1.avi.mp4"
for fn, bgr in yield_frames(video_file):
resized_bgr = cv2.resize(bgr, (299, 299))
gray = cv2.cvtColor(resized_bgr, cv2.COLOR_RGB2GRAY)
rgb = cv2.cvtColor(resized_bgr, cv2.COLOR_BGR2RGB)
x_rgb = rgb / 127.5 - 1
if old_gray is None:
old_gray = gray
if fn % 2:
if flow_fn > 19:
flow_fn = flow_fn - 2
x_flow_new = numpy.zeros_like(x_flow)
x_flow_new[:, :, :-2] = x_flow[:, :, 2:] # shift oflow left by two positions
x_flow = x_flow_new
cur_flow = calc_flow(gray, old_gray)
x_flow[:, :, flow_fn] = cur_flow[:, :, 0]
x_flow[:, :, flow_fn + 1] = cur_flow[:, :, 1]
old_gray = gray
flow_fn = flow_fn + 2
# xx = [x_rgb, x_flow]
xx = [numpy.array([x_rgb]), numpy.array([x_flow])]
y_batch = m.predict(xx, batch_size=1)
for y in y_batch:
yield (fn, y)
if show:
cls_id = numpy.argmax(y)
print("fn=%d; flow_frame=%d cls_id=%d y=%s" % (fn, flow_fn, cls_id, y.round(2)))
cv2.imshow("%d" % cls_id, resized_bgr)
hsv = numpy.zeros_like(resized_bgr)
mag = x_flow[:, :, 18]
ang = x_flow[:, :, 19]
hsv = SmokeGifSequence.flow_to_hsv(dst_hsv=hsv, mag_ang=(mag, ang))
flow_mask = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
cv2.imshow("mask", flow_mask)
cv2.waitKey(25)
def rgb_and_flows(self,
video_file: str,
flows_count: int = 10,
drop_every_n_frame: int = 2,
drop_first_n_frames: int = -1):
old_gray = None
rgb = None
crop_x1 = randint(0, 32)
crop_x2 = randint(0, 32)
crop_y1 = randint(0, 32)
crop_y2 = randint(0, 32)
if drop_first_n_frames < 0:
cap = cv2.VideoCapture(video_file)
frame_count = cap.get(cv2.CAP_PROP_FRAME_COUNT)
if frame_count > 10 * drop_every_n_frame:
drop_first_n_frames = randint(
0, frame_count - 10 * drop_every_n_frame - 1)
else:
drop_first_n_frames = 0
flows_mag_ang = np.zeros(shape=(self.input_hwc[0], self.input_hwc[1],
flows_count * 2))
for fn, bgr in yield_frames(video_file,
start_frame=drop_first_n_frames):
h, w, c = bgr.shape
bgr = bgr[crop_y1:h - crop_y2, crop_x1:w - crop_x2]
bgr = cv2.resize(bgr, dsize=(self.input_hwc[1], self.input_hwc[0]))
gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
if old_gray is None:
old_gray = gray
if fn % drop_every_n_frame != 0:
continue
end_frame = drop_every_n_frame * flows_count + 1
if fn < end_frame:
flow_frame = 2 * int(fn / drop_every_n_frame) - 2
curr_flow = calc_flow(gray, old_gray)
flows_mag_ang[:, :, flow_frame] = curr_flow[:, :, 0]
flows_mag_ang[:, :, flow_frame + 1] = curr_flow[:, :, 1]
old_gray = gray
else:
break
assert rgb is not None, "zero frame for %s; drop_first_n_frames=%d" % (
video_file, drop_first_n_frames)
return rgb, flows_mag_ang
def main():
model = load_model('model.h5')
video_file = 'basic.mp4'
# drop_every_n_frame = 2
h, w = model.get_input_shape_at(0)[1:3]
flows_count = model.get_input_shape_at(1)[-1]
frames = [bgr for fn, bgr in yield_frames(video_file)]
frame_count = len(frames)
resizes = [cv2.resize(bgr, dsize=(w, h)) for bgr in frames]
grays = [cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY) for bgr in resizes]
rgbs = [cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB) for bgr in resizes]
flows = []
mags = []
angs = []
for i, gray in enumerate(grays[1:]):
flow = cv2.calcOpticalFlowFarneback(grays[i - 1], gray, None, 0.5, 3, 15, 3, 5, 1.2, 0)
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
flows.append(flow)
mags.append(mag)
angs.append(ang)
batch_size = 1
for i in range(0, frame_count-flows_count, 5):
xrgb_batch = rgbs[i + flows_count]
xflow_batch = np.zeros(shape=(batch_size, h, w, flows_count * 2))
batch_no = 0
for j in range(flows_count):
xflow_batch[batch_no, , :, 2 * j] = mags[i + j]
xflow_batch[batch_no, :, 2 * j + 1] = angs[i + j]
predictions = model.predict([xrgb_batch, xflow_batch])
def main():
truth_f = "/Volumes/bstorage/home/chexov/smoking/basic_inst/basic-instinct_truth.csv"
truth_by_frame = truth_from_csv(truth_f)
jl = "/Volumes/bstorage/home/chexov/cls/classifiers/basic-cropped.mp4.jsonl/result.jsonl"
jl = "/Volumes/bstorage/home/chexov/cls/classifiers/500_500_basic-cropped.mp4.jsonl/result.jsonl"
with open(jl, "r") as _f:
jsons = map(lambda s: s.strip(), _f.readlines())
results_by_frame = list(map(lambda j: json.loads(j), jsons))
url = "/blender/storage/home/chexov/smoking/basic_inst/basic-cropped.mp4"
predictions = []
truth = []
for fn, bgr in yield_frames(url):
_fn, y = results_by_frame[fn]
true_y = truth_by_frame[fn]
assert _fn == fn
predictions.append(y)
truth.append(true_y)
# print(y, true_y)
i = numpy.argmax(numpy.array(y))
true_i = numpy.argmax(numpy.array(true_y))
show = True
if show:
if y[i] > 0.9:
cv2.imshow('i=%d;true=%d' % (i, true_i), bgr)
code = cv2.waitKey(50)
if code == 27:
sys.exit(1)
predictions = numpy.array(predictions)
truth = numpy.array(truth)
# predictions = numpy.array(predictions > 0.9).astype(int)
# print(truth.shape)
# print(predictions.shape)
# precision, recall, fbeta_score, support = precision_recall_fscore_support(y_true=truth, y_pred=predictions,
# average='micro')
# print(precision, recall)
p1 = list(map(lambda p: p[0], predictions))
t1 = list(map(lambda p: p[0], truth))
p1 = numpy.array(p1)
t1 = numpy.array(t1)
average_precision = average_precision_score(t1, p1)
precision, recall, thresholds = precision_recall_curve(t1, p1)
print("thresholds=", thresholds)
plt.subplot(211)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(
average_precision))
p1 = list(map(lambda p: p[1], predictions))
t1 = list(map(lambda p: p[1], truth))
p1 = numpy.array(p1)
t1 = numpy.array(t1)
average_precision = average_precision_score(t1, p1)
precision, recall, _ = precision_recall_curve(t1, p1)
plt.subplot(212)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(
average_precision))
plt.show()
import sys
import cv2
import numpy
from utils import yield_frames, truth_from_csv
if __name__ == '__main__':
in_csv = "/blender/storage/datasets/vg_smoke/valid/basic_inst/basic-instinct_truth.csv"
by_frame = truth_from_csv(truth_csv=in_csv, num_classes=2)
for fn, bgr in yield_frames(
"/blender/storage/datasets/vg_smoke/valid/basic_inst/basic-cropped.mp4"
):
bgr = cv2.resize(bgr, dsize=(640, 480))
cls_id = numpy.argmax(by_frame[fn])
cv2.imshow("frame %d" % cls_id, bgr)
code = cv2.waitKey(25)
if code == 27:
sys.exit(1)