def __init__(self, sAlgorithm:str = "tvl1-fast", bThirdChannel:bool = False, fBound:float = 20.):
self.bThirdChannel = bThirdChannel
self.fBound = fBound
self.arPrev = np.zeros((1,1))
if sAlgorithm == "tvl1-fast":
self.oTVL1 = cv2.DualTVL1OpticalFlow_create(
scaleStep = 0.5, warps = 3, epsilon = 0.02)
# Mo 25.6.2018: (theta = 0.1, nscales = 1, scaleStep = 0.3, warps = 4, epsilon = 0.02)
# Very Fast (theta = 0.1, nscales = 1, scaleStep = 0.5, warps = 1, epsilon = 0.1)
sAlgorithm = "tvl1"
elif sAlgorithm == "tvl1-warps1":
self.oTVL1 = cv2.DualTVL1OpticalFlow_create(warps = 1)
sAlgorithm = "tvl1"
elif sAlgorithm == "tvl1-quality":
self.oTVL1 = cv2.DualTVL1OpticalFlow_create()
# Default: (tau=0.25, lambda=0.15, theta=0.3, nscales=5, warps=5, epsilon=0.01,
#innnerIterations=30, outerIterations=10, scaleStep=0.8, gamma=0.0,
#medianFiltering=5, useInitialFlow=False)
sAlgorithm = "tvl1"
elif sAlgorithm == "farnback":
pass
else: raise ValueError("Unknown optical flow type")
self.sAlgorithm = sAlgorithm
return
def __init__(self,
sAlgorithm="tvl1-fast",
bThirdChannel=False,
fBound=20.):
"""
Initializes the OpticalFlow object
:param sAlgorithm: Type of algorithm to use for calculating Optical Flow.
:param bThirdChannel: Whether to use third channel. (Third channel is required when viewing optical flow)
:param fBound: Upper bound for normalization.
"""
self.bThirdChannel = bThirdChannel
self.fBound = fBound
self.arPrev = np.zeros((1, 1))
if sAlgorithm == "tvl1-fast":
self.oTVL1 = cv2.DualTVL1OpticalFlow_create(scaleStep=0.5,
warps=3,
epsilon=0.02)
self.sAlgorithm = "tvl1"
elif sAlgorithm == "tvl1-quality":
self.oTVL1 = cv2.DualTVL1OpticalFlow_create()
# Default: (tau=0.25, lambda=0.15, theta=0.3, nscales=5, warps=5, epsilon=0.01, scaleStep=0.5)
self.sAlgorithm = "tvl1"
return
def extract_flow(images_name):
# Extract TVL1 flow and encode it as jpg
num_frames = len(images_name)
optical_flow = cv2.DualTVL1OpticalFlow_create()
flow_x_list = []
flow_y_list = []
for i_frames in range(num_frames):
prev_frame = cv2.imread(images_name[max(0, i_frames - 1)])
prev_frame = cv2.cvtColor(prev_frame, cv2.COLOR_BGR2GRAY)
cur_frame = cv2.imread(images_name[i_frames])
cur_frame = cv2.cvtColor(cur_frame, cv2.COLOR_BGR2GRAY)
# Compute TVL1-flow
prev_frame_gpu = cv2.UMat(prev_frame)
cur_frame_gpu = cv2.UMat(cur_frame)
flow_gpu = optical_flow.calc(prev_frame_gpu, cur_frame_gpu, None)
flow = flow_gpu.get()
# truncate [-20, 20]
flow[flow >= 20] = 20
flow[flow <= -20] = -20
# scale to [0, 255]
flow = flow + 20
flow = flow / 40 # normalize the data to 0 - 1
flow = 255 * flow # Now scale by 255
flow = flow.astype(np.uint8)
# Encode to jpg
flow_x_encode = cv2.imencode('.jpg', flow[:, :, 0])[1]
flow_y_encode = cv2.imencode('.jpg', flow[:, :, 1])[1]
flow_x_encode = flow_x_encode.tostring()
flow_y_encode = flow_y_encode.tostring()
flow_x_list.append(flow_x_encode)
flow_y_list.append(flow_y_encode)
return flow_x_list, flow_y_list
def extract_opticalflow(threadName, list_videos):
"""
Launch a thread that extracts optical flow
:param threadName: The name of the thread
:param list_videos: The list of videos the thread treats
:return: Creates the optical flow for each video
"""
for video_name in list_videos:
cap = cv2.VideoCapture(crowd11_folder + video_name)
ret, frame1 = cap.read()
previous_frame = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
optical_flow = cv2.DualTVL1OpticalFlow_create()
flows = list()
cap_bool = True
while (cap_bool):
ret, frame2 = cap.read()
if ret == False:
cap_bool = False
continue
print(threadName, ret, frame2.shape)
next_frame = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
flow = optical_flow.calc(previous_frame, next_frame, None)
flows.append(flow)
previous_frame = next_frame
flow_filename = re.findall("(.*?)\.[ma][pv][4i]", video_name)[0]
video_flows = np.asarray(flows)
np.save(crowd11_of_folder + flow_filename, video_flows)
cap.release()
def save_flows((in_dir_meta, out_dir_u, out_dir_v, video_name, idx)):
print idx
in_dir_curr = os.path.join(in_dir_meta, video_name)
in_files = glob.glob(os.path.join(in_dir_curr, '*.jpg'))
in_files.sort()
optical_flow = cv2.DualTVL1OpticalFlow_create()
out_dir_u_curr = os.path.join(out_dir_u, video_name)
out_dir_v_curr = os.path.join(out_dir_v, video_name)
util.mkdir(out_dir_u_curr)
util.mkdir(out_dir_v_curr)
for idx_in_file, in_file in enumerate(in_files[:-1]):
im1 = cv2.imread(in_file, cv2.IMREAD_GRAYSCALE)
im2 = cv2.imread(in_files[idx_in_file + 1], cv2.IMREAD_GRAYSCALE)
flow = optical_flow.calc(im1, im2, None)
flow = np.clip(flow, -20, 20)
flow = (flow + 20) / 40 * 255
u = flow[:, :, 0]
v = flow[:, :, 1]
out_file_u = os.path.join(out_dir_u_curr, os.path.split(in_file)[1])
out_file_v = os.path.join(out_dir_v_curr, os.path.split(in_file)[1])
cv2.imwrite(out_file_u, u)
cv2.imwrite(out_file_v, v)
def main():
iext = 'tif'
tvl1 = cv2.DualTVL1OpticalFlow_create()
folderlists = [
os.path.join(input_root, 'UCSDped1', 'Train'),
os.path.join(input_root, 'UCSDped1', 'Test'),
os.path.join(input_root, 'UCSDped2', 'Train'),
os.path.join(input_root, 'UCSDped2', 'Test'),
]
for folder in folderlists:
framefolders = os.listdir(folder)
for framefolder in framefolders:
images = sorted(
glob.glob(os.path.join(folder, framefolder, '*.' + iext)))
for i in range(len(images) - 1):
output_path = os.path.join(
output_root,
os.path.dirname(os.path.relpath(images[i], input_root)))
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(output_path, '{:06d}.flo'.format(i))
print('Processing', output_path)
im1 = np.array(
Image.open(images[i]).resize((256, 256), Image.BILINEAR))
im2 = np.array(
Image.open(images[i + 1]).resize((256, 256),
Image.BILINEAR))
flow = tvl1.calc(im1, im2, None)
writeFlow(output_path, flow)
def main(argv=sys.argv[1:]):
if len(argv) != 2:
print('Please enter "denseOF.py [current image] [next image]')
# print('Please enter "denseOF.py [current image] [next image] [ground truth.flo] ')
return
# load images
img_curr = cv2.imread(argv[0], -1)
img_next = cv2.imread(argv[1], -1)
# color conversion
if len(img_curr.shape) != 2:
img_curr = cv2.cvtColor(img_curr, cv2.COLOR_BGR2GRAY)
img_next = cv2.cvtColor(img_next, cv2.COLOR_BGR2GRAY)
# do optical flow
optical_flow = cv2.DualTVL1OpticalFlow_create()
# optical_flow = cv2.FarnebackOpticalFlow_create()
flow = optical_flow.calc(img_curr, img_next, None)
# do coloring
bgr = flow2bgr_middlebury(flow)
cv2.imshow('flow', bgr)
'''
# load ground truth data
flow_gt = read_flo(argv[2])
bgr_gt = flow2bgr_middlebury(flow_gt)
cv2.imshow('flow_gt', bgr_gt)
'''
cv2.waitKey(0)
def compute_TVL1(video_path):
"""Compute the TV-L1 optical flow."""
flow = []
TVL1 = cv2.DualTVL1OpticalFlow_create()
vidcap = cv2.VideoCapture(video_path)
success,frame1 = vidcap.read()
bins = np.linspace(-20, 20, num=256)
prev = cv2.cvtColor(frame1,cv2.COLOR_RGB2GRAY)
vid_len = get_video_length(video_path)
for _ in range(0,vid_len-1):
success, frame2 = vidcap.read()
curr = cv2.cvtColor(frame2, cv2.COLOR_RGB2GRAY)
curr_flow = TVL1.calc(prev, curr, None)
assert(curr_flow.dtype == np.float32)
#Truncate large motions
curr_flow[curr_flow >= 20] = 20
curr_flow[curr_flow <= -20] = -20
#digitize and scale to [-1;1]
curr_flow = np.digitize(curr_flow, bins)
curr_flow = (curr_flow/255.)*2 - 1
#cropping the center
curr_flow = curr_flow[8:232, 48:272]
flow.append(curr_flow)
prev = curr
vidcap.release()
flow = np.asarray([np.array(flow)])
print('Save flow with shape ', flow.shape)
np.save(SAVE_DIR+'/flow.npy', flow)
return flow
def main():
iext = 'jpg'
tvl1 = cv2.DualTVL1OpticalFlow_create()
folderlists = [
os.path.join(input_root, 'Train'),
os.path.join(input_root, 'Test'),
]
for folder in folderlists:
framefolders = os.listdir(folder)
for framefolder in framefolders:
images = sorted(
glob.glob(os.path.join(folder, framefolder, '*.' + iext)))
for i in range(len(images) - 2):
output_path = os.path.join(
output_root,
os.path.dirname(os.path.relpath(images[i], input_root)))
if not os.path.exists(output_path):
os.makedirs(output_path)
print('Processing',
os.path.join(output_path, '{:06d}'.format(i)))
im1 = np.array(
Image.open(images[i]).resize((256, 256),
Image.BILINEAR).convert('L'))
im2 = np.array(
Image.open(images[i + 2]).resize(
(256, 256), Image.BILINEAR).convert('L'))
flow = tvl1.calc(im1, im2, None)
flow = to_img(flow)
Image.fromarray(flow[:, :, 0]).save(
os.path.join(output_path, '{:06d}_u.jpg'.format(i)))
Image.fromarray(flow[:, :, 1]).save(
os.path.join(output_path, '{:06d}_v.jpg'.format(i)))
def __init__(self,
algorithm: str = "tvl1",
thirdChannel: bool = False,
fBound: float = 20.,
mode='zero'):
self.thirdChannel = thirdChannel # add chanel third true or false : default False
self.fBound = fBound
self.algorithm = algorithm # name algorithm caculator
self.mode = mode # add chanel third by: all vector zero, or mean_square 2 chanels first
if cv2.__version__ >= '4':
if self.algorithm == 'tvl1':
self.oTVL1 = cv2.optflow.DualTVL1OpticalFlow_create(
scaleStep=0.5, warps=3, epsilon=0.02)
else:
pass
else:
if self.algorithm == 'tvl1':
self.oTVL1 = cv2.DualTVL1OpticalFlow_create(scaleStep=0.5,
warps=3,
epsilon=0.02)
else:
pass
def generate_flow_image(video_parts, sam_index, num_con_frame):
"""Generate optical flow images and delete original images.
Args:
list video_parts: Parts of a full path. Return of get_video_parts().
int sam_index: The index of current sample.
int num_con_frame: The number of continuous frames.
"""
assert num_con_frame > 1
train_or_test, classname, filename_no_ext, filename = video_parts
filename_no_ext_i = filename_no_ext + '-' + '%03d' % (sam_index)
images = sorted(
glob.glob(train_or_test + '/' + classname + '/' + filename_no_ext_i +
'*jpg'))
op_method = args.op_method
bound = args.bound
# Loop of sample interval
for i in range(len(images) // num_con_frame):
# Loop of continuous frames
for j in range(num_con_frame - 1):
img1 = cv2.imread(images[i * num_con_frame + j])
img2 = cv2.imread(images[i * num_con_frame + j + 1])
if args.crop != None:
img1 = img1[args.crop[0]:args.crop[1],
args.crop[2]:args.crop[3]]
img2 = img2[args.crop[0]:args.crop[1],
args.crop[2]:args.crop[3]]
# OpenCV uses BGR as its default colour order for images, matplotlib uses RGB.
img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
if op_method == 'tvl1': # Dual tvl1 algorithm
# dtvl1 = cv2.createOptFlow_DualTVL1()
dtvl1 = cv2.DualTVL1OpticalFlow_create(
) # the same with createOptFlow_DualTVL1
flows = dtvl1.calc(img1, img2, None)
elif op_method == 'fb': # Farneback
flows = cv2.calcOpticalFlowFarneback(
img1, img2, None, 0.6, 3, 25, 7, 5, 1.2,
cv2.OPTFLOW_FARNEBACK_GAUSSIAN)
# save flow image
flow_x = ToImg(flows[..., 0], bound)
flow_y = ToImg(flows[..., 1], bound)
flow_x_filename = train_or_test + '/' + classname + '/' + \
filename_no_ext_i + '_flow_x-%04d.jpg' %(i * (num_con_frame - 1) + j + 1)
flow_y_filename = train_or_test + '/' + classname + '/' + \
filename_no_ext_i + '_flow_y-%04d.jpg' %(i * (num_con_frame - 1) + j + 1)
cv2.imwrite(flow_x_filename, flow_x)
cv2.imwrite(flow_y_filename, flow_y)
# delete original frames
for im in images:
os.remove(im)
def __init__(self, third_channel=False, f_bound=20.0):
self.third_channel = third_channel
self.f_bound = f_bound
self.prev = np.zeros((1, 1))
self.algorithm = 'tvl1'
self.oTVL1 = cv2.DualTVL1OpticalFlow_create(scaleStep=0.5,
warps=3,
epsilon=0.02)
def compute_TVL1(prev, curr, bound=20):
"""Compute the TV-L1 optical flow."""
TVL1 = cv2.DualTVL1OpticalFlow_create()
flow = TVL1.calc(prev, curr, None)
flow[flow>bound]=bound
flow[flow<-bound]=-bound
flow-=-bound
flow*=(255/float(2*bound))
return flow
def compute_TVL1(prev, curr, bound=20):
"""Compute the TV-L1 optical flow."""
TVL1 = cv2.DualTVL1OpticalFlow_create()
flow = TVL1.calc(prev, curr, None)
flow = np.clip(flow, -bound, bound)
flow = (flow + bound) * (255.0 / (2 * bound))
flow = np.round(flow).astype('uint8')
return flow
def __init__(self,
camera,
detector_stride,
background,
delete_threshold_period=1.0,
stride=2,
detectLines=True,
readDials=True,
do_tracking=True,
alpha=0.8):
super().__init__(camera, ['track', 'line-segmentation'], stride)
self._tracking = []
self.do_tracking = do_tracking #this should only be False if we're using darkflow
self.alpha = alpha #this is the spring constant
self.labels = {}
self.stride = stride
self.ticks = 0
if (do_tracking):
self.optflow = cv2.DualTVL1OpticalFlow_create(
) #use dense optical flow to track
self.detect_interval = 3
self.prev_gray = None
self.tracks = []
self.min_pts_near = 4 #the minimum number of points we need to say an object's center is here
self.pts_dist_squared_th = int(75.0 / 2 / 720.0 *
background.shape[0])**2
self.feature_params = dict(maxCorners=500,
qualityLevel=0.3,
minDistance=7,
blockSize=7)
print('initializing trackerprocessor. background.shape is {} by {}'.
format(background.shape[0], background.shape[1]))
self.dist_th_upper = int(
150.0 / 720.0 *
background.shape[0]) # distance upper threshold, in pixels
self.dist_th_lower = int(
75.0 / 720.0 *
background.shape[0]) # to account for the size of the reactor
print('dist_th_upper is {} and dist_th_lower is {}'.format(
self.dist_th_upper, self.dist_th_lower))
self.max_obs_possible = 24
# set up line detector
if detectLines:
self.lineDetector = LineDetectionProcessor(camera, stride,
background)
else:
self.lineDetector = None
# need to keep our own ticks because
# we don't know frame index when track() is called
if detector_stride > 0:
self.ticks_per_obs = detector_stride * delete_threshold_period / self.stride
if readDials:
self.dialReader = DialProcessor(camera, stride=1)
else:
self.dialReader = None
def burst_flows_to_shm(vid_path, alg_type, temp_burst_dir, image_ext,
flow_size):
try:
cap = cv2.VideoCapture(vid_path)
if alg_type == 'farn':
optical_flow_hdl = cv2.FarnebackOpticalFlow_create(
pyrScale=0.702,
numLevels=5,
winSize=10,
numIters=2,
polyN=5,
polySigma=1.1,
flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN)
elif alg_type == 'tvl1':
optical_flow_hdl = cv2.DualTVL1OpticalFlow_create()
else:
raise NotImplementedError(
'optical flow algorithm {} is not implemented.'.format(
alg_type))
ret, frame1 = cap.read()
prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
prvs = resize_by_short_edge(prvs, flow_size)
hsv = np.zeros_like(frame1)
hsv[..., 1] = 255
idx = 1
while True:
ret, frame2 = cap.read()
if not ret:
break
next = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
next = resize_by_short_edge(next, flow_size)
flow = optical_flow_hdl.calc(prvs, next, None)
"""
flow_x = cv2.normalize(flow[..., 0], None, 0, 255, cv2.NORM_MINMAX)
flow_y = cv2.normalize(flow[..., 1], None, 0, 255, cv2.NORM_MINMAX)
flow_x = flow_x.astype('uint8')
flow_y = flow_y.astype('uint8')
video_flow_list.append([flow_x,flow_y])
"""
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
cv2.imwrite(
os.path.join(temp_burst_dir,
'{:04d}{}'.format(idx, image_ext)), bgr)
prvs = next
idx += 1
except Exception as e:
print(repr(e))
def compute_TVL1(prev, curr, bound=15):
"""Compute the TV-L1 optical flow."""
TVL1 = cv2.DualTVL1OpticalFlow_create()
flow = TVL1.calc(prev, curr, None)
assert flow.dtype == np.float32
flow = (flow + bound) * (255.0 / (2*bound))
flow = np.round(flow).astype(int)
flow[flow >= 255] = 255
flow[flow <= 0] = 0
return flow
def evaluate_parameter(config_item):
prev_img = cv2.imread(config_item["files"]["prevImg"])
curr_img = cv2.imread(config_item["files"]["currImg"])
flow_method = config_item["parameter"]["flow_method"]
estimate_base = config_item["files"]["estimatepath"] + "/"
if os.path.exists(estimate_base) == False:
os.makedirs(estimate_base)
if os.path.exists(config_item["files"]["estflow"]):
return
# compute optical flow
if flow_method.find("dual") >= 0:
dual_proc = cv2.DualTVL1OpticalFlow_create(config_item["parameter"]["tau"],
config_item["parameter"]["lambda"],
config_item["parameter"]["theta"],
config_item["parameter"]["nscales"],
config_item["parameter"]["warps"])
est_flow = np.zeros(shape=(prev_img.shape[0], prev_img.shape[1],2), dtype=np.float32)
dual_proc.calc(cv2.cvtColor(prev_img, cv2.COLOR_BGR2GRAY), cv2.cvtColor(curr_img, cv2.COLOR_BGR2GRAY), est_flow)
#
elif flow_method.find("farneback") >= 0:
est_flow = cv2.calcOpticalFlowFarneback(cv2.cvtColor(prev_img, cv2.COLOR_BGR2GRAY),
cv2.cvtColor(curr_img, cv2.COLOR_BGR2GRAY),
None, 0.5, 3, 15, 3, 5, 1.2, 0)
elif flow_method.find("PLK") >= 0:
prev_pts = list()
for r in range(prev_img.shape[0]):
for c in range(prev_img.shape[1]):
prev_pts.append((c,r))
prev_pts = np.array(prev_pts, dtype=np.float32)
parameters = get_plk_params()
curr_pts, st, err = cv2.calcOpticalFlowPyrLK(cv2.cvtColor(prev_img, cv2.COLOR_BGR2GRAY),
cv2.cvtColor(curr_img, cv2.COLOR_BGR2GRAY),
prev_pts, None,
parameters.winSize, parameters.maxLevel=3, parameters.criteria, parameters.20,
parameters.Epsilon)
est_flow = np.zeros(shape=(prev_img.shape[0], prev_img.shape[1],2), dtype=np.float32)
n = 0
flow_pts = curr_pts - prev_pts
for r in range(prev_img.shape[0]):
for c in range(prev_img.shape[1]):
est_flow[r, c, :] = flow_pts[n,:]
n = n + 1
#here alternative optical flow methods can be applied
#
else:
raise ValueError("flow method has not been implemented")
ut.writeFlowFile(config_item["files"]["estflow"], est_flow)
ut.drawFlowField(config_item["files"]["estflow"][:-3] + "png", est_flow)
print("Done -> ", config_item["files"]["estflow"])
def calc_opt_flow(vid_path, start, end, label):
optflow = cv2.DualTVL1OpticalFlow_create()
cap = cv2.VideoCapture(vid_path)
ret, frame = cap.read()
if ret:
prev = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
hsv = np.zeros_like(frame)
hsv[..., 1] = 255
nframes = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
for fnum in range(nframes):
ret, frame = cap.read()
if ret:
if start <= fnum and fnum <= end:
curr = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
h, w = curr.shape
flow = optflow.calc(prev, curr, None)
#flow = cv2.calcOpticalFlowFarneback(prev, curr, None, 0.5, 3, 15, 3, 7, 1.5, 0)
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
mag_thresh = np.mean(mag) + 3.0 * np.std(mag)
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
bgr_noisy = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
# Threshold the magnitude: Reject all values less than (mu + 3*std)
mag[mag < mag_thresh] = 0
flow[..., 0], flow[...,
1] = cv2.polarToCart(mag,
ang,
angleInDegrees=True)
flow[..., 0] = cv2.normalize(flow[..., 0], None, 0, 255,
cv2.NORM_MINMAX)
flow[..., 1] = cv2.normalize(flow[..., 1], None, 0, 255,
cv2.NORM_MINMAX)
flow = np.concatenate((flow, np.zeros((h, w, 1))), axis=2)
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
cv2.imshow('prev', prev)
cv2.imshow('curr', curr)
cv2.imshow('opt_flow_bgr_noisy', bgr_noisy)
cv2.imshow('opt_flow_bgr: ' + label, bgr)
k = cv2.waitKey(250) & 0xff
if k == 27:
break
prev = curr
#saveOptFlowToImage(flow, save_path, merge=True)
else:
break
cap.release()
def calc_optical_flow(im1, im2, image_height, image_width):
# calculate dense optical flow
# settings from tutorial
# https://docs.opencv.org/3.3.1/d7/d8b/tutorial_py_lucas_kanade.html
cv2_version = int(cv2.__version__.split('.')[0])
frame1 = image_manipulation.resize_image(cv2.imread(im1), image_height,
image_width)
f1_gray = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
frame2 = image_manipulation.resize_image(cv2.imread(im2), image_height,
image_width)
f2_gray = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
# if cv2_version > 2:
# # NATES CV2
# flow = cv2.calcOpticalFlowFarneback(f1_gray,f2_gray, None, 0.5, 3, 15, 3, 5, 1.2, 0)
# else:
# # MATTS CV2
# flow = cv2.calcOpticalFlowFarneback(f1_gray,f2_gray, 0.5, 3, 30, 3, 5, 1.2, 0)
oflow_tvl1 = cv2.DualTVL1OpticalFlow_create()
flow = oflow_tvl1.calc(f1_gray, f2_gray, None)
h_oflow = flow[..., 0]
v_oflow = flow[..., 1]
print "\tBefore adjustment..."
print "\tmax: ", h_oflow.max()
print "\tmin: ", h_oflow.min()
print "\tmean: ", h_oflow.mean()
# h_oflow[h_oflow < -127] = -127
# h_oflow[h_oflow > 127] = 127
# v_oflow[v_oflow < -127] = -127
# v_oflow[v_oflow > 127] = 127
# h_oflow = np.rint(h_oflow)
# v_oflow = np.rint(v_oflow)
#h_oflow += 127
#v_oflow += 127
#h_oflow[h_oflow > 255] = 255
#h_oflow[h_oflow < 0] = 0
#v_oflow[v_oflow > 255] = 255
#v_oflow[v_oflow < 0] = 0
#h_oflow = np.rint(h_oflow)
#_oflow = np.rint(v_oflow)
print "\tAfter adjustment..."
print "\tmax: ", h_oflow.max()
print "\tmin: ", h_oflow.min()
print "\tmean: ", h_oflow.mean()
return h_oflow, v_oflow
def convert_video(video_path, out_dir, ext):
cap = cv2.VideoCapture(video_path + ext)
ret, frame1 = cap.read()
frame1 = cv2.resize(frame1, outsize, interpolation=cv2.INTER_AREA)
prev = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
if algorithm == 'brox':
prev = np.float32(prev) / 255.0
u_list = list()
v_list = list()
ur_list = list()
vr_list = list()
count = 0
optical_flow = cv2.DualTVL1OpticalFlow_create()
while (ret):
ret, frame2 = cap.read()
if not ret:
break
count += 1
frame2 = cv2.resize(frame2, outsize, interpolation=cv2.INTER_AREA)
next = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
if algorithm == 'brox':
next = np.float32(next) / 255.0
flow = cv2.pythoncuda.gpuOpticalFlowBrox(prev, next, None)
elif algorithm == 'tvl1':
flow = optical_flow.calc(prev, next, None)
elif algorithm == 'farneback':
flow = cv2.pythoncuda.gpuOpticalFlowFarneback(
prev, next, None, 0.5, 3, 15, 3, 5, 1.2, 0)
u_out, v_out = prep_flow_frame(flow[..., 0].copy(), flow[...,
1].copy())
u_list += [u_out]
v_list += [v_out]
u_range = [np.min(flow[..., 0]), np.max(flow[..., 0])]
v_range = [np.min(flow[..., 1]), np.max(flow[..., 1])]
ur_list += [u_range]
vr_list += [v_range]
prev = next
video_name = video_path.split('/')[-1]
store_video_flow(u_list, v_list, ur_list, vr_list, out_dir, video_name)
cap.release()
def main():
# script_checking_flows()
script_save_flows_gpu()
return
print 'hello'
im1 = '../data/ucf101/val_data/rgb_10_fps_256/video_validation_0000001/frame000003.jpg'
im2 = '../data/ucf101/val_data/rgb_10_fps_256/video_validation_0000001/frame000004.jpg'
im1 = cv2.imread(im1, cv2.IMREAD_GRAYSCALE)
im2 = cv2.imread(im2, cv2.IMREAD_GRAYSCALE)
print im1.shape
print im2.shape
optical_flow = cv2.DualTVL1OpticalFlow_create()
flow = optical_flow.calc(im1, im2, None)
print flow.shape, np.min(flow), np.max(flow)
flow = np.clip(flow, -20, 20)
print flow.shape, np.min(flow), np.max(flow)
flow = (flow + 20) / 40 * 255
print flow.shape, np.min(flow), np.max(flow)
u = flow[:, :, 0]
v = flow[:, :, 1]
out_file_u = '../scratch/check_u.jpg'
out_file_v = '../scratch/check_v.jpg'
combo_file = '../scratch/combo.jpg'
combo_file_cv2 = '../scratch/combo_cv2.jpg'
cv2.imwrite(out_file_u, u)
cv2.imwrite(out_file_v, v)
u = cv2.imread(out_file_u, cv2.IMREAD_GRAYSCALE)
v = cv2.imread(out_file_v, cv2.IMREAD_GRAYSCALE)
print u.shape, np.min(u), np.max(u)
print v.shape, np.min(v), np.max(v)
combo = np.concatenate([
u[:, :, np.newaxis], v[:, :, np.newaxis], 128 * np.ones(
(v.shape[0], v.shape[1], 1))
],
axis=2)
print u.shape, np.min(u), np.max(u)
print v.shape, np.min(v), np.max(v)
print combo.shape, np.min(combo), np.max(combo)
scipy.misc.imsave(combo_file, combo.astype(dtype=np.uint32))
cv2.imwrite(combo_file_cv2, combo)
def flow(X, size=256, center=224):
X_flow = []
optical_flow = cv2.DualTVL1OpticalFlow_create()
for i in range(len(X)):
X[i] = cv2.cvtColor(cv2.resize(X[i], (224, 224)), cv2.COLOR_BGR2GRAY)
for i in range(1, len(X)):
flow = np.zeros(X[i].shape)
#flow = cv2.calcOpticalFlowFarneback(X[i-1], X[i], flow=flow, pyr_scale=0.5, levels=3, winsize=15, iterations=3, poly_n=5, poly_sigma=1.2, flags=0)
flow = optical_flow.calc(X[i - 1], X[i], None)
flow[flow >= 20] = 20
flow[flow <= -20] = -20
# scale to [-1, 1]
max_val = lambda x: max(max(x.flatten()), abs(min(x.flatten())))
flow = flow / max_val(flow)
def sample_one_flow(self, video_list):
optical_flow = cv2.DualTVL1OpticalFlow_create()
num_frame = len(video_list)
frame_index = np.random.randint(0, num_frame - 1)
p_frame, n_frame = video_list[frame_index], video_list[frame_index + 1] # R,G,B
# Convert RGB to GRAY
prev_frame = cv2.cvtColor(p_frame[:, :, ::-1], cv2.COLOR_BGR2GRAY)
cur_frame = cv2.cvtColor(n_frame[:, :, ::-1], cv2.COLOR_BGR2GRAY)
# convert to gpu
prev_frame_gpu = cv2.UMat(prev_frame)
cur_frame_gpu = cv2.UMat(cur_frame)
# Compute TVL1-flow
flow_gpu = optical_flow.calc(prev_frame_gpu, cur_frame_gpu, None)
# back to cpu
flow = flow_gpu.get()
# cropping
flow = self.cv_random_crop(flow)
return flow #(height, width, 2)
def generate_optical_flow(index):
clip_length = 2
beg = index - clip_length
end = index + clip_length
image_file = image_path[index].split('_')
idx = int(image_file[3].strip('.png'))
image = Image.open(image_path[index]).resize((228, 128))
img_size = (228, 128)
last_img_filename = image_path[index]
flowfile = image_filename = '_'.join(image_file)
flowpath = image_folder + '/flow/' + flowfile.split('/')[-1].split(
'.')[0] + '.npy'
prev_image = image
F = []
for i in range(idx - 2, idx - 2 * clip_length, -2):
image_file[3] = str(i) + '.png'
image_filename = '_'.join(image_file)
try:
image = Image.open(image_filename).resize(img_size, Image.NEAREST)
last_img_filename = image_filename
except Exception:
#print(image_filename, last_img_filename)
image = Image.open(last_img_filename).resize(
img_size, Image.NEAREST)
prvs = cv2.cvtColor(np.array(prev_image), cv2.COLOR_BGR2GRAY)
curr = cv2.cvtColor(np.array(image), cv2.COLOR_BGR2GRAY)
optical_flow = cv2.DualTVL1OpticalFlow_create()
#optical_flow = cv2.optflow.DualTVL1OpticalFlow_create()
flow = optical_flow.calc(curr, prvs, None)
flow = np.array(flow).reshape((2, ) + (128, 228))
F.append(flow)
prev_image = image
Flow = np.array(F)
last_img_filename
#print(Flow, Flow.dtype)
np.save(flowpath, Flow)
def estimate_invflow(img0, img1, me_algo):
'''
Estimates inverse optical flow by using the me_algo algorithm.
'''
# # # img0, img1 have to be uint8 grayscale
assert img0.dtype == 'uint8' and img1.dtype == 'uint8'
# Create estimator object
if me_algo == "DeepFlow":
of_estim = cv2.optflow.createOptFlow_DeepFlow()
elif me_algo == "SimpleFlow":
of_estim = cv2.optflow.createOptFlow_SimpleFlow()
elif me_algo == "TVL1":
of_estim = cv2.DualTVL1OpticalFlow_create()
else:
raise Exception("Incorrect motion estimation algorithm")
# Run flow estimation (inverse flow)
flow = of_estim.calc(img1, img0, None)
# flow = cv.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0)
return flow
def task(train_videos):
for video in train_videos:
rgb_array = []
cap = cv2.VideoCapture(os.path.join(video_path, video))
base_name = video.split('.')[0]
#rgb
while True:
ret, frame = cap.read()
if frame is None:
break
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame = cv2.resize(frame, (224, 224))
rgb_array.append(frame)
cap.release()
rgb_array = np.array(rgb_array)
frames = rgb_array.shape[0]
if frames < 65: #多出一帧计算光流使用
rgb_array = np.repeat(rgb_array, (65 // frames + 1), axis=0)
frames = rgb_array.shape[0]
start = random.randint(0, frames - 64)
np.save(os.path.join(rgb_np_train_path, base_name + '_rgb'),
rgb_array[start:start + 64]) #只保存64帧
#flow
flow_array = []
prvs = cv2.cvtColor(rgb_array[0], cv2.COLOR_RGB2GRAY)
p_flow = cv2.DualTVL1OpticalFlow_create()
for i in range(1, 65): #1-64
next = cv2.cvtColor(rgb_array[i], cv2.COLOR_RGB2GRAY)
flow = p_flow.calc(prvs, next, None)
flow[flow > 20] = 20
flow[flow < -20] = -20
flow_array.append(flow)
prvs = next
flow_array = np.array(flow_array)
np.save(os.path.join(flow_np_train_path, base_name + '_flow'),
flow_array)
img_dir = 'sub01/EP02_01f'
# Image_prefix = './tst_gif'
# suffix = ''
# img_dir = ''
img_seq = glob(os.path.join(Image_prefix, suffix, img_dir, '*.jpg'))
prev = cv.imread(img_seq[0])
prevgray = cv.cvtColor(prev, cv.COLOR_BGR2GRAY)
show_hsv = False
show_glitch = False
cur_glitch = prev.copy()
for i in range(1, len(img_seq)):
img = cv.imread(img_seq[i])
gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
# flow = cv.calcOpticalFlowFarneback(prevgray, gray, None, 0.5, 3, 15, 3, 5, 1.2, 0)
TVL1 = cv.DualTVL1OpticalFlow_create()
flow = TVL1.calc(prevgray, gray, None)
prevgray = gray
cv.imshow('flow', draw_flow(gray, flow))
if show_hsv:
cv.imshow('flow HSV', draw_hsv(flow))
if show_glitch:
cur_glitch = warp_flow(cur_glitch, flow)
cv.imshow('glitch', cur_glitch)
ch = cv.waitKey(500)
if ch == 27:
break
if ch == ord('1'):
show_hsv = not show_hsv
import os
import h5py # needs conda/pip install h5py
import matplotlib.pyplot as plt
import sys
from sevir.display import get_cmap
import cv2
import re
import numpy as np
import matplotlib
import glob
from tqdm import tqdm
import pickle
OPTICAL_FLOW_CALC_METHOD = cv2.DualTVL1OpticalFlow_create()
def calc_optical_flow(sevir_np_data):
flows = []
for index in range(len(sevir_np_data) - 1):
flow = OPTICAL_FLOW_CALC_METHOD.calc(sevir_np_data[index],
sevir_np_data[index + 1], None)
flows.append(flow)
return np.array(flows)
inFile = sys.argv[1]
outFile = sys.argv[2]
print("Reading from " + inFile + " and writing to " + outFile)
with open(inFile, "rb") as pickle_file:
data = pickle.load(pickle_file)
opt_flows = []
for curr_event_id, curr_np_data_arr in data:
def video_to_image_and_of(video_path,
target_fps=25,
resize_height=256,
crop_size=224,
n_steps=100,
plotting=False,
flow=False):
# preprocessing:
# video_path = '/media/ROIPO/Data/projects/Adversarial/database/video/v_LongJump_g03_c06.avi'
#video_path = '/home/ADAMGE/Downloads/test_jog.mp4'
# target_fps = 25.0
# n_steps = 100
# resize_height = 256
# crop_size = 224
clip_frames = []
clip_frames_flow = []
bit = 0
capture = cv2.VideoCapture(video_path)
fps = capture.get(cv2.CAP_PROP_FPS)
frame_gap = int(round(fps / target_fps))
if frame_gap == 0:
raise ValueError
frame_num = 0
ret, frame1 = capture.read()
resized_frame1 = image_resize(frame1,
height=resize_height,
width=resize_height)
prvs = cv2.cvtColor(resized_frame1, cv2.COLOR_BGR2GRAY)
hsv = np.zeros_like(frame1)
hsv[..., 1] = 255
# extract features
while (capture.isOpened()) & (bit == 0):
flag, frame = capture.read()
if flag == 0:
bit = 1
print("******ERROR: Could not read frame in " + video_path +
" frame_num: " + str(frame_num))
break
#name = params['res_vids_path'] + str(frame_num) + 'frame.jpg'
#cv2.imwrite(name, frame)
#cv2.imshow("Vid", frame)
#key_pressed = cv2.waitKey(10) # Escape to exit
# process frame (according to the correct frame rate)
if frame_num % frame_gap == 0:
# RGB
image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
resized_frame = image_resize(image,
width=resize_height,
height=resize_height)
res = resized_frame.astype('float32') / 128. - 1
res = crop_center_image(res, crop_size)
clip_frames.append(res)
if plotting:
res_to_plot = cv2.cvtColor(res, cv2.COLOR_RGB2BGR)
res_to_plot = res_to_plot + 1.0 / 2.0
cv2.imshow("Vid", res_to_plot)
key_pressed = cv2.waitKey(10) # Escape to exit
# FLOW
if flow:
image_flow = cv2.cvtColor(resized_frame, cv2.COLOR_RGB2GRAY)
# flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0)
optical_flow = cv2.DualTVL1OpticalFlow_create()
flow = optical_flow.calc(prvs, image_flow, None)
flow[flow > 20] = 20
flow[flow < -20] = -20
flow = flow / 20.
flow = crop_center_image(flow, crop_size)
clip_frames_flow.append(flow)
# potting:
if plotting:
flow_temp = (flow + 1.0) / 2.0
last_channel = np.zeros(
(crop_size, crop_size), dtype=float) + 0.5
flow_to_plot = np.dstack((flow_temp, last_channel))
cv2.imshow("Vid-flow", flow_to_plot)
key_pressed = cv2.waitKey(10) # Escape to exit
#mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
#hsv[..., 0] = ang * 180 / np.pi / 2
#hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
#bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
#cv2.imshow('frame2', bgr)
#k = cv2.waitKey(30) & 0xff
prvs = image_flow
frame_num += 1
capture.release()
# (int(round(frame_num / frame_gap)) < n_steps)
if frame_num >= n_steps:
frames = np.array(clip_frames)[-n_steps:]
frames_flow = np.array(clip_frames_flow)
frames = np.expand_dims(frames, axis=0)
frames_flow = np.expand_dims(frames_flow, axis=0)
else:
frames = np.array(clip_frames)
frames = np.pad(frames, ((0, n_steps - frames.shape[0]), (0, 0),
(0, 0), (0, 0)), 'wrap')
frames_flow = np.array(clip_frames_flow)
frames = np.expand_dims(frames, axis=0)
frames_flow = np.expand_dims(frames_flow, axis=0)
return frames, frames_flow
