def _check_outputs_open(self):
"""checks that output video and event datasets files are open"""
if self.video_output_file is not None:
return
if not self.height or not self.width:
raise ValueError('height and width not set for output video')
if self.output_path is None and self.video_output_file is str:
logger.warning('output_path is None; will not write DVS video')
if self.output_path and type(self.video_output_file_name) is str:
fn = checkAddSuffix(
os.path.join(self.output_path, self.video_output_file_name),
'.avi')
logger.info('opening DVS video output file ' + fn)
self.video_output_file = video_writer(
fn, self.height, self.width, frame_rate=self.avi_frame_rate)
fn = checkAddSuffix(
os.path.join(self.output_path, self.video_output_file_name),
self.dvs_frame_times_suffix)
logger.info('opening DVS frame times file ' + fn)
self.frame_times_output_file = open(fn, 'w')
s = '# frame times for {}\n# frame# time(s)\n'.format(
self.video_output_file_name)
self.frame_times_output_file.write(s)
def render(self):
"""Render event frames."""
(event_arr, time_list, pos_list,
num_frames, height, width) = self._get_events()
output_ts = np.linspace(
0,
num_frames / self.input_fps,
int(num_frames / self.input_fps * self.output_fps),
dtype=np.float)
clip_value = 2
histrange = [(0, v) for v in (height, width)]
out = video_writer(
os.path.join(self.output_path, 'output.avi'),
width=width, height=height, frame_rate=self.avi_frame_rate)
for ts_idx in range(output_ts.shape[0] - 1):
# assume time_list is sorted.
start = np.searchsorted(time_list,
output_ts[ts_idx],
side='right')
end = np.searchsorted(time_list,
output_ts[ts_idx + 1],
side='right')
# select events, assume that pos_list is sorted
if end < len(pos_list):
events = event_arr[pos_list[start]: pos_list[end], :]
else:
events = event_arr[pos_list[start]:, :]
pol_on = (events[:, 3] == 1)
pol_off = np.logical_not(pol_on)
img_on, _, _ = np.histogram2d(
events[pol_on, 2], events[pol_on, 1],
bins=(height, width), range=histrange)
img_off, _, _ = np.histogram2d(
events[pol_off, 2], events[pol_off, 1],
bins=(height, width), range=histrange)
if clip_value is not None:
integrated_img = np.clip(
(img_on - img_off), -clip_value, clip_value)
else:
integrated_img = (img_on - img_off)
img = (integrated_img + clip_value) / float(clip_value * 2)
out.write(cv2.cvtColor(
(img * 255).astype(np.uint8), cv2.COLOR_GRAY2BGR))
# if self.preview:
# cv2.namedWindow(__name__, cv2.WINDOW_NORMAL)
# cv2.imshow(__name__, img)
# if not self.preview_resized:
# cv2.resizeWindow(__name__, 800, 600)
# self.preview_resized = True
# cv2.waitKey(30) # 30 hz playback
if ts_idx % 20 == 0:
logger.info('Rendered {} frames'.format(ts_idx))
# if cv2.waitKey(int(1000 / 30)) & 0xFF == ord('q'):
# break
out.release()
def interpolate(self, source_frame_path, output_folder, frame_size):
"""Run interpolation. \
Interpolated frames will be saved in folder self.output_path.
Parameters
----------
source_frame_path: path that contains source file
output_folder:str, folder that stores the interpolated images,
numbered 1:N*slowdown_factor.
frame_size: tuple (width, height)
Frames will include the input frames, i.e.
if there are 2 input frames and slowdown_factor=10,
there will be 10 frames written,
starting with the first input frame, and ending before
the 2nd input frame.
If slowdown factor=2, then the first output frame will be
the first input frame, and the 2nd output frame will be
a new synthetic frame halfway to the 2nd frame.
If the slowdown_factor is 3, then there will
the first input frame followed by 2 more interframes.
The output will never include the 2nd input frame.
i.e. if there are 2 input frames and slowdown_factor=10,
there will be 10 frames written, frame0 is the first input frame, and frame9 is the 9th interpolated frame.
Frame1 is *not* included, so that it can be fed as input for the next interpolation.
Returns
deltaTimes: np.array,
Array of delta times relative to src frame intervals. This array must be multiplied by the source frame interval to obtain the times of the frames. There will be a variable number of times depending on auto_upsample and upsampling_factor.
avg_upsampling_factor: float,
Average upsampling factor, which can be used to compute the average timestamp resolution.
"""
if not output_folder:
raise ValueError(
'output_folder is None; it must be supplied to store '
'the interpolated frames')
ls=os.listdir(source_frame_path)
nframes=len(ls)
del ls
if nframes/self.batch_size<2:
logger.warning(f'only {nframes} input frames with batch_size={self.batch_size}, automatically reducing batch size to provide at least 2 batches')
while nframes/self.batch_size<2:
self.batch_size=int(self.batch_size/2)
logger.info(f'using batch_size={self.batch_size}')
video_frame_loader, dim, ori_dim = self.__load_data(
source_frame_path, frame_size)
if not self.model_loaded:
(self.flow_estimator, self.warper,
self.interpolator) = self.__model(dim)
self.model_loaded = True
# construct AVI video output writer now that we know the frame size
if self.video_path is not None and self.vid_orig is not None and \
self.ori_writer is None:
self.ori_writer = video_writer(
os.path.join(self.video_path, self.vid_orig),
ori_dim[1],
ori_dim[0], frame_rate=self.avi_frame_rate
)
if self.video_path is not None and self.vid_slomo is not None and \
self.slomo_writer is None:
self.slomo_writer = video_writer(
os.path.join(self.video_path, self.vid_slomo),
ori_dim[1],
ori_dim[0], frame_rate=self.avi_frame_rate
)
numUpsamplingReportsLeft=3 # number of times to report automatic upsampling
# prepare preview
if self.preview:
self.name = str(__file__)
cv2.namedWindow(self.name, cv2.WINDOW_NORMAL)
outputFrameCounter=0 # counts frames written out (input + interpolated)
inputFrameCounter=0 # counts source video input frames
# torch.cuda.empty_cache()
upsamplingSum=0 #stats
nUpsamplingSamples=0
with torch.no_grad():
# logger.debug(
# "using " + str(output_folder) +
# " to store interpolated frames")
nImages = len(video_frame_loader)
logger.info(f'interpolating {len(video_frame_loader)} batches of frames using batch_size={self.batch_size} with auto_upsample={self.auto_upsample} and minimum upsampling_factor={self.upsampling_factor}')
if nImages<2:
raise Exception('there are only {} batches in {} and we need at least 2; maybe you need to reduce batch size or increase number of input frames'.format(nImages, source_frame_path))
interpTimes=None # array to hold times normalized to 1 unit per input frame interval
unit = ' fr' if self.batch_size == 1 \
else ' batch of '+str(self.batch_size)+' fr'
for _, (frame0, frame1) in enumerate(
tqdm(video_frame_loader, desc='slomo-interp',
unit=unit), 0):
# video_frame_loader delivers self.batch_size batch of frame0 and frame1
# frame0 is actually frame0, frame1,.... frameN
# frame1 is actually frame1, frame2, .... frameN+1, where N is batch_size-1
# that way the slomo computes in parallel the flow from 0->1, 1->2, 2->3... N-1->N
I0 = frame0.to(self.device)
I1 = frame1.to(self.device)
# actual number of frames, account for < batch_size
num_batch_frames = I0.shape[0]
flowOut = self.flow_estimator(torch.cat((I0, I1), dim=1))
F_0_1 = flowOut[:, :2, :, :] # flow from 0 to 1
F_1_0 = flowOut[:, 2:, :, :] # flow from 1 to 0
# dimensions [batch, flow[vx,vy], loc_x,loc_y]
if self.preview:
start_frame_count = outputFrameCounter
# compute the upsampling factor
if self.auto_upsample:
# compute automatic sample time from maximum flow magnitude such that
# # dt(s)*speed(pix/s)=1pix,
# # i.e., dt(s)=1pix/speed(pix/s)
# we have no time here, so our flow is computed in pixels of motion between frames
# we need to compute speed, so first compute the sum square of x and y vel components
vFlat=torch.flatten(flowOut,2,3) # [batch, [v01x, v01y, v10x, v10y] ]
vx0=vFlat[:,0,:]
vx1=vFlat[:,2,:]
vy0=vFlat[:,1,:]
vy1=vFlat[:,3,:]
sp0=torch.sqrt(vx0*vx0+vy0*vy0)
sp1=torch.sqrt(vx1*vx1+vy1*vy1)
sp=torch.cat((sp0,sp1),1)
maxSpeed= torch.max(torch.max(sp,dim=1)[0]).cpu().item() # this is maximimum movement between frames in pixels dim [batch]
# dim=1 gets max over all pixels
# [0] gets value of max, rather than idx which would be 1
# outer max get max over entire batch
# .cpu() moves to cpu to get actual value as float
# outer .item() gets first element of 0-dim tensor which is the speed
upsampling_factor=int(np.ceil(maxSpeed)) # use ceil to ensure oversampling. compute overall maximum needed upsampling ratio
# it is shared over all frames in batch so just use max value for all of them
# logger.info('upsampling factor={}'.format(upsampling_factor))
if self.upsampling_factor is not None and self.upsampling_factor>upsampling_factor:
upsampling_factor=self.upsampling_factor
if numUpsamplingReportsLeft>0:
logger.info('upsampled by factor {}'.format(upsampling_factor))
numUpsamplingReportsLeft-=1
else:
upsampling_factor=self.upsampling_factor
if upsampling_factor<2:
logger.warning('upsampling_factor was less than 2 (maybe very slow motion caused this); set it to 2')
upsampling_factor=2
nUpsamplingSamples+=1
upsamplingSum+=upsampling_factor
# compute normalized frame times where 1 is full interval between frames
# each src frame increments time by 1 unit, interframes fill between.
numOutputFramesThisBatch= upsampling_factor*num_batch_frames
interframeTime = 1/upsampling_factor
# compute the times of *all* the new frames, covering upsampling_factor * numFramesThisBatch total frames
# they all share the same upsampling_factor within this batch, hence same interframeTime
interframeTimes = inputFrameCounter + np.array(range(numOutputFramesThisBatch))*interframeTime
interframeTimes = interframeTimes.squeeze() # remove trailing , dimension
if interpTimes is None:
interpTimes=interframeTimes
else:
interpTimes=np.concatenate((interpTimes,interframeTimes))
# Generate intermediate frames using upsampling_factor
# this part is also done in batch mode
for intermediateIndex in range(0, upsampling_factor):
t = (intermediateIndex + 0.5) / upsampling_factor
temp = -t * (1 - t)
fCoeff = [temp, t * t, (1 - t) * (1 - t), temp]
F_t_0 = fCoeff[0] * F_0_1 + fCoeff[1] * F_1_0
F_t_1 = fCoeff[2] * F_0_1 + fCoeff[3] * F_1_0
g_I0_F_t_0 = self.warper(I0, F_t_0)
g_I1_F_t_1 = self.warper(I1, F_t_1)
intrpOut = self.interpolator(
torch.cat(
(I0, I1, F_0_1, F_1_0,
F_t_1, F_t_0, g_I1_F_t_1,
g_I0_F_t_0), dim=1))
F_t_0_f = intrpOut[:, :2, :, :] + F_t_0
F_t_1_f = intrpOut[:, 2:4, :, :] + F_t_1
V_t_0 = torch.sigmoid(intrpOut[:, 4:5, :, :])
V_t_1 = 1 - V_t_0
g_I0_F_t_0_f = self.warper(I0, F_t_0_f)
g_I1_F_t_1_f = self.warper(I1, F_t_1_f)
wCoeff = [1 - t, t]
Ft_p = (wCoeff[0] * V_t_0 * g_I0_F_t_0_f +
wCoeff[1] * V_t_1 * g_I1_F_t_1_f) / \
(wCoeff[0] * V_t_0 + wCoeff[1] * V_t_1)
# Save intermediate frames from this particular upsampling point between src frames
for batchIndex in range(num_batch_frames):
img = self.to_image(Ft_p[batchIndex].cpu().detach())
img_resize = img.resize(ori_dim, Image.BILINEAR)
# the output frame index is computed
outputFrameIdx=outputFrameCounter + upsampling_factor * batchIndex + intermediateIndex
save_path = os.path.join(
output_folder,
str(outputFrameIdx) + ".png")
img_resize.save(save_path)
# for preview
if self.preview:
stop_frame_count = outputFrameCounter
for frame_idx in range(
start_frame_count,
stop_frame_count + upsampling_factor * (num_batch_frames - 1)):
frame_path = os.path.join(
output_folder, str(frame_idx) + ".png")
frame = cv2.imread(frame_path)
cv2.imshow(self.name, frame)
if not self.preview_resized:
cv2.resizeWindow(self.name, 800, 600)
self.preview_resized = True
# wait minimally since interp takes time anyhow
cv2.waitKey(1)
# Set counter accounting for batching of frames
inputFrameCounter += num_batch_frames # batch_size-1 because we repeat frame1 as frame0
outputFrameCounter += numOutputFramesThisBatch # batch_size-1 because we repeat frame1 as frame0
# write input frames into video
# don't duplicate each frame if called using rotating buffer
# of two frames in a row
if self.ori_writer:
src_files = sorted(
glob.glob("{}".format(source_frame_path) + "/*.npy"))
# write original frames into stop-motion video
for frame_idx, src_file_path in enumerate(
tqdm(src_files, desc='write-orig-avi',
unit='fr'), 0):
src_frame = np.load(src_file_path)
self.ori_writer.write(
cv2.cvtColor(src_frame, cv2.COLOR_GRAY2BGR))
self.numOrigVideoFramesWritten += 1
frame_paths = self.__all_images(output_folder)
if self.slomo_writer:
for path in tqdm(frame_paths,desc='write-slomo-vid',unit='fr'):
frame = self.__read_image(path)
self.slomo_writer.write(
cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR))
self.numSlomoVideoFramesWritten += 1
nFramesWritten=len(frame_paths)
nTimePoints=len(interpTimes)
avgUpsampling=upsamplingSum/nUpsamplingSamples
logger.info('Wrote {} frames and returning {} frame times.\nAverage upsampling factor={:5.1f}'.format(nFramesWritten,nTimePoints,avgUpsampling))
return interpTimes, avgUpsampling