def _check_outputs_open(self): """checks that output video and event datasets files are open""" if self.video_output_file is not None: return if not self.height or not self.width: raise ValueError('height and width not set for output video') if self.output_path is None and self.video_output_file is str: logger.warning('output_path is None; will not write DVS video') if self.output_path and type(self.video_output_file_name) is str: fn = checkAddSuffix( os.path.join(self.output_path, self.video_output_file_name), '.avi') logger.info('opening DVS video output file ' + fn) self.video_output_file = video_writer( fn, self.height, self.width, frame_rate=self.avi_frame_rate) fn = checkAddSuffix( os.path.join(self.output_path, self.video_output_file_name), self.dvs_frame_times_suffix) logger.info('opening DVS frame times file ' + fn) self.frame_times_output_file = open(fn, 'w') s = '# frame times for {}\n# frame# time(s)\n'.format( self.video_output_file_name) self.frame_times_output_file.write(s)
def render(self): """Render event frames.""" (event_arr, time_list, pos_list, num_frames, height, width) = self._get_events() output_ts = np.linspace( 0, num_frames / self.input_fps, int(num_frames / self.input_fps * self.output_fps), dtype=np.float) clip_value = 2 histrange = [(0, v) for v in (height, width)] out = video_writer( os.path.join(self.output_path, 'output.avi'), width=width, height=height, frame_rate=self.avi_frame_rate) for ts_idx in range(output_ts.shape[0] - 1): # assume time_list is sorted. start = np.searchsorted(time_list, output_ts[ts_idx], side='right') end = np.searchsorted(time_list, output_ts[ts_idx + 1], side='right') # select events, assume that pos_list is sorted if end < len(pos_list): events = event_arr[pos_list[start]: pos_list[end], :] else: events = event_arr[pos_list[start]:, :] pol_on = (events[:, 3] == 1) pol_off = np.logical_not(pol_on) img_on, _, _ = np.histogram2d( events[pol_on, 2], events[pol_on, 1], bins=(height, width), range=histrange) img_off, _, _ = np.histogram2d( events[pol_off, 2], events[pol_off, 1], bins=(height, width), range=histrange) if clip_value is not None: integrated_img = np.clip( (img_on - img_off), -clip_value, clip_value) else: integrated_img = (img_on - img_off) img = (integrated_img + clip_value) / float(clip_value * 2) out.write(cv2.cvtColor( (img * 255).astype(np.uint8), cv2.COLOR_GRAY2BGR)) # if self.preview: # cv2.namedWindow(__name__, cv2.WINDOW_NORMAL) # cv2.imshow(__name__, img) # if not self.preview_resized: # cv2.resizeWindow(__name__, 800, 600) # self.preview_resized = True # cv2.waitKey(30) # 30 hz playback if ts_idx % 20 == 0: logger.info('Rendered {} frames'.format(ts_idx)) # if cv2.waitKey(int(1000 / 30)) & 0xFF == ord('q'): # break out.release()
def interpolate(self, source_frame_path, output_folder, frame_size): """Run interpolation. \ Interpolated frames will be saved in folder self.output_path. Parameters ---------- source_frame_path: path that contains source file output_folder:str, folder that stores the interpolated images, numbered 1:N*slowdown_factor. frame_size: tuple (width, height) Frames will include the input frames, i.e. if there are 2 input frames and slowdown_factor=10, there will be 10 frames written, starting with the first input frame, and ending before the 2nd input frame. If slowdown factor=2, then the first output frame will be the first input frame, and the 2nd output frame will be a new synthetic frame halfway to the 2nd frame. If the slowdown_factor is 3, then there will the first input frame followed by 2 more interframes. The output will never include the 2nd input frame. i.e. if there are 2 input frames and slowdown_factor=10, there will be 10 frames written, frame0 is the first input frame, and frame9 is the 9th interpolated frame. Frame1 is *not* included, so that it can be fed as input for the next interpolation. Returns deltaTimes: np.array, Array of delta times relative to src frame intervals. This array must be multiplied by the source frame interval to obtain the times of the frames. There will be a variable number of times depending on auto_upsample and upsampling_factor. avg_upsampling_factor: float, Average upsampling factor, which can be used to compute the average timestamp resolution. """ if not output_folder: raise ValueError( 'output_folder is None; it must be supplied to store ' 'the interpolated frames') ls=os.listdir(source_frame_path) nframes=len(ls) del ls if nframes/self.batch_size<2: logger.warning(f'only {nframes} input frames with batch_size={self.batch_size}, automatically reducing batch size to provide at least 2 batches') while nframes/self.batch_size<2: self.batch_size=int(self.batch_size/2) logger.info(f'using batch_size={self.batch_size}') video_frame_loader, dim, ori_dim = self.__load_data( source_frame_path, frame_size) if not self.model_loaded: (self.flow_estimator, self.warper, self.interpolator) = self.__model(dim) self.model_loaded = True # construct AVI video output writer now that we know the frame size if self.video_path is not None and self.vid_orig is not None and \ self.ori_writer is None: self.ori_writer = video_writer( os.path.join(self.video_path, self.vid_orig), ori_dim[1], ori_dim[0], frame_rate=self.avi_frame_rate ) if self.video_path is not None and self.vid_slomo is not None and \ self.slomo_writer is None: self.slomo_writer = video_writer( os.path.join(self.video_path, self.vid_slomo), ori_dim[1], ori_dim[0], frame_rate=self.avi_frame_rate ) numUpsamplingReportsLeft=3 # number of times to report automatic upsampling # prepare preview if self.preview: self.name = str(__file__) cv2.namedWindow(self.name, cv2.WINDOW_NORMAL) outputFrameCounter=0 # counts frames written out (input + interpolated) inputFrameCounter=0 # counts source video input frames # torch.cuda.empty_cache() upsamplingSum=0 #stats nUpsamplingSamples=0 with torch.no_grad(): # logger.debug( # "using " + str(output_folder) + # " to store interpolated frames") nImages = len(video_frame_loader) logger.info(f'interpolating {len(video_frame_loader)} batches of frames using batch_size={self.batch_size} with auto_upsample={self.auto_upsample} and minimum upsampling_factor={self.upsampling_factor}') if nImages<2: raise Exception('there are only {} batches in {} and we need at least 2; maybe you need to reduce batch size or increase number of input frames'.format(nImages, source_frame_path)) interpTimes=None # array to hold times normalized to 1 unit per input frame interval unit = ' fr' if self.batch_size == 1 \ else ' batch of '+str(self.batch_size)+' fr' for _, (frame0, frame1) in enumerate( tqdm(video_frame_loader, desc='slomo-interp', unit=unit), 0): # video_frame_loader delivers self.batch_size batch of frame0 and frame1 # frame0 is actually frame0, frame1,.... frameN # frame1 is actually frame1, frame2, .... frameN+1, where N is batch_size-1 # that way the slomo computes in parallel the flow from 0->1, 1->2, 2->3... N-1->N I0 = frame0.to(self.device) I1 = frame1.to(self.device) # actual number of frames, account for < batch_size num_batch_frames = I0.shape[0] flowOut = self.flow_estimator(torch.cat((I0, I1), dim=1)) F_0_1 = flowOut[:, :2, :, :] # flow from 0 to 1 F_1_0 = flowOut[:, 2:, :, :] # flow from 1 to 0 # dimensions [batch, flow[vx,vy], loc_x,loc_y] if self.preview: start_frame_count = outputFrameCounter # compute the upsampling factor if self.auto_upsample: # compute automatic sample time from maximum flow magnitude such that # # dt(s)*speed(pix/s)=1pix, # # i.e., dt(s)=1pix/speed(pix/s) # we have no time here, so our flow is computed in pixels of motion between frames # we need to compute speed, so first compute the sum square of x and y vel components vFlat=torch.flatten(flowOut,2,3) # [batch, [v01x, v01y, v10x, v10y] ] vx0=vFlat[:,0,:] vx1=vFlat[:,2,:] vy0=vFlat[:,1,:] vy1=vFlat[:,3,:] sp0=torch.sqrt(vx0*vx0+vy0*vy0) sp1=torch.sqrt(vx1*vx1+vy1*vy1) sp=torch.cat((sp0,sp1),1) maxSpeed= torch.max(torch.max(sp,dim=1)[0]).cpu().item() # this is maximimum movement between frames in pixels dim [batch] # dim=1 gets max over all pixels # [0] gets value of max, rather than idx which would be 1 # outer max get max over entire batch # .cpu() moves to cpu to get actual value as float # outer .item() gets first element of 0-dim tensor which is the speed upsampling_factor=int(np.ceil(maxSpeed)) # use ceil to ensure oversampling. compute overall maximum needed upsampling ratio # it is shared over all frames in batch so just use max value for all of them # logger.info('upsampling factor={}'.format(upsampling_factor)) if self.upsampling_factor is not None and self.upsampling_factor>upsampling_factor: upsampling_factor=self.upsampling_factor if numUpsamplingReportsLeft>0: logger.info('upsampled by factor {}'.format(upsampling_factor)) numUpsamplingReportsLeft-=1 else: upsampling_factor=self.upsampling_factor if upsampling_factor<2: logger.warning('upsampling_factor was less than 2 (maybe very slow motion caused this); set it to 2') upsampling_factor=2 nUpsamplingSamples+=1 upsamplingSum+=upsampling_factor # compute normalized frame times where 1 is full interval between frames # each src frame increments time by 1 unit, interframes fill between. numOutputFramesThisBatch= upsampling_factor*num_batch_frames interframeTime = 1/upsampling_factor # compute the times of *all* the new frames, covering upsampling_factor * numFramesThisBatch total frames # they all share the same upsampling_factor within this batch, hence same interframeTime interframeTimes = inputFrameCounter + np.array(range(numOutputFramesThisBatch))*interframeTime interframeTimes = interframeTimes.squeeze() # remove trailing , dimension if interpTimes is None: interpTimes=interframeTimes else: interpTimes=np.concatenate((interpTimes,interframeTimes)) # Generate intermediate frames using upsampling_factor # this part is also done in batch mode for intermediateIndex in range(0, upsampling_factor): t = (intermediateIndex + 0.5) / upsampling_factor temp = -t * (1 - t) fCoeff = [temp, t * t, (1 - t) * (1 - t), temp] F_t_0 = fCoeff[0] * F_0_1 + fCoeff[1] * F_1_0 F_t_1 = fCoeff[2] * F_0_1 + fCoeff[3] * F_1_0 g_I0_F_t_0 = self.warper(I0, F_t_0) g_I1_F_t_1 = self.warper(I1, F_t_1) intrpOut = self.interpolator( torch.cat( (I0, I1, F_0_1, F_1_0, F_t_1, F_t_0, g_I1_F_t_1, g_I0_F_t_0), dim=1)) F_t_0_f = intrpOut[:, :2, :, :] + F_t_0 F_t_1_f = intrpOut[:, 2:4, :, :] + F_t_1 V_t_0 = torch.sigmoid(intrpOut[:, 4:5, :, :]) V_t_1 = 1 - V_t_0 g_I0_F_t_0_f = self.warper(I0, F_t_0_f) g_I1_F_t_1_f = self.warper(I1, F_t_1_f) wCoeff = [1 - t, t] Ft_p = (wCoeff[0] * V_t_0 * g_I0_F_t_0_f + wCoeff[1] * V_t_1 * g_I1_F_t_1_f) / \ (wCoeff[0] * V_t_0 + wCoeff[1] * V_t_1) # Save intermediate frames from this particular upsampling point between src frames for batchIndex in range(num_batch_frames): img = self.to_image(Ft_p[batchIndex].cpu().detach()) img_resize = img.resize(ori_dim, Image.BILINEAR) # the output frame index is computed outputFrameIdx=outputFrameCounter + upsampling_factor * batchIndex + intermediateIndex save_path = os.path.join( output_folder, str(outputFrameIdx) + ".png") img_resize.save(save_path) # for preview if self.preview: stop_frame_count = outputFrameCounter for frame_idx in range( start_frame_count, stop_frame_count + upsampling_factor * (num_batch_frames - 1)): frame_path = os.path.join( output_folder, str(frame_idx) + ".png") frame = cv2.imread(frame_path) cv2.imshow(self.name, frame) if not self.preview_resized: cv2.resizeWindow(self.name, 800, 600) self.preview_resized = True # wait minimally since interp takes time anyhow cv2.waitKey(1) # Set counter accounting for batching of frames inputFrameCounter += num_batch_frames # batch_size-1 because we repeat frame1 as frame0 outputFrameCounter += numOutputFramesThisBatch # batch_size-1 because we repeat frame1 as frame0 # write input frames into video # don't duplicate each frame if called using rotating buffer # of two frames in a row if self.ori_writer: src_files = sorted( glob.glob("{}".format(source_frame_path) + "/*.npy")) # write original frames into stop-motion video for frame_idx, src_file_path in enumerate( tqdm(src_files, desc='write-orig-avi', unit='fr'), 0): src_frame = np.load(src_file_path) self.ori_writer.write( cv2.cvtColor(src_frame, cv2.COLOR_GRAY2BGR)) self.numOrigVideoFramesWritten += 1 frame_paths = self.__all_images(output_folder) if self.slomo_writer: for path in tqdm(frame_paths,desc='write-slomo-vid',unit='fr'): frame = self.__read_image(path) self.slomo_writer.write( cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)) self.numSlomoVideoFramesWritten += 1 nFramesWritten=len(frame_paths) nTimePoints=len(interpTimes) avgUpsampling=upsamplingSum/nUpsamplingSamples logger.info('Wrote {} frames and returning {} frame times.\nAverage upsampling factor={:5.1f}'.format(nFramesWritten,nTimePoints,avgUpsampling)) return interpTimes, avgUpsampling