def check_lowpass(cutoffhz, fs, logger): """ checks if cutoffhz is ok given sample rate fs """ if cutoffhz == 0 or fs == 0: logger.info('lowpass filter is disabled, no need for check') return maxeps = 0.3 tau = 1/(2*np.pi*cutoffhz) dt = 1/fs eps = dt/tau maxdt = tau*maxeps maxcutoff = maxeps/(2*np.pi*dt) if eps > maxeps: logger.warning( 'Lowpass 3dB cutoff is f_3dB={}Hz (time constant tau={}s) with ' 'sample rate fs={}Hz (sample interval dt={}s) ' ',\n but this results in large IIR mixing factor ' 'eps = dt/tau = {:5.3f} > {:4.1f} (maxeps),' '\n which means the lowpass will filter few or even just ' 'last sample, i.e. you will not be lowpassing as expected.' '\nWe recommend either' '\n -decreasing --timestamp_resolution of DVS events below {}s' '\n -decreasing --cutoff_frequency_hz below {}Hz'.format( eng(cutoffhz), eng(tau), eng(fs), eng(dt), eps, maxeps, eng(maxdt), eng(maxcutoff))) else: logger.info( 'Lowpass cutoff is f_3dB={}Hz with tau={}s and ' 'with sample rate fs={}Hz (sample interval dt={}s)' ',\nIt has IIR mixing factor eps={:5.3f} which is OK ' 'because it is less than recommended maxeps={:4.1f}'.format( eng(cutoffhz), eng(tau), eng(fs), eng(dt), eps, maxeps))
def check_lowpass(cutoffhz, fs, logger): """ checks if cutoffhz is ok given sample rate fs """ import numpy as np from engineering_notation import EngNumber as eng if cutoffhz == 0 or fs == 0: logger.info('lowpass filter is disabled, no need for check') return tau = 1 / (2 * np.pi * cutoffhz) dt = 1 / fs eps = dt / tau if eps > 0.3: logger.warning( ' Lowpass cutoff is {}Hz with sample rate {}Hz ' '(sample interval {}ms),\nbut this results in tau={}ms,' 'and large IIR mixing factor eps={:5.3f}>0.3,\n which means your lowpass ' 'will filter few or even 1 samples. \nDecrease --timestamp_resolution of DVS events or decrease --cutoff_frequency_hz' .format(eng(cutoffhz), eng(fs), eng(dt * 1000), eng(tau * 1000), eps)) else: logger.info(' Lowpass cutoff is {}Hz with sample rate {}Hz ' '(sample interval {}ms),\nIt has tau={}ms and ' 'mixing factor eps={:5.3f}'.format(eng(cutoffhz), eng(fs), eng(dt * 1000), eng(tau * 1000), eps))
def print_timing_info(): for k, v in times.items(): # k is the name, v is the list of times a = np.array(v) timing_mean = np.mean(a) timing_std = np.std(a) timing_median = np.median(a) timing_min = np.min(a) timing_max = np.max(a) log.info( '== Timing statistics from all Timer ==\n{} n={}: {}s +/- {}s (median {}s, min {}s max {}s)' .format(k, len(a), eng(timing_mean), eng(timing_std), eng(timing_median), eng(timing_min), eng(timing_max))) if timers[k].numpy_file is not None: try: log.info( f'saving timing data for {k} in numpy file {timers[k].numpy_file}' ) log.info('there are {} times'.format(len(a))) np.save(timers[k].numpy_file, a) except Exception as e: log.error( f'could not save numpy file {timers[k].numpy_file}; caught {e}' ) if timers[k].show_hist: def plot_loghist(x, bins): hist, bins = np.histogram(x, bins=bins) # histogram x linearly if len(bins) < 2 or bins[0] <= 0: log.error(f'cannot plot histogram since bins={bins}') return logbins = np.logspace( np.log10(bins[0]), np.log10(bins[-1]), len(bins)) # use resulting bin ends to get log bins plt.hist( x, bins=logbins ) # now again histogram x, but with the log-spaced bins, and plot this histogram plt.xscale('log') dt = np.clip(a, 1e-6, None) # logbins = np.logspace(np.log10(bins[0]), np.log10(bins[-1]), len(bins)) try: plot_loghist(dt, bins=100) plt.xlabel('interval[ms]') plt.ylabel('frequency') plt.title(k) plt.show() except Exception as e: log.error(f'could not plot histogram: got {e}')
def parse_value(search_str1, search_str2=" "): try: i = info_str.index(search_str1) + len(search_str1) j = info_str.find(search_str2, i) return eng(info_str[i:j]) except (ValueError, IndexError): return None
def print_timing_info(self, logger=None): """ Prints the timing information accumulated for this Timer :param logger: write to the supplied logger, otherwise use the built-in logger """ if len(times) == 0: log.error( f'Timer {self.timer_name} has no statistics; was it used without a "with" statement?' ) return a = np.array(times[self.timer_name]) timing_mean = np.mean(a) # todo use built in print method for timer timing_std = np.std(a) timing_median = np.median(a) timing_min = np.min(a) timing_max = np.max(a) s = '{} n={}: {}s +/- {}s (median {}s, min {}s max {}s)'.format( self.timer_name, len(a), eng(timing_mean), eng(timing_std), eng(timing_median), eng(timing_min), eng(timing_max)) if logger is not None: logger.info(s) else: log.info(s)
def main(): try: ga = Gooey(get_args, program_name="v2e", default_size=(575, 600)) logger.info("Use --ignore-gooey to disable GUI and " "run with command line arguments") ga() except Exception as e: logger.warning( f'{e}: Gooey GUI not available, using command line arguments. \n' f'You can try to install with "pip install Gooey"') args = get_args() # Set output width and height based on the arguments output_width, output_height = set_output_dimension( args.output_width, args.output_height, args.dvs128, args.dvs240, args.dvs346, args.dvs640, args.dvs1024, logger) # setup synthetic input classes and method synthetic_input = args.synthetic_input synthetic_input_module = None synthetic_input_class = None synthetic_input_instance = None synthetic_input_next_frame_method = None if synthetic_input is not None: try: synthetic_input_module = importlib.import_module(synthetic_input) synthetic_input_class = getattr(synthetic_input_module, synthetic_input) synthetic_input_instance = synthetic_input_class( width=output_width, height=output_height, preview=not args.no_preview) synthetic_input_next_frame_method = getattr( synthetic_input_class, 'next_frame') logger.info( f'successfully instanced {synthetic_input_instance} with' 'method {synthetic_input_next_frame_method}:' '{synthetic_input_module.__doc__}') except ModuleNotFoundError as e: logger.error(f'Could not import {synthetic_input}: {e}') v2e_quit(1) except AttributeError as e: logger.error(f'{synthetic_input} method incorrect?: {e}') v2e_quit(1) # set input file input_file = args.input if synthetic_input is None and not input_file: input_file = inputVideoFileDialog() if not input_file: logger.info('no file selected, quitting') v2e_quit() # Set output folder output_folder = set_output_folder( args.output_folder, input_file, args.unique_output_folder if not args.overwrite else False, args.overwrite, args.output_in_place if (not synthetic_input and args.output_folder is None) else False, logger) # input file checking # if (not input_file or not os.path.isfile(input_file) # or not os.path.isdir(input_file)) \ # and not synthetic_input: if not input_file and not synthetic_input: logger.error('input file {} does not exist'.format(input_file)) v2e_quit(1) num_frames = 0 srcNumFramesToBeProccessed = 0 # define video parameters # the input start and stop time, may be round to actual # frame timestamp input_start_time = args.start_time input_stop_time = args.stop_time input_slowmotion_factor: float = args.input_slowmotion_factor timestamp_resolution: float = args.timestamp_resolution auto_timestamp_resolution: bool = args.auto_timestamp_resolution disable_slomo: bool = args.disable_slomo slomo = None # make it later on if not disable_slomo and auto_timestamp_resolution is False \ and timestamp_resolution is None: logger.error('if --auto_timestamp_resolution=False, ' 'then --timestamp_resolution must be set to ' 'some desired DVS event timestamp resolution in seconds, ' 'e.g. 0.01') v2e_quit() if auto_timestamp_resolution is True \ and timestamp_resolution is not None: logger.info( f'auto_timestamp_resolution=True and ' f'timestamp_resolution={timestamp_resolution}: ' f'Limiting automatic upsampling to maximum timestamp interval.') # DVS pixel thresholds pos_thres = args.pos_thres neg_thres = args.neg_thres sigma_thres = args.sigma_thres # Cutoff and noise frequencies cutoff_hz = args.cutoff_hz leak_rate_hz = args.leak_rate_hz if leak_rate_hz > 0 and sigma_thres == 0: logger.warning('leak_rate_hz>0 but sigma_thres==0, ' 'so all leak events will be synchronous') shot_noise_rate_hz = args.shot_noise_rate_hz # Visualization avi_frame_rate = args.avi_frame_rate dvs_vid = args.dvs_vid dvs_vid_full_scale = args.dvs_vid_full_scale vid_orig = args.vid_orig vid_slomo = args.vid_slomo preview = not args.no_preview # Event saving options dvs_h5 = args.dvs_h5 dvs_aedat2 = args.dvs_aedat2 dvs_text = args.dvs_text # Debug feature: if show slomo stats slomo_stats_plot = args.slomo_stats_plot # rotate180 = args.rotate180 # never used, consider removing batch_size = args.batch_size # DVS exposure exposure_mode, exposure_val, area_dimension = \ v2e_check_dvs_exposure_args(args) if exposure_mode == ExposureMode.DURATION: dvsFps = 1. / exposure_val # Writing the info file infofile = write_args_info(args, output_folder) fh = logging.FileHandler(infofile) fh.setLevel(logging.INFO) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) logger.addHandler(fh) # TODO: fix this, this is not reasonable import time time_run_started = time.time() slomoTimestampResolutionS = None if synthetic_input is None: logger.info("opening video input file " + input_file) if os.path.isdir(input_file): if args.input_frame_rate is None: logger.error("When the video is presented as a folder, " "The user has to set --input_frame_rate manually") v2e_quit(1) cap = ImageFolderReader(input_file, args.input_frame_rate) srcFps = cap.frame_rate srcNumFrames = cap.num_frames else: cap = cv2.VideoCapture(input_file) srcFps = cap.get(cv2.CAP_PROP_FPS) srcNumFrames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) # Check frame rate and number of frames if srcFps == 0: logger.error('source {} fps is 0; v2e needs to have a timescale ' 'for input video'.format(input_file)) v2e_quit() if srcNumFrames < 2: logger.warning( 'num frames is less than 2, probably cannot be determined ' 'from cv2.CAP_PROP_FRAME_COUNT') srcTotalDuration = (srcNumFrames - 1) / srcFps # the index of the frames, from 0 to srcNumFrames-1 start_frame = int(srcNumFrames*(input_start_time/srcTotalDuration)) \ if input_start_time else 0 stop_frame = int(srcNumFrames*(input_stop_time/srcTotalDuration)) \ if input_stop_time else srcNumFrames-1 srcNumFramesToBeProccessed = stop_frame - start_frame + 1 # the duration to be processed, should subtract 1 frame when # calculating duration srcDurationToBeProcessed = (srcNumFramesToBeProccessed - 1) / srcFps # redefining start and end time using the time calculated # from the frames, the minimum resolution there is start_time = start_frame / srcFps stop_time = stop_frame / srcFps srcFrameIntervalS = (1. / srcFps) / input_slowmotion_factor slowdown_factor = NO_SLOWDOWN # start with factor 1 for upsampling if disable_slomo: logger.warning( 'slomo interpolation disabled by command line option; ' 'output DVS timestamps will have source frame interval ' 'resolution') # time stamp resolution equals to source frame interval slomoTimestampResolutionS = srcFrameIntervalS elif not auto_timestamp_resolution: slowdown_factor = int( np.ceil(srcFrameIntervalS / timestamp_resolution)) if slowdown_factor < NO_SLOWDOWN: slowdown_factor = NO_SLOWDOWN logger.warning('timestamp resolution={}s is >= source ' 'frame interval={}s, will not upsample'.format( timestamp_resolution, srcFrameIntervalS)) elif slowdown_factor > 100 and cutoff_hz == 0: logger.warning( f'slowdown_factor={slowdown_factor} is >100 but ' 'cutoff_hz={cutoff_hz}. We have observed that ' 'numerical errors in SuperSloMo can cause noise ' 'that makes fake events at the upsampling rate. ' 'Recommend to set physical cutoff_hz, ' 'e.g. --cutoff_hz=200 (or leave the default cutoff_hz)') slomoTimestampResolutionS = srcFrameIntervalS / slowdown_factor logger.info( f'--auto_timestamp_resolution is False, ' f'srcFps={srcFps}Hz ' f'input_slowmotion_factor={input_slowmotion_factor}, ' f'real src FPS={srcFps*input_slowmotion_factor}Hz, ' f'srcFrameIntervalS={eng(srcFrameIntervalS)}s, ' f'timestamp_resolution={eng(timestamp_resolution)}s, ' f'so SuperSloMo will use slowdown_factor={slowdown_factor} ' f'and have ' f'slomoTimestampResolutionS={eng(slomoTimestampResolutionS)}s') if slomoTimestampResolutionS > timestamp_resolution: logger.warning( 'Upsampled src frame intervals of {}s is larger than\n ' 'the desired DVS timestamp resolution of {}s'.format( slomoTimestampResolutionS, timestamp_resolution)) check_lowpass(cutoff_hz, 1 / slomoTimestampResolutionS, logger) else: # auto_timestamp_resolution if timestamp_resolution is not None: slowdown_factor = int( np.ceil(srcFrameIntervalS / timestamp_resolution)) logger.info( f'--auto_timestamp_resolution=True and ' f'timestamp_resolution={eng(timestamp_resolution)}s: ' f'source video will be automatically upsampled but ' f'with at least upsampling factor of {slowdown_factor}') else: logger.info('--auto_timestamp_resolution=True and ' 'timestamp_resolution is not set: ' 'source video will be automatically upsampled to ' 'limit maximum interframe motion to 1 pixel') # the SloMo model, set no SloMo model if no slowdown if not disable_slomo and \ (auto_timestamp_resolution or slowdown_factor != NO_SLOWDOWN): slomo = SuperSloMo(model=args.slomo_model, auto_upsample=auto_timestamp_resolution, upsampling_factor=slowdown_factor, video_path=output_folder, vid_orig=vid_orig, vid_slomo=vid_slomo, preview=preview, batch_size=batch_size) if not synthetic_input and not auto_timestamp_resolution: logger.info(f'\n events will have timestamp resolution ' f'{eng(slomoTimestampResolutionS)}s,') if exposure_mode == ExposureMode.DURATION \ and dvsFps > (1 / slomoTimestampResolutionS): logger.warning('DVS video frame rate={}Hz is larger than ' 'the effective DVS frame rate of {}Hz; ' 'DVS video will have blank frames'.format( dvsFps, (1 / slomoTimestampResolutionS))) if not synthetic_input: logger.info( 'Source video {} has total {} frames with total duration {}s. ' '\nSource video is {}fps with slowmotion_factor {} ' '(frame interval {}s),' '\nWill convert {} frames {} to {}\n' '(From {}s to {}s, duration {}s)'.format( input_file, srcNumFrames, eng(srcTotalDuration), eng(srcFps), eng(input_slowmotion_factor), eng(srcFrameIntervalS), stop_frame - start_frame + 1, start_frame, stop_frame, start_time, stop_time, (stop_time - start_time))) if exposure_mode == ExposureMode.DURATION: dvsNumFrames = np.math.floor(dvsFps * srcDurationToBeProcessed / input_slowmotion_factor) dvsDuration = dvsNumFrames / dvsFps dvsPlaybackDuration = dvsNumFrames / avi_frame_rate start_time = start_frame / srcFps stop_time = stop_frame / srcFps # todo something replicated here, already have start and stop times logger.info('v2e DVS video will have constant-duration frames \n' 'at {}fps (accumulation time {}s), ' '\nDVS video will have {} frames with duration {}s ' 'and playback duration {}s\n'.format( eng(dvsFps), eng(1 / dvsFps), dvsNumFrames, eng(dvsDuration), eng(dvsPlaybackDuration))) else: logger.info('v2e DVS video will have constant-count ' 'frames with {} events), '.format(exposure_val)) emulator = EventEmulator(pos_thres=pos_thres, neg_thres=neg_thres, sigma_thres=sigma_thres, cutoff_hz=cutoff_hz, leak_rate_hz=leak_rate_hz, shot_noise_rate_hz=shot_noise_rate_hz, seed=args.dvs_emulator_seed, output_folder=output_folder, dvs_h5=dvs_h5, dvs_aedat2=dvs_aedat2, dvs_text=dvs_text, show_dvs_model_state=args.show_dvs_model_state, output_width=output_width, output_height=output_height) if args.dvs_params is not None: logger.warning( f'--dvs_param={args.dvs_params} option overrides your selected options for threshold, threshold-mismatch, leak and shot noise rates' ) emulator.set_dvs_params(args.dvs_params) eventRenderer = EventRenderer(output_path=output_folder, dvs_vid=dvs_vid, preview=preview, full_scale_count=dvs_vid_full_scale, exposure_mode=exposure_mode, exposure_value=exposure_val, area_dimension=area_dimension) if synthetic_input_next_frame_method is not None: # array to batch events for rendering to DVS frames events = np.zeros((0, 4), dtype=np.float32) (fr, time) = synthetic_input_instance.next_frame() i = 0 with tqdm(total=synthetic_input_instance.total_frames(), desc='dvs', unit='fr') as pbar: while fr is not None: newEvents = emulator.generate_events(fr, time) pbar.update(1) i += 1 if newEvents is not None and newEvents.shape[0] > 0 \ and not args.skip_video_output: events = np.append(events, newEvents, axis=0) events = np.array(events) if i % batch_size == 0: eventRenderer.render_events_to_frames( events, height=output_height, width=output_width) events = np.zeros((0, 4), dtype=np.float32) (fr, time) = synthetic_input_instance.next_frame() # process leftover events if len(events) > 0 and not args.skip_video_output: eventRenderer.render_events_to_frames(events, height=output_height, width=output_width) else: # file input # timestamps of DVS start at zero and end with # span of video we processed srcVideoRealProcessedDuration = (stop_time-start_time) / \ input_slowmotion_factor num_frames = srcNumFramesToBeProccessed inputHeight = None inputWidth = None inputChannels = None if start_frame > 0: logger.info('skipping to frame {}'.format(start_frame)) for i in tqdm(range(start_frame), unit='fr', desc='src'): ret, _ = cap.read() if not ret: raise ValueError( 'something wrong, got to end of file before ' 'reaching start_frame') logger.info('processing frames {} to {} from video input'.format( start_frame, stop_frame)) with TemporaryDirectory() as source_frames_dir: if os.path.isdir(input_file): # folder input inputWidth = cap.frame_width inputHeight = cap.frame_height inputChannels = cap.frame_channels else: inputWidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) inputHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) inputChannels = 1 if int(cap.get(cv2.CAP_PROP_MONOCHROME)) \ else 3 logger.info( 'Input video {} has W={} x H={} frames each with {} channels'. format(input_file, inputWidth, inputHeight, inputChannels)) if (output_width is None) and (output_height is None): output_width = inputWidth output_height = inputHeight logger.warning( 'output size ({}x{}) was set automatically to ' 'input video size\n Are you sure you want this? ' 'It might be slow.\n Consider using\n ' ' --output_width=346 --output_height=260\n ' 'to match Davis346.'.format(output_width, output_height)) logger.info(f'*** Stage 1/3: ' f'Resizing {srcNumFramesToBeProccessed} input frames ' f'to output size ' f'(with possible RGB to luma conversion)') for inputFrameIndex in tqdm(range(srcNumFramesToBeProccessed), desc='rgb2luma', unit='fr'): # read frame ret, inputVideoFrame = cap.read() if not ret or inputFrameIndex + start_frame > stop_frame: break if output_height and output_width and \ (inputHeight != output_height or inputWidth != output_width): dim = (output_width, output_height) (fx, fy) = (float(output_width) / inputWidth, float(output_height) / inputHeight) inputVideoFrame = cv2.resize(src=inputVideoFrame, dsize=dim, fx=fx, fy=fy, interpolation=cv2.INTER_AREA) if inputChannels == 3: # color if inputFrameIndex == 0: # print info once logger.info( '\nConverting input frames from RGB color to luma') # TODO would break resize if input is gray frames # convert RGB frame into luminance. inputVideoFrame = cv2.cvtColor( inputVideoFrame, cv2.COLOR_BGR2GRAY) # much faster # save frame into numpy records save_path = os.path.join( source_frames_dir, str(inputFrameIndex).zfill(8) + ".npy") np.save(save_path, inputVideoFrame) # print("Writing source frame {}".format(save_path), end="\r") cap.release() with TemporaryDirectory() as interpFramesFolder: interpTimes = None # make input to slomo if auto_timestamp_resolution or slowdown_factor != NO_SLOWDOWN: # interpolated frames are stored to tmpfolder as # 1.png, 2.png, etc logger.info(f'*** Stage 2/3: SloMo upsampling from ' f'{source_frames_dir}') interpTimes, avgUpsamplingFactor = slomo.interpolate( source_frames_dir, interpFramesFolder, (output_width, output_height)) avgTs = srcFrameIntervalS / avgUpsamplingFactor logger.info('SloMo average upsampling factor={:5.2f}; ' 'average DVS timestamp resolution={}s'.format( avgUpsamplingFactor, eng(avgTs))) # check for undersampling wrt the # photoreceptor lowpass filtering if cutoff_hz > 0: logger.warning('Using auto_timestamp_resolution. ' 'checking if cutoff hz is ok given ' 'samplee rate {}'.format(1 / avgTs)) check_lowpass(cutoff_hz, 1 / avgTs, logger) # read back to memory interpFramesFilenames = all_images(interpFramesFolder) # number of frames n = len(interpFramesFilenames) else: logger.info( f'*** Stage 2/3:turning npy frame files to png ' f'from {source_frames_dir}') interpFramesFilenames = [] n = 0 src_files = sorted( glob.glob("{}".format(source_frames_dir) + "/*.npy")) for frame_idx, src_file_path in tqdm(enumerate(src_files), desc='npy2png', unit='fr'): src_frame = np.load(src_file_path) tgt_file_path = os.path.join(interpFramesFolder, str(frame_idx) + ".png") interpFramesFilenames.append(tgt_file_path) n += 1 cv2.imwrite(tgt_file_path, src_frame) interpTimes = np.array(range(n)) # compute times of output integrated frames nFrames = len(interpFramesFilenames) # interpTimes is in units of 1 per input frame, # normalize it to src video time range f = srcVideoRealProcessedDuration / (np.max(interpTimes) - np.min(interpTimes)) # compute actual times from video times interpTimes = f * interpTimes # debug if slomo_stats_plot: from matplotlib import pyplot as plt # TODO debug dt = np.diff(interpTimes) fig = plt.figure() ax1 = fig.add_subplot(111) ax1.set_title( 'Slo-Mo frame interval stats (close to continue)') ax1.plot(interpTimes) ax1.plot(interpTimes, 'x') ax1.set_xlabel('frame') ax1.set_ylabel('frame time (s)') ax2 = ax1.twinx() ax2.plot(dt * 1e3) ax2.set_ylabel('frame interval (ms)') logger.info('close plot to continue') fig.show() # array to batch events for rendering to DVS frames events = np.zeros((0, 4), dtype=np.float32) logger.info(f'*** Stage 3/3: emulating DVS events from ' f'{nFrames} frames') with tqdm(total=nFrames, desc='dvs', unit='fr') as pbar: for i in range(nFrames): fr = read_image(interpFramesFilenames[i]) newEvents = emulator.generate_events( fr, interpTimes[i]) pbar.update(1) if newEvents is not None and newEvents.shape[0] > 0 \ and not args.skip_video_output: events = np.append(events, newEvents, axis=0) events = np.array(events) if i % batch_size == 0: eventRenderer.render_events_to_frames( events, height=output_height, width=output_width) events = np.zeros((0, 4), dtype=np.float32) # process leftover events if len(events) > 0 and not args.skip_video_output: eventRenderer.render_events_to_frames( events, height=output_height, width=output_width) # Clean up eventRenderer.cleanup() emulator.cleanup() if slomo is not None: slomo.cleanup() if num_frames == 0: logger.error('no frames read from file') v2e_quit() totalTime = (time.time() - time_run_started) framePerS = num_frames / totalTime sPerFrame = 1 / framePerS throughputStr = (str(eng(framePerS)) + 'fr/s') \ if framePerS > 1 else (str(eng(sPerFrame)) + 's/fr') logger.info( 'done processing {} frames in {}s ({})\n see output folder {}'.format( num_frames, eng(totalTime), throughputStr, output_folder)) logger.info('generated total {} events ({} on, {} off)'.format( eng(emulator.num_events_total), eng(emulator.num_events_on), eng(emulator.num_events_off))) logger.info('avg event rate {}Hz ({}Hz on, {}Hz off)'.format( eng(emulator.num_events_total / srcDurationToBeProcessed), eng(emulator.num_events_on / srcDurationToBeProcessed), eng(emulator.num_events_off / srcDurationToBeProcessed))) # try to show desktop # suppress folder opening if it's not necessary if not args.skip_video_output and not args.no_preview: try: desktop.open(os.path.abspath(output_folder)) except Exception as e: logger.warning('{}: could not open {} in desktop'.format( e, output_folder))
def main(): try: ga=Gooey(get_args, program_name="v2e", default_size=(575, 600)) logger.info('Use --ignore-gooey to disable GUI and run with command line arguments') ga() except: logger.warning('Gooey GUI not available, using command line arguments. \n' 'You can try to install with "pip install Gooey"') args=get_args() # args=get_args() overwrite: bool = args.overwrite output_folder: str = args.output_folder unique_output_folder: bool = args.unique_output_folder output_folder = makeOutputFolder(output_folder, 0, overwrite, unique_output_folder) input_file = args.input if not input_file: input_file = inputVideoFileDialog() if not input_file: logger.info('no file selected, quitting') v2e_quit() output_width: int = args.output_width output_height: int = args.output_height if (output_width is None) ^ (output_height is None): logger.error('set neither or both of output_width and output_height') v2e_quit() dvs128=args.dvs128 dvs240=args.dvs240 dvs346=args.dvs346 dvs640=args.dvs640 dvs1024=args.dvs1024 if dvs128: output_width=128 output_height=128 elif dvs240: output_width=240 output_height=180 elif dvs346: output_width=346 output_height=260 elif dvs640: output_width=640 output_height=480 elif dvs1024: output_width=1024 output_height=768 # input file checking if not input_file or not Path(input_file).exists(): logger.error('input file {} does not exist'.format(input_file)) v2e_quit() start_time = args.start_time stop_time = args.stop_time input_slowmotion_factor:float = args.input_slowmotion_factor timestamp_resolution:float = args.timestamp_resolution auto_timestamp_resolution:bool=args.auto_timestamp_resolution if auto_timestamp_resolution==False and timestamp_resolution is None: logger.error('if --auto_timestamp_resolution=False, then --timestamp_resolution must be set to ' 'some desired DVS event timestamp resolution in seconds, ' 'e.g. 0.01') v2e_quit() pos_thres = args.pos_thres neg_thres = args.neg_thres sigma_thres = args.sigma_thres cutoff_hz = args.cutoff_hz leak_rate_hz = args.leak_rate_hz if leak_rate_hz > 0 and sigma_thres == 0: logger.warning( 'leak_rate_hz>0 but sigma_thres==0, ' 'so all leak events will be synchronous') shot_noise_rate_hz = args.shot_noise_rate_hz avi_frame_rate = args.avi_frame_rate dvs_vid = args.dvs_vid dvs_vid_full_scale = args.dvs_vid_full_scale dvs_h5 = args.dvs_h5 # dvs_np = args.dvs_np dvs_aedat2 = args.dvs_aedat2 dvs_text = args.dvs_text vid_orig = args.vid_orig vid_slomo = args.vid_slomo slomo_stats_plot=args.slomo_stats_plot preview = not args.no_preview rotate180 = args.rotate180 batch_size = args.batch_size exposure_mode, exposure_val, area_dimension = \ v2e_check_dvs_exposure_args(args) infofile = write_args_info(args, output_folder) fh = logging.FileHandler(infofile) fh.setLevel(logging.INFO) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) logger.addHandler(fh) import time time_run_started = time.time() logger.info("opening video input file " + input_file) cap = cv2.VideoCapture(input_file) srcFps = cap.get(cv2.CAP_PROP_FPS) if srcFps == 0: logger.error('source {} fps is 0'.format(input_file)) v2e_quit() # https://stackoverflow.com/questions/25359288/how-to-know-total-number-of-frame-in-a-file-with-cv2-in-python srcNumFrames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) if srcNumFrames < 2: logger.warning( 'num frames is less than 2, probably cannot be determined ' 'from cv2.CAP_PROP_FRAME_COUNT') srcTotalDuration = (srcNumFrames - 1) / srcFps start_frame = int(srcNumFrames * (start_time / srcTotalDuration)) \ if start_time else 0 stop_frame = int(srcNumFrames * (stop_time / srcTotalDuration)) \ if stop_time else srcNumFrames srcNumFramesToBeProccessed = stop_frame - start_frame + 1 srcDurationToBeProcessed = srcNumFramesToBeProccessed / srcFps start_time = start_frame / srcFps stop_time = stop_frame / srcFps # todo something replicated here, already have start and stop times srcFrameIntervalS = (1. / srcFps) / input_slowmotion_factor slomoTimestampResolutionS = None slowdown_factor=None if not auto_timestamp_resolution: slowdown_factor = int(np.ceil(srcFrameIntervalS / timestamp_resolution)) if slowdown_factor <= 1: slowdown_factor = 1 logger.warning( 'timestamp resolution={}s is >= source ' 'frame interval={}s, will not upsample' .format(timestamp_resolution, srcFrameIntervalS)) logger.info( 'Src video frame rate={:.2f} Hz with slowmotion_factor={:.2f}, \n' 'timestamp resolution={:.3f} ms, computed slomo upsampling factor={}' .format( srcFps, input_slowmotion_factor, timestamp_resolution * 1000, slowdown_factor)) slomoTimestampResolutionS = srcFrameIntervalS / slowdown_factor if slomoTimestampResolutionS > timestamp_resolution: logger.warning( 'Upsampled src frame intervals of {}s is larger than\n ' 'the desired DVS timestamp resolution of {}s' .format(slomoTimestampResolutionS, timestamp_resolution)) check_lowpass(cutoff_hz, 1 / slomoTimestampResolutionS, logger) else: # auto_timestamp_resolution logger.info('--auto_timestamp_resolution=True, \n' 'so source video will be automatically upsampled to limit' 'maximum interframe motion to 1 pixel') # the SloMo model, set no SloMo model if no slowdown if auto_timestamp_resolution or slowdown_factor != NO_SLOWDOWN: slomo = SuperSloMo( model=args.slomo_model, auto_upsample=auto_timestamp_resolution, upsampling_factor=slowdown_factor, video_path=output_folder, vid_orig=vid_orig, vid_slomo=vid_slomo, preview=preview, batch_size=batch_size) else: slomo = None if exposure_mode == ExposureMode.DURATION: dvsFps = 1. / exposure_val if not auto_timestamp_resolution: logger.info('\n events will have timestamp resolution {}s,'.format(slomoTimestampResolutionS)) if exposure_mode == ExposureMode.DURATION and dvsFps > (1 / slomoTimestampResolutionS): logger.warning( 'DVS video frame rate={}Hz is larger than ' 'the effective DVS frame rate of {}Hz; ' 'DVS video will have blank frames'.format( dvsFps, (1 / slomoTimestampResolutionS))) logger.info( 'Source video {} has total {} frames with total duration {}s. ' '\nSource video is {}fps with slowmotion_factor {} ' '(frame interval {}s),' '\nWill convert frames {} to {}\n' '(From {}s to {}s, duration {}s)' .format(input_file, srcNumFrames, eng(srcTotalDuration), eng(srcFps), eng(input_slowmotion_factor), eng(srcFrameIntervalS), start_frame,stop_frame, start_time,stop_time,(stop_time-start_time))) if exposure_mode == ExposureMode.DURATION: dvsNumFrames = np.math.floor( dvsFps * srcDurationToBeProcessed / input_slowmotion_factor) dvsDuration = dvsNumFrames / dvsFps dvsPlaybackDuration = dvsNumFrames / avi_frame_rate start_time = start_frame / srcFps stop_time = stop_frame / srcFps # todo something replicated here, already have start and stop times logger.info('v2e DVS video will have constant-duration frames \n' 'at {}fps (accumulation time {}s), ' '\nDVS video will have {} frames with duration {}s ' 'and playback duration {}s\n' .format(eng(dvsFps), eng(1 / dvsFps), dvsNumFrames, eng(dvsDuration), eng(dvsPlaybackDuration))) else: logger.info('v2e DVS video will have constant-count ' 'frames with {} events), ' .format(exposure_val)) emulator = EventEmulator( pos_thres=pos_thres, neg_thres=neg_thres, sigma_thres=sigma_thres, cutoff_hz=cutoff_hz, leak_rate_hz=leak_rate_hz, shot_noise_rate_hz=shot_noise_rate_hz, output_folder=output_folder, dvs_h5=dvs_h5, dvs_aedat2=dvs_aedat2, dvs_text=dvs_text) if args.dvs_params: emulator.set_dvs_params(args.dvs_params) eventRenderer = EventRenderer( output_path=output_folder, dvs_vid=dvs_vid, preview=preview, full_scale_count=dvs_vid_full_scale, exposure_mode=exposure_mode, exposure_value=exposure_val, area_dimension=area_dimension) # timestamps of DVS start at zero and end with span of video we processed srcVideoRealProcessedDuration = (stop_time - start_time) / input_slowmotion_factor num_frames = srcNumFramesToBeProccessed inputHeight = None inputWidth = None inputChannels = None if start_frame > 0: logger.info('skipping to frame {}'.format(start_frame)) for i in tqdm(range(start_frame), unit='fr', desc='src'): ret, _ = cap.read() if not ret: raise ValueError( 'something wrong, got to end of file before ' 'reaching start_frame') logger.info( 'processing frames {} to {} from video input'.format( start_frame, stop_frame)) with TemporaryDirectory() as source_frames_dir: inputWidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) inputHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) inputChannels = 3 if (output_width is None) and (output_height is None): output_width = inputWidth output_height = inputHeight logger.warning( 'output size ({}x{}) was set automatically to ' 'input video size\n Are you sure you want this? ' 'It might be slow.\n Consider using\n ' ' --output_width=346 --output_height=260\n to match Davis346.' .format(output_width, output_height)) logger.info('Resizing input frames to output size ' '(with possible RGG to luma conversion)') for inputFrameIndex in tqdm( range(srcNumFramesToBeProccessed), desc='rgb2luma', unit='fr'): # read frame ret, inputVideoFrame = cap.read() if not ret or inputFrameIndex + start_frame > stop_frame: break if output_height and output_width and \ (inputHeight != output_height or inputWidth != output_width): dim = (output_width, output_height) (fx, fy) = (float(output_width) / inputWidth, float(output_height) / inputHeight) inputVideoFrame = cv2.resize( src=inputVideoFrame, dsize=dim, fx=fx, fy=fy, interpolation=cv2.INTER_AREA) if inputChannels == 3: # color if inputFrameIndex == 0: # print info once logger.info( '\nConverting input frames from RGB color to luma') # TODO would break resize if input is gray frames # convert RGB frame into luminance. inputVideoFrame = cv2.cvtColor( inputVideoFrame, cv2.COLOR_BGR2GRAY) # much faster # save frame into numpy records save_path = os.path.join( source_frames_dir, str(inputFrameIndex).zfill(8) + ".npy") np.save(save_path, inputVideoFrame) # print("Writing source frame {}".format(save_path), end="\r") cap.release() with TemporaryDirectory() as interpFramesFolder: interpTimes=None # make input to slomo if slowdown_factor != NO_SLOWDOWN: # interpolated frames are stored to tmpfolder as # 1.png, 2.png, etc interpTimes,avgUpsamplingFactor=slomo.interpolate( source_frames_dir, interpFramesFolder, (output_width, output_height)) avgTs = srcFrameIntervalS / avgUpsamplingFactor logger.info('SloMo average upsampling factor={:5.2f}; average DVS timestamp resolution={}s' .format(avgUpsamplingFactor, eng(avgTs))) # read back to memory interpFramesFilenames = all_images(interpFramesFolder) # number of frames n = len(interpFramesFilenames) else: logger.info('turning npy frame files to png from {}' .format(source_frames_dir)) interpFramesFilenames = [] n=0 src_files = sorted( glob.glob("{}".format(source_frames_dir) + "/*.npy")) for frame_idx, src_file_path in tqdm( enumerate(src_files), desc='npy2png', unit='fr'): src_frame = np.load(src_file_path) tgt_file_path = os.path.join( interpFramesFolder, str(frame_idx) + ".png") interpFramesFilenames.append(tgt_file_path) n+=1 cv2.imwrite(tgt_file_path, src_frame) interpTimes=np.array(range(n)) # compute times of output integrated frames nFrames=len(interpFramesFilenames) # interpTimes is in units of 1 per input frame, normalize it to src video time range f=srcVideoRealProcessedDuration/(np.max(interpTimes)-np.min(interpTimes)) interpTimes = f*interpTimes # compute actual times from video times # debug if slomo_stats_plot: from matplotlib import pyplot as plt # TODO debug dt = np.diff(interpTimes) fig=plt.figure() ax1=fig.add_subplot(111) ax1.set_title('Slo-Mo frame interval stats (close to continue)') ax1.plot(interpTimes) ax1.plot(interpTimes,'x') ax1.set_xlabel('frame') ax1.set_ylabel('frame time (s)') ax2=ax1.twinx() ax2.plot(dt*1e3) ax2.set_ylabel('frame interval (ms)') logger.info('close plot to continue') fig.show() events = np.zeros((0, 4), dtype=np.float32) # array to batch events for rendering to DVS frames with tqdm(total=nFrames, desc='dvs', unit='fr') as pbar: # instantiate progress bar for i in range(nFrames): fr = read_image(interpFramesFilenames[i]) newEvents = emulator.generate_events(fr, interpTimes[i]) pbar.update(1) if newEvents is not None and newEvents.shape[0] > 0: events = np.append(events, newEvents, axis=0) events = np.array(events) if i%batch_size==0: eventRenderer.render_events_to_frames(events, height=output_height, width=output_width) events = np.zeros((0, 4), dtype=np.float32) # clear array if len(events)>0: # process leftover eventRenderer.render_events_to_frames(events, height=output_height, width=output_width) eventRenderer.cleanup() emulator.cleanup() if slomo is not None: slomo.cleanup() if num_frames == 0: logger.error('no frames read from file') v2e_quit() totalTime = (time.time() - time_run_started) framePerS = num_frames / totalTime sPerFrame = 1 / framePerS throughputStr = (str(eng(framePerS)) + 'fr/s') \ if framePerS > 1 else (str(eng(sPerFrame)) + 's/fr') logger.info( 'done processing {} frames in {}s ({})\n see output folder {}' .format(num_frames, eng(totalTime), throughputStr, output_folder)) logger.info('generated total {} events ({} on, {} off)' .format(eng(emulator.num_events_total), eng(emulator.num_events_on), eng(emulator.num_events_off))) logger.info( 'avg event rate {}Hz ({}Hz on, {}Hz off)' .format( eng(emulator.num_events_total / srcDurationToBeProcessed), eng(emulator.num_events_on / srcDurationToBeProcessed), eng(emulator.num_events_off / srcDurationToBeProcessed))) try: desktop.open(os.path.abspath(output_folder)) except Exception as e: logger.warning( '{}: could not open {} in desktop'.format(e, output_folder))