class Renderer(object): def __init__( self, dst_path, vid_info, sequence, playback_rate, flow_function=settings['flow_function'], interpolate_function=settings['interpolate_function'], w=None, h=None, lossless=False, trim=False, show_preview=True, add_info=False, text_type=settings['text_type'], mark=False, mux=False): self.dst_path = dst_path self.vid_info = vid_info self.sequence = sequence self.playback_rate = float(playback_rate) self.flow_function = flow_function self.interpolate_function = interpolate_function self.w = w self.h = h self.lossless = lossless self.trim = trim self.show_preview = show_preview self.add_info = add_info self.text_type = text_type self.mark = mark self.mux = mux self.scaler = None self.pipe = None self.source = None self.tot_frs_wrt = 0 self.tot_tgt_frs = 0 self.tot_src_frs = 0 self.tot_frs_int = 0 self.tot_frs_dup = 0 self.tot_frs_drp = 0 self.subs_to_render = 0 self.curr_sub_idx = 0 def make_pipe(self, dst_path, rate): vf = [] vf.append('format=yuv420p') call = [ settings['avutil'], '-loglevel', settings['av_loglevel'], '-y', '-threads', '0', '-f', 'rawvideo', '-pix_fmt', 'bgr24', '-s', '{}x{}'.format(self.w, self.h), '-r', str(rate), '-i', '-', '-map_metadata', '-1', '-map_chapters', '-1', '-vf', ','.join(vf), '-r', str(rate), '-an', '-sn', '-c:v', settings['cv'], '-preset', settings['preset']] if settings['cv'] == 'libx264': quality = ['-crf', str(settings['crf'])] # `-qp 0` is recommended over `-crf` for lossless # See: https://trac.ffmpeg.org/wiki/Encode/H.264#LosslessH.264 if self.lossless: quality = ['-qp', '0'] call.extend(quality) call.extend(['-level', '4.2']) params = [] call.extend(['-{}-params'.format(settings['cv'].replace('lib', ''))]) params.append('log-level={}'.format(settings['enc_loglevel'])) if settings['cv'] == 'libx265': quality = 'crf={}'.format(settings['crf']) if self.lossless: # ffmpeg doesn't pass `-x265-params` to x265 correctly, must # provide keys for every single value until fixed # See: https://trac.ffmpeg.org/ticket/4284 quality = 'lossless=1' params.append(quality) if len(params) > 0: call.extend([':'.join(params)]) call.extend([dst_path]) self.pipe = subprocess.Popen( call, stdin=subprocess.PIPE ) if self.pipe == 1: raise RuntimeError('render failed') def close_pipe(self): # `flush()` does not necessarily write the file's data to disk. Use # `flush()` followed by `os.fsync()` to ensure this behavior if self.pipe is not None and not self.pipe.stdin.closed: self.pipe.stdin.flush() self.pipe.stdin.close() self.pipe.wait() def write_frame_to_pipe(self, frame): try: self.pipe.stdin.write(bytes(frame.data)) except Exception: log.error('Writing frame to pipe failed:', exc_info=True) def render_subregion(self, subregion): log.debug('Working on subregion: %s', self.curr_sub_idx + 1) fa = subregion.fa fb = subregion.fb ta = subregion.ta tb = subregion.tb reg_len = (fb - fa) + 1 # num of frames in the region reg_dur = (tb - ta) / 1000.0 # duration of subregion in seconds tgt_frs = 0 # num of frames we're targeting to render # only one of these needs to be set to calculate tgt_frames if subregion.dur: tgt_frs = int(self.playback_rate * (subregion.dur / 1000.0)) elif subregion.fps: tgt_frs = int(subregion.fps * reg_dur) elif subregion.spd: tgt_frs = int(self.playback_rate * reg_dur * (1 / subregion.spd)) tgt_frs = max(0, tgt_frs) # the make factor or inverse time step int_each_go = float(tgt_frs) / max(1, (reg_len - 1)) # stop a division by zero error when only a single frame needs to be # written if int_each_go == 0: tgt_frs = 1 self.tot_tgt_frs += tgt_frs # TODO: overcompensate for frames? # int_each_go = math.ceil(int_each_go) int_each_go = int(int_each_go) pairs = reg_len - 1 if pairs >= 1: will_make = (int_each_go * pairs) + pairs else: # no pairs available. will only add src frame to to_wrt will_make = 1 extra_frs = will_make - tgt_frs # frames will need to be dropped or duped based on how many # frames are expected to be generated. this includes source and # interpolated frames drp_every = 0 if extra_frs > 0: drp_every = will_make / math.fabs(extra_frs) dup_every = 0 if extra_frs < 0: dup_every = will_make / math.fabs(extra_frs) log.debug('fa: %s', fa) log.debug('fb: %s', fb) log.debug('ta: %s', ta) log.debug('tb: %s', tb) log.debug('reg_dur: %s', reg_dur * 1000.0) log.debug('reg_len: %s', reg_len) log.debug('tgt_fps: %s', subregion.fps) log.debug('tgt_dur: %s', subregion.dur) with np.errstate(divide='ignore', invalid='ignore'): s = subregion.spd if subregion.spd is None: s = 0 log.debug('tgt_spd: %s %.2gx', subregion.spd, np.divide(1, s)) log.debug('tgt_frs: %s', tgt_frs) sub_div = int_each_go + 1 ts = [] for x in range(int_each_go): y = max(0.0, min(1.0, (1.0 / sub_div) * (x + 1))) y = '{:.2f}'.format(y) ts.append(y) if len(ts) > 0: log.debug('ts: %s..%s', ts[0], ts[-1]) log.debug('int_each_go: %s', int_each_go) log.debug('wr_per_pair: %s', int_each_go + 1) # +1 because of `fr_1` log.debug('pairs: %s', pairs) log.debug('will_make: %s', will_make) log.debug('extra_frs: %s', extra_frs) log.debug('dup_every: %s', dup_every) log.debug('drp_every: %s', drp_every) est_dur = tgt_frs / self.playback_rate log.debug('est_dur: %s', est_dur) # audio may drift because of the change in which frames are rendered in # relation to the source video this is used for debugging: pot_aud_drift = extra_frs / self.playback_rate log.debug('pot_aud_drift: %s', pot_aud_drift) # keep track of progress in this subregion src_gen = 0 # num of source frames seen frs_int = 0 # num of frames interpolated frs_src_drp = 0 # num of source frames dropped frs_int_drp = 0 # num of interpolated frames dropped wrk_idx = 0 # idx in the subregion being worked on frs_wrt = 0 # num of frames written in this subregion frs_dup = 0 # num of frames duped frs_drp = 0 # num of frames dropped fin_run = False # is this the final run? frs_fin_dup = 0 # num of frames duped on the final run runs = 0 # num of runs through the loop fr_1 = None self.source.seek_to_frame(fa) # log.debug('seek: %s', self.source.idx) # seek pos of first frame # log.debug('read: %s', self.source.idx) # next frame to be read fr_2 = self.source.read() # first frame in the region # scale down now but wait after drawing on the frame before scaling up if self.scaler == settings['scaler_dn']: fr_2 = cv2.resize(fr_2, (self.w, self.h), interpolation=self.scaler) src_gen += 1 if fa == fb or tgt_frs == 1: # only 1 frame expected. run through the main loop once fin_run = True runs = 1 else: # at least one frame pair is available. num of runs is equal to the # the total number of frames in the region - 1. range will run # from [0,runs) self.source.seek_to_frame(fa + 1) # seek to the next frame # log.debug('seek: %s', self.source.idx) runs = reg_len log.debug('wrt_one: %s', fin_run) # only write 1 frame log.debug('runs: %s', runs) for run_idx in range(0, runs): # which frame in the video is being worked on pair_a = fa + run_idx pair_b = pair_a + 1 if run_idx + 1 < runs else pair_a # if working on the last frame, write it out because we cant # interpolate without a pair. if run_idx >= runs - 1: fin_run = True frs_to_wrt = [] # hold frames to be written fr_1 = fr_2 # reference to prev fr saves a seek & read if fin_run: frs_to_wrt.append((fr_1, 'source', 1)) else: # begin interpolating frames between pairs # the frame being read should always be valid otherwise break try: # log.debug('read: %s', self.source.idx) fr_2 = self.source.read() src_gen += 1 except Exception: log.error('Could not read frame:', exc_info=True) break if fr_2 is None: break elif self.scaler == settings['scaler_dn']: fr_2 = cv2.resize(fr_2, (self.w, self.h), interpolation=self.scaler) fr_1_gr = cv2.cvtColor(fr_1, cv2.COLOR_BGR2GRAY) fr_2_gr = cv2.cvtColor(fr_2, cv2.COLOR_BGR2GRAY) fuv = self.flow_function(fr_1_gr, fr_2_gr) buv = self.flow_function(fr_2_gr, fr_1_gr) if isinstance(fuv, np.ndarray): fu = fuv[:,:,0] fv = fuv[:,:,1] bu = buv[:,:,0] bv = buv[:,:,1] else: fu, fv = fuv bu, bv = buv fr_1_32 = np.float32(fr_1) * 1/255.0 fr_2_32 = np.float32(fr_2) * 1/255.0 will_wrt = True # frames will be written? # look ahead to see if frames will be dropped # compensate by lowering the num of frames to be interpolated cmp_int_each_go = int_each_go # compensated `int_each_go` w_drp = [] # frames that would be dropped tmp_wrk_idx = wrk_idx - 1 # zero indexed for x in range(1 + int_each_go): # 1 real + interpolated frame tmp_wrk_idx += 1 if drp_every > 0: if math.fmod(tmp_wrk_idx, drp_every) < 1.0: w_drp.append(x + 1) n_drp = len(w_drp) # warn if a src frame was going to be dropped log_msg = log.debug if 1 in w_drp: log_msg = log.warning # start compensating if n_drp > 0: # can compensate by reducing num of frames to be # interpolated since they are available if n_drp <= int_each_go: cmp_int_each_go -= n_drp else: # can't compensate using interpolated frames alone # will have to drop the source frame. nothing will be # written will_wrt = False # log_msg('w_drp: %3s,%3s,%2s %s,-%s', # pair_a, # pair_b, # ','.join([str(x) for x in w_drp]), # cmp_int_each_go, # n_drp) if not will_wrt: # nothing will be written this go wrk_idx += 1 # still have to increment the work index log.warning('will_wrt: %s', will_wrt) self.tot_frs_drp += 1 if will_wrt: int_frs = self.interpolate_function( fr_1_32, fr_2_32, fu, fv, bu, bv, cmp_int_each_go) if len(int_frs) != cmp_int_each_go: log.warning('unexpected frs interpolated: act=%s ' 'est=%s', len(int_frs), cmp_int_each_go) frs_int += len(int_frs) frs_to_wrt.append((fr_1, 'source', 0)) for x, fr in enumerate(int_frs): frs_to_wrt.append((fr, 'interpolated', x + 1)) for (fr, fr_type, btw_idx) in frs_to_wrt: wrk_idx += 1 wrts_needed = 1 # duping should never happen unless the subregion being worked # on only has one frame if dup_every > 0: if math.fmod(wrk_idx, dup_every) < 1.0: frs_dup += 1 wrts_needed = 2 log.warning('dup: %s,%s,%s 2x', pair_a, pair_b, btw_idx) if fin_run: wrts_needed = (tgt_frs - frs_wrt) frs_fin_dup = wrts_needed - 1 log.debug('fin_dup: %s,%s,%s wrts=%sx', pair_a, pair_b, btw_idx, wrts_needed) # final frame should be dropped if needed if drp_every > 0: if math.fmod(wrk_idx, drp_every) < 1.0: log.warning('drp last frame') self.tot_frs_drp += 1 continue for wrt_idx in range(wrts_needed): fr_to_write = fr frs_wrt += 1 if wrt_idx == 0: if fr_type == 'source': self.tot_src_frs += 1 else: self.tot_frs_int += 1 else: self.tot_frs_dup += 1 self.tot_frs_wrt += 1 if self.scaler == settings['scaler_up']: fr = cv2.resize(fr, (self.w, self.h), interpolation=self.scaler) if self.mark: draw.marker(fr, fr_type == 'interpolated') if self.add_info: if wrts_needed > 1: fr_to_write = fr.copy() draw.debug_text(fr_to_write, self.text_type, self.playback_rate, self.flow_function, self.tot_frs_wrt, pair_a, pair_b, btw_idx, fr_type, wrt_idx > 0, tgt_frs, frs_wrt, subregion, self.curr_sub_idx, self.subs_to_render, drp_every, dup_every, src_gen, frs_int, frs_drp, frs_dup) if self.show_preview: fr_to_show = fr.copy() draw.progress_bar(fr_to_show, float(frs_wrt) / tgt_frs) cv2.imshow(self.window_title, np.asarray(fr_to_show)) # every imshow call should be followed by waitKey to # display the image for x milliseconds, otherwise it # won't display the image cv2.waitKey(settings['imshow_ms']) self.write_frame_to_pipe(fr_to_write) # log.debug('wrt: %s,%s,%s (%s)', pair_a, pair_b, btw_idx, # self.tot_frs_wrt) # finished encoding act_aud_drift = float(tgt_frs - frs_wrt) / self.playback_rate log.debug('act_aud_drift: %s', act_aud_drift) log.debug('src_gen: %s', src_gen) log.debug('frs_int: %s', frs_int) log.debug('frs_drp: %s', frs_drp) with np.errstate(divide='ignore', invalid='ignore'): log.debug('frs_src_drp: %s %.2f', frs_src_drp, np.divide(float(frs_src_drp), frs_drp)) log.debug('frs_int_drp: %s %.2f', frs_int_drp, np.divide(float(frs_int_drp), frs_drp)) # 1 - (frames dropped : real and interpolated frames) efficiency = 1 - (frs_drp * 1.0 / (src_gen + frs_int)) log.debug('efficiency: %.2f%%', efficiency * 100.0) log.debug('frs_dup: %s', frs_dup,) log.debug('frs_fin_dup: %s', frs_fin_dup) act_dur = frs_wrt / self.playback_rate log.debug('act_dur: %s', act_dur) if not np.isclose(act_dur, est_dur, rtol=1e-03): log.warning('unexpected dur: est_dur=%s act_dur=%s', est_dur, act_dur) if tgt_frs == 0: wrt_ratio = 0 else: wrt_ratio = float(frs_wrt) / tgt_frs log_msg = log.debug if frs_wrt != tgt_frs: log_msg = log.warning log_msg('wrt_ratio: {}/{}, {:.2f}%'.format( frs_wrt, tgt_frs, wrt_ratio * 100)) def get_renderable_sequence(self): # this method will fill holes in the sequence with dummy subregions dur = self.vid_info['duration'] frs = self.vid_info['frames'] new_subregions = [] if self.sequence.subregions is None or \ len(self.sequence.subregions) == 0: # make a subregion from 0 to vid duration if there are no regions # in the video sequence. only the framerate could be changing fa, ta = (0, 0) fb, tb = (frs - 1, dur) s = RenderSubregion(ta, tb) s.fa = fa s.fb = fb s.fps = self.playback_rate s.dur = tb - ta s.spd = 1.0 setattr(s, 'trim', False) new_subregions.append(s) else: # create placeholder/dummy subregions that fill holes in the video # sequence where subregions were not explicity specified cut_points = set([]) # add start and end of video cutting points # (fr index, dur in milliseconds) cut_points.add((0, 0)) # frame 0 and time 0 cut_points.add((frs - 1, dur)) # last frame and end time # add current subregions for s in self.sequence.subregions: cut_points.add((s.fa, s.ta)) cut_points.add((s.fb, s.tb)) # sort them out cut_points = list(cut_points) cut_points = sorted(cut_points, key=lambda x: (x[0], x[1]), reverse=False) # make dummy regions to_make = len(cut_points) - 1 for x in range(0, to_make): fa, ta = cut_points[x] # get start of region fb, tb = cut_points[x + 1] # get end sub_for_range = None # matching subregion in range # look for matching subregion for s in self.sequence.subregions: if s.fa == fa and s.fb == fb: sub_for_range = s setattr(s, 'trim', False) # found it, won't trim it break # if subregion isnt found, make a dummy region if sub_for_range is None: s = RenderSubregion(ta, tb) s.fa = fa s.fb = fb s.fps = self.playback_rate s.dur = tb - ta s.spd = 1.0 sub_for_range = s setattr(s, 'trim', self.trim) new_subregions.append(sub_for_range) # create a new video sequence w/ original plus dummy regions # they will automatically be validated and sorted as they are added in seq = VideoSequence(dur, frs) for s in new_subregions: seq.add_subregion(s) return seq def render(self): src_path = self.vid_info['path'] src_name = os.path.splitext(os.path.basename(src_path))[0] tmp_name = '~{filename}.{ext}'.format(filename=src_name, ext=settings['v_container']).lower() tmp_path = os.path.join(settings['tmp_dir'], tmp_name) if self.show_preview: # to get opengl on osx you have to build opencv --with-opengl # TODO: butterflow.rb and wiki needs to be updated for this self.window_title = '{} - Butterflow'.format( os.path.basename(src_path)) flag = cv2.WINDOW_OPENGL cv2.namedWindow(self.window_title, flag) cv2.resizeWindow(self.window_title, self.w, self.h) self.make_pipe(tmp_path, self.playback_rate) self.source = FrameSource(src_path) self.source.open() renderable_seq = self.get_renderable_sequence() log.debug('Rendering sequence:') for s in renderable_seq.subregions: ra = renderable_seq.relative_position(s.ta) rb = renderable_seq.relative_position(s.tb) log.debug( 'subregion: {},{},{} {:.3g},{:.3g},{:.3g} {:.3g},{:.3g},{:.3g}'. format(s.fa, s.fb, (s.fb - s.fa + 1), s.ta / 1000.0, s.tb / 1000.0, (s.tb - s.ta) / 1000.0, ra, rb, rb - ra)) new_res = self.w * self.h src_res = self.vid_info['w'] * self.vid_info['h'] if new_res == src_res: self.scaler = None elif new_res < src_res: self.scaler = settings['scaler_dn'] else: self.scaler = settings['scaler_up'] self.subs_to_render = 0 for s in renderable_seq.subregions: if not s.trim: self.subs_to_render += 1 self.curr_sub_idx = 0 for x, s in enumerate(renderable_seq.subregions): if s.trim: # the region is being trimmed and shouldn't be rendered continue else: self.curr_sub_idx += 1 self.render_subregion(s) self.source.close() if self.show_preview: cv2.destroyAllWindows() self.close_pipe() if self.mux: log.debug('muxing ...') aud_files = [] for x, s in enumerate(renderable_seq.subregions): if s.trim: continue tmp_name = '~{filename}.{sub}.{ext}'.format( filename=src_name, sub=x, ext=settings['a_container']).lower() aud_path = os.path.join(settings['tmp_dir'], tmp_name) extract_audio(src_path, aud_path, s.ta, s.tb, s.spd) aud_files.append(aud_path) merged_audio = '~{filename}.merged.{ext}'.format( filename=src_name, ext=settings['a_container'] ).lower() merged_audio = os.path.join(settings['tmp_dir'], merged_audio) concat_files(merged_audio, aud_files) mux(tmp_path, merged_audio, self.dst_path) for f in aud_files: os.remove(f) os.remove(merged_audio) os.remove(tmp_path) else: shutil.move(tmp_path, self.dst_path) def __del__(self): # close the pipe if it was inadvertently left open. this could happen # if the user does ctr+c while rendering. this would leave temporary # files in the cache self.close_pipe()
class Renderer(object): def __init__(self, dst_path, vid_info, sequence, playback_rate, flow_function=settings['flow_function'], interpolate_function=settings['interpolate_function'], w=None, h=None, lossless=False, trim=False, show_preview=True, add_info=False, text_type=settings['text_type'], mark=False, mux=False): self.dst_path = dst_path self.vid_info = vid_info self.sequence = sequence self.playback_rate = float(playback_rate) self.flow_function = flow_function self.interpolate_function = interpolate_function self.w = w self.h = h self.lossless = lossless self.trim = trim self.show_preview = show_preview self.add_info = add_info self.text_type = text_type self.mark = mark self.mux = mux self.scaler = None self.pipe = None self.source = None self.tot_frs_wrt = 0 self.tot_tgt_frs = 0 self.tot_src_frs = 0 self.tot_frs_int = 0 self.tot_frs_dup = 0 self.tot_frs_drp = 0 self.subs_to_render = 0 self.curr_sub_idx = 0 def make_pipe(self, dst_path, rate): vf = [] vf.append('format=yuv420p') call = [ settings['avutil'], '-loglevel', settings['av_loglevel'], '-y', '-threads', '0', '-f', 'rawvideo', '-pix_fmt', 'bgr24', '-s', '{}x{}'.format(self.w, self.h), '-r', str(rate), '-i', '-', '-map_metadata', '-1', '-map_chapters', '-1', '-vf', ','.join(vf), '-r', str(rate), '-an', '-sn', '-c:v', settings['cv'], '-preset', settings['preset'] ] if settings['cv'] == 'libx264': quality = ['-crf', str(settings['crf'])] # `-qp 0` is recommended over `-crf` for lossless # See: https://trac.ffmpeg.org/wiki/Encode/H.264#LosslessH.264 if self.lossless: quality = ['-qp', '0'] call.extend(quality) call.extend(['-level', '4.2']) params = [] call.extend(['-{}-params'.format(settings['cv'].replace('lib', ''))]) params.append('log-level={}'.format(settings['enc_loglevel'])) if settings['cv'] == 'libx265': quality = 'crf={}'.format(settings['crf']) if self.lossless: # ffmpeg doesn't pass `-x265-params` to x265 correctly, must # provide keys for every single value until fixed # See: https://trac.ffmpeg.org/ticket/4284 quality = 'lossless=1' params.append(quality) if len(params) > 0: call.extend([':'.join(params)]) call.extend([dst_path]) self.pipe = subprocess.Popen(call, stdin=subprocess.PIPE) if self.pipe == 1: raise RuntimeError('render failed') def close_pipe(self): # `flush()` does not necessarily write the file's data to disk. Use # `flush()` followed by `os.fsync()` to ensure this behavior if self.pipe is not None and not self.pipe.stdin.closed: self.pipe.stdin.flush() self.pipe.stdin.close() self.pipe.wait() def write_frame_to_pipe(self, frame): try: self.pipe.stdin.write(bytes(frame.data)) except Exception: log.error('Writing frame to pipe failed:', exc_info=True) def render_subregion(self, subregion): log.debug('Working on subregion: %s', self.curr_sub_idx + 1) fa = subregion.fa fb = subregion.fb ta = subregion.ta tb = subregion.tb reg_len = (fb - fa) + 1 # num of frames in the region reg_dur = (tb - ta) / 1000.0 # duration of subregion in seconds tgt_frs = 0 # num of frames we're targeting to render # only one of these needs to be set to calculate tgt_frames if subregion.dur: tgt_frs = int(self.playback_rate * (subregion.dur / 1000.0)) elif subregion.fps: tgt_frs = int(subregion.fps * reg_dur) elif subregion.spd: tgt_frs = int(self.playback_rate * reg_dur * (1 / subregion.spd)) tgt_frs = max(0, tgt_frs) # the make factor or inverse time step int_each_go = float(tgt_frs) / max(1, (reg_len - 1)) # stop a division by zero error when only a single frame needs to be # written if int_each_go == 0: tgt_frs = 1 self.tot_tgt_frs += tgt_frs # TODO: overcompensate for frames? # int_each_go = math.ceil(int_each_go) int_each_go = int(int_each_go) pairs = reg_len - 1 if pairs >= 1: will_make = (int_each_go * pairs) + pairs else: # no pairs available. will only add src frame to to_wrt will_make = 1 extra_frs = will_make - tgt_frs # frames will need to be dropped or duped based on how many # frames are expected to be generated. this includes source and # interpolated frames drp_every = 0 if extra_frs > 0: drp_every = will_make / math.fabs(extra_frs) dup_every = 0 if extra_frs < 0: dup_every = will_make / math.fabs(extra_frs) log.debug('fa: %s', fa) log.debug('fb: %s', fb) log.debug('ta: %s', ta) log.debug('tb: %s', tb) log.debug('reg_dur: %s', reg_dur * 1000.0) log.debug('reg_len: %s', reg_len) log.debug('tgt_fps: %s', subregion.fps) log.debug('tgt_dur: %s', subregion.dur) with np.errstate(divide='ignore', invalid='ignore'): s = subregion.spd if subregion.spd is None: s = 0 log.debug('tgt_spd: %s %.2gx', subregion.spd, np.divide(1, s)) log.debug('tgt_frs: %s', tgt_frs) sub_div = int_each_go + 1 ts = [] for x in range(int_each_go): y = max(0.0, min(1.0, (1.0 / sub_div) * (x + 1))) y = '{:.2f}'.format(y) ts.append(y) if len(ts) > 0: log.debug('ts: %s..%s', ts[0], ts[-1]) log.debug('int_each_go: %s', int_each_go) log.debug('wr_per_pair: %s', int_each_go + 1) # +1 because of `fr_1` log.debug('pairs: %s', pairs) log.debug('will_make: %s', will_make) log.debug('extra_frs: %s', extra_frs) log.debug('dup_every: %s', dup_every) log.debug('drp_every: %s', drp_every) est_dur = tgt_frs / self.playback_rate log.debug('est_dur: %s', est_dur) # audio may drift because of the change in which frames are rendered in # relation to the source video this is used for debugging: pot_aud_drift = extra_frs / self.playback_rate log.debug('pot_aud_drift: %s', pot_aud_drift) # keep track of progress in this subregion src_gen = 0 # num of source frames seen frs_int = 0 # num of frames interpolated frs_src_drp = 0 # num of source frames dropped frs_int_drp = 0 # num of interpolated frames dropped wrk_idx = 0 # idx in the subregion being worked on frs_wrt = 0 # num of frames written in this subregion frs_dup = 0 # num of frames duped frs_drp = 0 # num of frames dropped fin_run = False # is this the final run? frs_fin_dup = 0 # num of frames duped on the final run runs = 0 # num of runs through the loop fr_1 = None self.source.seek_to_frame(fa) # log.debug('seek: %s', self.source.idx) # seek pos of first frame # log.debug('read: %s', self.source.idx) # next frame to be read fr_2 = self.source.read() # first frame in the region # scale down now but wait after drawing on the frame before scaling up if self.scaler == settings['scaler_dn']: fr_2 = cv2.resize(fr_2, (self.w, self.h), interpolation=self.scaler) src_gen += 1 if fa == fb or tgt_frs == 1: # only 1 frame expected. run through the main loop once fin_run = True runs = 1 else: # at least one frame pair is available. num of runs is equal to the # the total number of frames in the region - 1. range will run # from [0,runs) self.source.seek_to_frame(fa + 1) # seek to the next frame # log.debug('seek: %s', self.source.idx) runs = reg_len log.debug('wrt_one: %s', fin_run) # only write 1 frame log.debug('runs: %s', runs) for run_idx in range(0, runs): # which frame in the video is being worked on pair_a = fa + run_idx pair_b = pair_a + 1 if run_idx + 1 < runs else pair_a # if working on the last frame, write it out because we cant # interpolate without a pair. if run_idx >= runs - 1: fin_run = True frs_to_wrt = [] # hold frames to be written fr_1 = fr_2 # reference to prev fr saves a seek & read if fin_run: frs_to_wrt.append((fr_1, 'source', 1)) else: # begin interpolating frames between pairs # the frame being read should always be valid otherwise break try: # log.debug('read: %s', self.source.idx) fr_2 = self.source.read() src_gen += 1 except Exception: log.error('Could not read frame:', exc_info=True) break if fr_2 is None: break elif self.scaler == settings['scaler_dn']: fr_2 = cv2.resize(fr_2, (self.w, self.h), interpolation=self.scaler) fr_1_gr = cv2.cvtColor(fr_1, cv2.COLOR_BGR2GRAY) fr_2_gr = cv2.cvtColor(fr_2, cv2.COLOR_BGR2GRAY) fuv = self.flow_function(fr_1_gr, fr_2_gr) buv = self.flow_function(fr_2_gr, fr_1_gr) if isinstance(fuv, np.ndarray): fu = fuv[:, :, 0] fv = fuv[:, :, 1] bu = buv[:, :, 0] bv = buv[:, :, 1] else: fu, fv = fuv bu, bv = buv fr_1_32 = np.float32(fr_1) * 1 / 255.0 fr_2_32 = np.float32(fr_2) * 1 / 255.0 will_wrt = True # frames will be written? # look ahead to see if frames will be dropped # compensate by lowering the num of frames to be interpolated cmp_int_each_go = int_each_go # compensated `int_each_go` w_drp = [] # frames that would be dropped tmp_wrk_idx = wrk_idx - 1 # zero indexed for x in range(1 + int_each_go): # 1 real + interpolated frame tmp_wrk_idx += 1 if drp_every > 0: if math.fmod(tmp_wrk_idx, drp_every) < 1.0: w_drp.append(x + 1) n_drp = len(w_drp) # warn if a src frame was going to be dropped log_msg = log.debug if 1 in w_drp: log_msg = log.warning # start compensating if n_drp > 0: # can compensate by reducing num of frames to be # interpolated since they are available if n_drp <= int_each_go: cmp_int_each_go -= n_drp else: # can't compensate using interpolated frames alone # will have to drop the source frame. nothing will be # written will_wrt = False # log_msg('w_drp: %3s,%3s,%2s %s,-%s', # pair_a, # pair_b, # ','.join([str(x) for x in w_drp]), # cmp_int_each_go, # n_drp) if not will_wrt: # nothing will be written this go wrk_idx += 1 # still have to increment the work index log.warning('will_wrt: %s', will_wrt) self.tot_frs_drp += 1 if will_wrt: int_frs = self.interpolate_function( fr_1_32, fr_2_32, fu, fv, bu, bv, cmp_int_each_go) if len(int_frs) != cmp_int_each_go: log.warning( 'unexpected frs interpolated: act=%s ' 'est=%s', len(int_frs), cmp_int_each_go) frs_int += len(int_frs) frs_to_wrt.append((fr_1, 'source', 0)) for x, fr in enumerate(int_frs): frs_to_wrt.append((fr, 'interpolated', x + 1)) for (fr, fr_type, btw_idx) in frs_to_wrt: wrk_idx += 1 wrts_needed = 1 # duping should never happen unless the subregion being worked # on only has one frame if dup_every > 0: if math.fmod(wrk_idx, dup_every) < 1.0: frs_dup += 1 wrts_needed = 2 log.warning('dup: %s,%s,%s 2x', pair_a, pair_b, btw_idx) if fin_run: wrts_needed = (tgt_frs - frs_wrt) frs_fin_dup = wrts_needed - 1 log.debug('fin_dup: %s,%s,%s wrts=%sx', pair_a, pair_b, btw_idx, wrts_needed) # final frame should be dropped if needed if drp_every > 0: if math.fmod(wrk_idx, drp_every) < 1.0: log.warning('drp last frame') self.tot_frs_drp += 1 continue for wrt_idx in range(wrts_needed): fr_to_write = fr frs_wrt += 1 if wrt_idx == 0: if fr_type == 'source': self.tot_src_frs += 1 else: self.tot_frs_int += 1 else: self.tot_frs_dup += 1 self.tot_frs_wrt += 1 if self.scaler == settings['scaler_up']: fr = cv2.resize(fr, (self.w, self.h), interpolation=self.scaler) if self.mark: draw.marker(fr, fr_type == 'interpolated') if self.add_info: if wrts_needed > 1: fr_to_write = fr.copy() draw.debug_text(fr_to_write, self.text_type, self.playback_rate, self.flow_function, self.tot_frs_wrt, pair_a, pair_b, btw_idx, fr_type, wrt_idx > 0, tgt_frs, frs_wrt, subregion, self.curr_sub_idx, self.subs_to_render, drp_every, dup_every, src_gen, frs_int, frs_drp, frs_dup) if self.show_preview: fr_to_show = fr.copy() draw.progress_bar(fr_to_show, float(frs_wrt) / tgt_frs) cv2.imshow(self.window_title, np.asarray(fr_to_show)) # every imshow call should be followed by waitKey to # display the image for x milliseconds, otherwise it # won't display the image cv2.waitKey(settings['imshow_ms']) self.write_frame_to_pipe(fr_to_write) # log.debug('wrt: %s,%s,%s (%s)', pair_a, pair_b, btw_idx, # self.tot_frs_wrt) # finished encoding act_aud_drift = float(tgt_frs - frs_wrt) / self.playback_rate log.debug('act_aud_drift: %s', act_aud_drift) log.debug('src_gen: %s', src_gen) log.debug('frs_int: %s', frs_int) log.debug('frs_drp: %s', frs_drp) with np.errstate(divide='ignore', invalid='ignore'): log.debug('frs_src_drp: %s %.2f', frs_src_drp, np.divide(float(frs_src_drp), frs_drp)) log.debug('frs_int_drp: %s %.2f', frs_int_drp, np.divide(float(frs_int_drp), frs_drp)) # 1 - (frames dropped : real and interpolated frames) efficiency = 1 - (frs_drp * 1.0 / (src_gen + frs_int)) log.debug('efficiency: %.2f%%', efficiency * 100.0) log.debug( 'frs_dup: %s', frs_dup, ) log.debug('frs_fin_dup: %s', frs_fin_dup) act_dur = frs_wrt / self.playback_rate log.debug('act_dur: %s', act_dur) if not np.isclose(act_dur, est_dur, rtol=1e-03): log.warning('unexpected dur: est_dur=%s act_dur=%s', est_dur, act_dur) if tgt_frs == 0: wrt_ratio = 0 else: wrt_ratio = float(frs_wrt) / tgt_frs log_msg = log.debug if frs_wrt != tgt_frs: log_msg = log.warning log_msg('wrt_ratio: {}/{}, {:.2f}%'.format(frs_wrt, tgt_frs, wrt_ratio * 100)) def get_renderable_sequence(self): # this method will fill holes in the sequence with dummy subregions dur = self.vid_info['duration'] frs = self.vid_info['frames'] new_subregions = [] if self.sequence.subregions is None or \ len(self.sequence.subregions) == 0: # make a subregion from 0 to vid duration if there are no regions # in the video sequence. only the framerate could be changing fa, ta = (0, 0) fb, tb = (frs - 1, dur) s = RenderSubregion(ta, tb) s.fa = fa s.fb = fb s.fps = self.playback_rate s.dur = tb - ta s.spd = 1.0 setattr(s, 'trim', False) new_subregions.append(s) else: # create placeholder/dummy subregions that fill holes in the video # sequence where subregions were not explicity specified cut_points = set([]) # add start and end of video cutting points # (fr index, dur in milliseconds) cut_points.add((0, 0)) # frame 0 and time 0 cut_points.add((frs - 1, dur)) # last frame and end time # add current subregions for s in self.sequence.subregions: cut_points.add((s.fa, s.ta)) cut_points.add((s.fb, s.tb)) # sort them out cut_points = list(cut_points) cut_points = sorted(cut_points, key=lambda x: (x[0], x[1]), reverse=False) # make dummy regions to_make = len(cut_points) - 1 for x in range(0, to_make): fa, ta = cut_points[x] # get start of region fb, tb = cut_points[x + 1] # get end sub_for_range = None # matching subregion in range # look for matching subregion for s in self.sequence.subregions: if s.fa == fa and s.fb == fb: sub_for_range = s setattr(s, 'trim', False) # found it, won't trim it break # if subregion isnt found, make a dummy region if sub_for_range is None: s = RenderSubregion(ta, tb) s.fa = fa s.fb = fb s.fps = self.playback_rate s.dur = tb - ta s.spd = 1.0 sub_for_range = s setattr(s, 'trim', self.trim) new_subregions.append(sub_for_range) # create a new video sequence w/ original plus dummy regions # they will automatically be validated and sorted as they are added in seq = VideoSequence(dur, frs) for s in new_subregions: seq.add_subregion(s) return seq def render(self): src_path = self.vid_info['path'] src_name = os.path.splitext(os.path.basename(src_path))[0] tmp_name = '~{filename}.{ext}'.format( filename=src_name, ext=settings['v_container']).lower() tmp_path = os.path.join(settings['tmp_dir'], tmp_name) if self.show_preview: # to get opengl on osx you have to build opencv --with-opengl # TODO: butterflow.rb and wiki needs to be updated for this self.window_title = '{} - Butterflow'.format( os.path.basename(src_path)) flag = cv2.WINDOW_OPENGL cv2.namedWindow(self.window_title, flag) cv2.resizeWindow(self.window_title, self.w, self.h) self.make_pipe(tmp_path, self.playback_rate) self.source = FrameSource(src_path) self.source.open() renderable_seq = self.get_renderable_sequence() log.debug('Rendering sequence:') for s in renderable_seq.subregions: ra = renderable_seq.relative_position(s.ta) rb = renderable_seq.relative_position(s.tb) log.debug( 'subregion: {},{},{} {:.3g},{:.3g},{:.3g} {:.3g},{:.3g},{:.3g}' .format(s.fa, s.fb, (s.fb - s.fa + 1), s.ta / 1000.0, s.tb / 1000.0, (s.tb - s.ta) / 1000.0, ra, rb, rb - ra)) new_res = self.w * self.h src_res = self.vid_info['w'] * self.vid_info['h'] if new_res == src_res: self.scaler = None elif new_res < src_res: self.scaler = settings['scaler_dn'] else: self.scaler = settings['scaler_up'] self.subs_to_render = 0 for s in renderable_seq.subregions: if not s.trim: self.subs_to_render += 1 self.curr_sub_idx = 0 for x, s in enumerate(renderable_seq.subregions): if s.trim: # the region is being trimmed and shouldn't be rendered continue else: self.curr_sub_idx += 1 self.render_subregion(s) self.source.close() if self.show_preview: cv2.destroyAllWindows() self.close_pipe() if self.mux: log.debug('muxing ...') aud_files = [] for x, s in enumerate(renderable_seq.subregions): if s.trim: continue tmp_name = '~{filename}.{sub}.{ext}'.format( filename=src_name, sub=x, ext=settings['a_container']).lower() aud_path = os.path.join(settings['tmp_dir'], tmp_name) extract_audio(src_path, aud_path, s.ta, s.tb, s.spd) aud_files.append(aud_path) merged_audio = '~{filename}.merged.{ext}'.format( filename=src_name, ext=settings['a_container']).lower() merged_audio = os.path.join(settings['tmp_dir'], merged_audio) concat_files(merged_audio, aud_files) mux(tmp_path, merged_audio, self.dst_path) for f in aud_files: os.remove(f) os.remove(merged_audio) os.remove(tmp_path) else: shutil.move(tmp_path, self.dst_path) def __del__(self): # close the pipe if it was inadvertently left open. this could happen # if the user does ctr+c while rendering. this would leave temporary # files in the cache self.close_pipe()
class FrameSourceTestCase(unittest.TestCase): def setUp(self): self.videofile_fr1 = os.path.join( settings['tmp_dir'], '~test_av_frame_source_test_case_fr_1.mp4') self.videofile_fr3 = os.path.join( settings['tmp_dir'], '~test_av_frame_source_test_case_fr_3.mp4') # make a 1fr and 3fr video mk_sample_video(self.videofile_fr1, 1, 320, 240, fractions.Fraction(1)) mk_sample_video(self.videofile_fr3, 1, 320, 240, fractions.Fraction(3)) # path to avutil fr to compare with self.imagefile = os.path.join(settings['tmp_dir'], '~test_av_frame_source_test_case.png') self.src_1 = FrameSource(self.videofile_fr1) self.src_3 = FrameSource(self.videofile_fr3) self.src_3.open() def tearDown(self): self.src_3.close() def test_seek_to_frame_initial_index_zero(self): self.assertEqual(self.src_3.idx, 0) def test_seek_to_frame_inside(self): self.src_3.seek_to_frame(1) self.assertEqual(self.src_3.idx, 1) def test_seek_to_frame_inside_same_frame_back_to_back(self): self.assertEqual(self.src_3.idx, 0) self.src_3.seek_to_frame(1) self.assertEqual(self.src_3.idx, 1) self.src_3.seek_to_frame(1) self.assertEqual(self.src_3.idx, 1) def test_seek_to_frame_at_edges(self): self.src_3.seek_to_frame(0) self.assertEqual(self.src_3.idx, 0) self.src_3.seek_to_frame(2) self.assertEqual(self.src_3.idx, 2) def test_seek_to_frame_outside_fails(self): with self.assertRaises(IndexError): self.src_3.seek_to_frame(-1) with self.assertRaises(IndexError): self.src_3.seek_to_frame(3) def test_read_frame_after_seek_to_frame_inside(self): self.src_3.seek_to_frame(1) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 1) self.assertTrue(np.array_equal(f1, f2)) def test_read_frame_after_seek_to_frame_at_edges(self): self.src_3.seek_to_frame(0) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 0) self.assertTrue(np.array_equal(f1, f2)) self.src_3.seek_to_frame(2) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 2) self.assertTrue(np.array_equal(f1, f2)) def test_seek_forward_then_backward(self): self.src_3.seek_to_frame(2) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 2) self.assertTrue(np.array_equal(f1, f2)) self.src_3.seek_to_frame(1) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 1) self.assertTrue(np.array_equal(f1, f2)) self.src_3.seek_to_frame(0) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 0) self.assertTrue(np.array_equal(f1, f2)) def test_n_frames_same_as_avinfo(self): av = avinfo.get_av_info(self.src_3.path) self.assertEqual(self.src_3.frames, av['frames']) def test_open_close(self): self.assertIsNone(self.src_1.src) self.src_1.open() self.assertIsNotNone(self.src_1.src) self.src_1.close() self.assertIsNone(self.src_1.src) self.src_1.open() self.assertIsNotNone(self.src_1.src) self.src_1.close() def test_open_twice(self): self.assertIsNone(self.src_1.src) self.src_1.open() self.src_1.open() self.assertIsNotNone(self.src_1.src) self.src_1.close() def test_close_twice(self): self.assertIsNone(self.src_1.src) self.src_1.open() self.src_1.close() self.assertIsNone(self.src_1.src) self.src_1.close() self.assertIsNone(self.src_1.src)
class FrameSourceTestCase(unittest.TestCase): def setUp(self): self.videofile_fr1 = os.path.join(settings['tmp_dir'], '~test_av_frame_source_test_case_fr_1.mp4') self.videofile_fr3 = os.path.join(settings['tmp_dir'], '~test_av_frame_source_test_case_fr_3.mp4') # make a 1fr and 3fr video mk_sample_video(self.videofile_fr1, 1, 320, 240, fractions.Fraction(1)) mk_sample_video(self.videofile_fr3, 1, 320, 240, fractions.Fraction(3)) # path to avutil fr to compare with self.imagefile = os.path.join(settings['tmp_dir'], '~test_av_frame_source_test_case.png') self.src_1 = FrameSource(self.videofile_fr1) self.src_3 = FrameSource(self.videofile_fr3) self.src_3.open() def tearDown(self): self.src_3.close() def test_seek_to_frame_initial_index_zero(self): self.assertEqual(self.src_3.idx, 0) def test_seek_to_frame_inside(self): self.src_3.seek_to_frame(1) self.assertEqual(self.src_3.idx, 1) def test_seek_to_frame_inside_same_frame_back_to_back(self): self.assertEqual(self.src_3.idx, 0) self.src_3.seek_to_frame(1) self.assertEqual(self.src_3.idx, 1) self.src_3.seek_to_frame(1) self.assertEqual(self.src_3.idx, 1) def test_seek_to_frame_at_edges(self): self.src_3.seek_to_frame(0) self.assertEqual(self.src_3.idx, 0) self.src_3.seek_to_frame(2) self.assertEqual(self.src_3.idx, 2) def test_seek_to_frame_outside_fails(self): with self.assertRaises(IndexError): self.src_3.seek_to_frame(-1) with self.assertRaises(IndexError): self.src_3.seek_to_frame(3) def test_read_frame_after_seek_to_frame_inside(self): self.src_3.seek_to_frame(1) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 1) self.assertTrue(np.array_equal(f1,f2)) def test_read_frame_after_seek_to_frame_at_edges(self): self.src_3.seek_to_frame(0) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 0) self.assertTrue(np.array_equal(f1,f2)) self.src_3.seek_to_frame(2) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 2) self.assertTrue(np.array_equal(f1,f2)) def test_seek_forward_then_backward(self): self.src_3.seek_to_frame(2) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 2) self.assertTrue(np.array_equal(f1,f2)) self.src_3.seek_to_frame(1) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 1) self.assertTrue(np.array_equal(f1,f2)) self.src_3.seek_to_frame(0) f1 = self.src_3.read() f2 = av_frame_at_idx(self.src_3.path, self.imagefile, 0) self.assertTrue(np.array_equal(f1,f2)) def test_n_frames_same_as_avinfo(self): av = avinfo.get_av_info(self.src_3.path) self.assertEqual(self.src_3.frames, av['frames']) def test_open_close(self): self.assertIsNone(self.src_1.src) self.src_1.open() self.assertIsNotNone(self.src_1.src) self.src_1.close() self.assertIsNone(self.src_1.src) self.src_1.open() self.assertIsNotNone(self.src_1.src) self.src_1.close() def test_open_twice(self): self.assertIsNone(self.src_1.src) self.src_1.open() self.src_1.open() self.assertIsNotNone(self.src_1.src) self.src_1.close() def test_close_twice(self): self.assertIsNone(self.src_1.src) self.src_1.open() self.src_1.close() self.assertIsNone(self.src_1.src) self.src_1.close() self.assertIsNone(self.src_1.src)