def thumbnail( self, target_range: VideoCutRange, to_dir: str = None, compress_rate: float = None, is_vertical: bool = None, *_, **__, ) -> np.ndarray: """ build a thumbnail, for easier debug or something else :param target_range: VideoCutRange :param to_dir: your thumbnail will be saved to this path :param compress_rate: float, 0 - 1, about thumbnail's size, default to 0.1 (1/10) :param is_vertical: direction :return: """ if not compress_rate: compress_rate = 0.1 # direction if is_vertical: stack_func = np.vstack def get_split_line(f): return np.zeros((5, f.shape[1])) else: stack_func = np.hstack def get_split_line(f): return np.zeros((f.shape[0], 5)) frame_list = list() with toolbox.video_capture(self.video.path) as cap: toolbox.video_jump(cap, target_range.start) ret, frame = cap.read() count = 1 length = target_range.get_length() while ret and count <= length: frame = toolbox.compress_frame(frame, compress_rate) frame_list.append(frame) frame_list.append(get_split_line(frame)) ret, frame = cap.read() count += 1 merged = stack_func(frame_list) # create parent dir if to_dir: target_path = os.path.join( to_dir, f"thumbnail_{target_range.start}-{target_range.end}.png" ) cv2.imwrite(target_path, merged) logger.debug(f"save thumbnail to {target_path}") return merged
def get_range( self, limit: int = None, unstable_limit: int = None, **kwargs ) -> typing.Tuple[typing.List[VideoCutRange], typing.List[VideoCutRange]]: """ return stable_range_list and unstable_range_list :param limit: ignore some ranges which are too short, 5 means ignore stable ranges which length < 5 :param unstable_limit: ignore some ranges which are too short, 5 means ignore unstable ranges which length < 5 :param kwargs: threshold: float, 0-1, default to 0.95. decided whether a range is stable. larger => more unstable ranges range_threshold: same as threshold, but it decided whether a merged range is stable. see https://github.com/williamfzc/stagesepx/issues/17 for details offset: it will change the way to decided whether two ranges can be merged before: first_range.end == second_range.start after: first_range.end + offset >= secord_range.start :return: """ """ videos have 4 kinds of status: - stable start + stable end (usually) - stable start + unstable end - unstable start + stable end - unstable start + unstable end so, unstable range list can be: - start > 0, end < frame_count - start = 0, end < frame_count - start > 0, end = frame_count - start = 0, end = frame_count """ unstable_range_list = self.get_unstable_range(unstable_limit, **kwargs) # it is not a real frame (not existed) # just take it as a beginning # real frame id is started with 1, with non-zero timestamp video_start_frame_id = 0 video_start_timestamp = 0.0 video_end_frame_id = self.range_list[-1].end video_end_timestamp = self.range_list[-1].end_time # stable all the time if len(unstable_range_list) == 0: logger.warning( "no unstable stage detected, seems nothing happened in your video" ) return ( # stable [ VideoCutRange( self.video, video_start_frame_id, video_end_frame_id, [1.0], [0.0], [0.0], video_start_timestamp, video_end_timestamp, ) ], # unstable [], ) # IMPORTANT: +1 and -1 easily cause error # end of first stable range == start of first unstable range first_stable_range_end_id = unstable_range_list[0].start - 1 # start of last stable range == end of last unstable range end_stable_range_start_id = unstable_range_list[-1].end + 1 # IMPORTANT: len(ssim_list) + 1 == video_end_frame_id range_list: typing.List[VideoCutRange] = list() # stable start if first_stable_range_end_id >= 1: logger.debug(f"stable start") range_list.append( VideoCutRange( self.video, video_start_frame_id, first_stable_range_end_id, [1.0], [0.0], [0.0], video_start_timestamp, self.get_target_range_by_id(first_stable_range_end_id).end_time, ) ) # unstable start else: logger.debug("unstable start") # stable end if end_stable_range_start_id <= video_end_frame_id: logger.debug("stable end") range_list.append( VideoCutRange( self.video, end_stable_range_start_id, video_end_frame_id, [1.0], [0.0], [0.0], self.get_target_range_by_id(end_stable_range_start_id).end_time, video_end_timestamp, ) ) # unstable end else: logger.debug("unstable end") # diff range for i in range(len(unstable_range_list) - 1): range_start_id = unstable_range_list[i].end + 1 range_end_id = unstable_range_list[i + 1].start - 1 # stable range's length is 1 if range_start_id > range_end_id: range_start_id, range_end_id = range_end_id, range_start_id range_list.append( # IMPORTANT: frame's timestamp => end time of this frame # because frame 0's timestamp is 0.0 # frame {range_start_id} end time - frame {range_end_id} end time VideoCutRange( self.video, range_start_id, range_end_id, [1.0], [0.0], [0.0], self.get_target_range_by_id(range_start_id).end_time, self.get_target_range_by_id(range_end_id).end_time, ) ) # remove some ranges, which is limit if limit: range_list = self._length_filter(range_list, limit) logger.debug(f"stable range of [{self.video.path}]: {range_list}") stable_range_list = sorted(range_list, key=lambda x: x.start) return stable_range_list, unstable_range_list
def loads(cls, content: str) -> "VideoCutResult": json_dict: dict = json.loads(content) return cls( VideoObject(**json_dict["video"]), [VideoCutRange(**each) for each in json_dict["range_list"]], )
def _convert_video_into_range_list( self, video: VideoObject, block: int, window_size: int, window_coefficient: int) -> typing.List[VideoCutRange]: step = self.step video_length = video.frame_count class _Window(object): def __init__(self): self.start = 1 self.size = window_size self.end = self.start + window_size * step def load_data(self) -> typing.List[VideoFrame]: cur = self.start result = [] video_operator = video.get_operator() while cur <= self.end: frame = video_operator.get_frame_by_id(cur) result.append(frame) cur += step # at least 2 if len(result) < 2: last = video_operator.get_frame_by_id(self.end) result.append(last) return result def shift(self) -> bool: logger.debug(f"window before: {self.start}, {self.end}") self.start += step self.end += step if self.start >= video_length: # out of range return False # window end if self.end >= video_length: self.end = video_length logger.debug(f"window after: {self.start}, {self.end}") return True def _float_merge(float_list: typing.List[float]) -> float: # the first, the largest. length = len(float_list) result = 0.0 denominator = 0.0 for i, each in enumerate(float_list): weight = pow(length - i, window_coefficient) denominator += weight result += each * weight logger.debug(f"calc: {each} x {weight}") final = result / denominator logger.debug(f"calc final: {final} from {result} / {denominator}") return final range_list: typing.List[VideoCutRange] = list() logger.info( f"total frame count: {video_length}, size: {video.frame_size}") window = _Window() while True: frame_list = window.load_data() frame_list = [self._apply_hook(each) for each in frame_list] # window loop ssim_list = [] mse_list = [] psnr_list = [] cur_frame = frame_list[0] first_target_frame = frame_list[1] cur_frame_list = self.pic_split(cur_frame.data, block) for each in frame_list[1:]: each_frame_list = self.pic_split(each.data, block) ssim, mse, psnr = self.compare_frame_list( cur_frame_list, each_frame_list) ssim_list.append(ssim) mse_list.append(mse) psnr_list.append(psnr) logger.debug( f"between {cur_frame.frame_id} & {each.frame_id}: ssim={ssim}; mse={mse}; psnr={psnr}" ) ssim = _float_merge(ssim_list) mse = _float_merge(mse_list) psnr = _float_merge(psnr_list) range_list.append( VideoCutRange( video, start=cur_frame.frame_id, end=first_target_frame.frame_id, ssim=[ssim], mse=[mse], psnr=[psnr], start_time=cur_frame.timestamp, end_time=first_target_frame.timestamp, )) continue_flag = window.shift() if not continue_flag: break return range_list
def _convert_video_into_range_list(self, video: VideoObject, block: int = None, *args, **kwargs) -> typing.List[VideoCutRange]: if not block: block = 2 range_list: typing.List[VideoCutRange] = list() with toolbox.video_capture(video.path) as cap: logger.debug( f'total frame count: {video.frame_count}, size: {video.frame_size}' ) # load the first two frames _, start = cap.read() start_frame_id = toolbox.get_current_frame_id(cap) start_frame_time = toolbox.get_current_frame_time(cap) toolbox.video_jump(cap, self.step + 1) ret, end = cap.read() end_frame_id = toolbox.get_current_frame_id(cap) end_frame_time = toolbox.get_current_frame_time(cap) # hook start = self._apply_hook(start_frame_id, start) # check block if not self.is_block_valid(start, block): logger.warning( 'array split does not result in an equal division, set block to 1' ) block = 1 while ret: # hook end = self._apply_hook(end_frame_id, end, *args, **kwargs) logger.debug( f'computing {start_frame_id}({start_frame_time}) & {end_frame_id}({end_frame_time}) ...' ) start_part_list = self.pic_split(start, block) end_part_list = self.pic_split(end, block) # find the min ssim and the max mse / psnr ssim = 1. mse = 0. psnr = 0. for part_index, (each_start, each_end) in enumerate( zip(start_part_list, end_part_list)): part_ssim = toolbox.compare_ssim(each_start, each_end) if part_ssim < ssim: ssim = part_ssim # mse is very sensitive part_mse = toolbox.calc_mse(each_start, each_end) if part_mse > mse: mse = part_mse part_psnr = toolbox.calc_psnr(each_start, each_end) if part_psnr > psnr: psnr = part_psnr logger.debug( f'part {part_index}: ssim={part_ssim}; mse={part_mse}; psnr={part_psnr}' ) logger.debug( f'between {start_frame_id} & {end_frame_id}: ssim={ssim}; mse={mse}; psnr={psnr}' ) range_list.append( VideoCutRange( video, start=start_frame_id, end=end_frame_id, ssim=[ssim], mse=[mse], psnr=[psnr], start_time=start_frame_time, end_time=end_frame_time, )) # load the next one start = end start_frame_id, end_frame_id = end_frame_id, end_frame_id + self.step start_frame_time = end_frame_time toolbox.video_jump(cap, end_frame_id) ret, end = cap.read() end_frame_time = toolbox.get_current_frame_time(cap) return range_list
def _convert_video_into_range_list(self, video: VideoObject, block: int = None, *args, **kwargs) -> typing.List[VideoCutRange]: range_list: typing.List[VideoCutRange] = list() logger.info( f"total frame count: {video.frame_count}, size: {video.frame_size}" ) # load the first two frames video_operator = video.get_operator() cur_frame = video_operator.get_frame_by_id(1) next_frame = video_operator.get_frame_by_id(1 + self.step) # hook cur_frame.data = self._apply_hook(cur_frame.frame_id, cur_frame.data) # check block if not block: block = 2 if not self.is_block_valid(cur_frame.data, block): logger.warning( "array split does not result in an equal division, set block to 1" ) block = 1 while True: # hook next_frame.data = self._apply_hook(next_frame.frame_id, next_frame.data, *args, **kwargs) logger.debug( f"computing {cur_frame.frame_id}({cur_frame.timestamp}) & {next_frame.frame_id}({next_frame.timestamp}) ..." ) start_part_list = self.pic_split(cur_frame.data, block) end_part_list = self.pic_split(next_frame.data, block) # find the min ssim and the max mse / psnr ssim = 1.0 mse = 0.0 psnr = 0.0 for part_index, (each_start, each_end) in enumerate( zip(start_part_list, end_part_list)): part_ssim = toolbox.compare_ssim(each_start, each_end) if part_ssim < ssim: ssim = part_ssim # mse is very sensitive part_mse = toolbox.calc_mse(each_start, each_end) if part_mse > mse: mse = part_mse part_psnr = toolbox.calc_psnr(each_start, each_end) if part_psnr > psnr: psnr = part_psnr logger.debug( f"part {part_index}: ssim={part_ssim}; mse={part_mse}; psnr={part_psnr}" ) logger.debug( f"between {cur_frame.frame_id} & {next_frame.frame_id}: ssim={ssim}; mse={mse}; psnr={psnr}" ) range_list.append( VideoCutRange( video, start=cur_frame.frame_id, end=next_frame.frame_id, ssim=[ssim], mse=[mse], psnr=[psnr], start_time=cur_frame.timestamp, end_time=next_frame.timestamp, )) # load the next one cur_frame = next_frame next_frame = video_operator.get_frame_by_id(next_frame.frame_id + self.step) if next_frame is None: break return range_list
def loads(cls, content: str) -> 'VideoCutResult': json_dict: dict = json.loads(content) return cls( VideoObject(**json_dict['video']), [VideoCutRange(**each) for each in json_dict['range_list']] )