def _classify_frame(self, frame: np.ndarray, video_cap: cv2.VideoCapture, threshold: float = None, *_, **__) -> str: if not threshold: threshold = 0.85 frame = toolbox.compress_frame( frame, self.compress_rate, self.target_size, ) result = list() for each_stage_name, each_stage_pic_list in self.read(video_cap): each_result = list() for target_pic in each_stage_pic_list: target_pic = toolbox.compress_frame(target_pic, self.compress_rate, self.target_size) each_pic_ssim = toolbox.compare_ssim(frame, target_pic) each_result.append(each_pic_ssim) ssim = max(each_result) result.append((each_stage_name, ssim)) logger.debug(f'stage [{each_stage_name}]: {ssim}') result = max(result, key=lambda x: x[1]) if result[1] < threshold: logger.debug('not a known stage, set it -1') result = ('-1', result[1]) return result[0]
def _classify_frame(self, frame: np.ndarray, threshold: float = None, *_, **__) -> str: if not threshold: threshold = 0.85 frame = toolbox.compress_frame(frame) result = list() for each_stage_name, each_stage_pic_list in self.data.items(): each_result = list() for each in each_stage_pic_list: target_pic = cv2.imread(each.as_posix()) target_pic = toolbox.compress_frame(target_pic) each_pic_ssim = toolbox.compare_ssim(frame, target_pic) each_result.append(each_pic_ssim) ssim = max(each_result) result.append((each_stage_name, ssim)) logger.debug(f'stage [{each_stage_name}]: {ssim}') result = max(result, key=lambda x: x[1]) if result[1] < threshold: logger.debug('not a known stage, set it -1') result = ('-1', result[1]) return result[0]
def pick_and_save(self, range_list: typing.List[VideoCutRange], frame_count: int, to_dir: str = None, compress_rate: float = None, *args, **kwargs) -> str: stage_list = list() for index, each_range in enumerate(range_list): picked = each_range.pick(frame_count, *args, **kwargs) logger.info(f'pick {picked} in range {each_range}') stage_list.append((index, picked)) # create parent dir if not to_dir: to_dir = toolbox.get_timestamp_str() os.makedirs(to_dir, exist_ok=True) for each_stage_id, each_frame_list in stage_list: # create sub dir each_stage_dir = os.path.join(to_dir, str(each_stage_id)) os.makedirs(each_stage_dir, exist_ok=True) with toolbox.video_capture(self.video_path) as cap: for each_frame_id in each_frame_list: each_frame_path = os.path.join(each_stage_dir, f'{uuid.uuid4()}.png') each_frame = toolbox.get_frame(cap, each_frame_id - 1) if compress_rate: each_frame = toolbox.compress_frame(each_frame, compress_rate) cv2.imwrite(each_frame_path, each_frame) logger.debug(f'frame [{each_frame_id}] saved to {each_frame_path}') return to_dir
def do( self, frame_id: int, frame: np.ndarray, *_, **__ ) -> typing.Optional[np.ndarray]: super().do(frame_id, frame, *_, **__) return toolbox.compress_frame( frame, compress_rate=self.compress_rate, target_size=self.target_size )
def get_stable_stage_sample( data_list: typing.List[ClassifierResult], *args, **kwargs ) -> np.ndarray: last = data_list[0] picked: typing.List[ClassifierResult] = [last] for each in data_list: # ignore unstable stage if each.stage == UNSTABLE_FLAG: continue if last.stage != each.stage: last = each picked.append(each) def get_split_line(f): return np.zeros((f.shape[0], 5)) with toolbox.video_capture(last.video_path) as cap: frame_list: typing.List[np.ndarray] = [] for each in picked: frame = toolbox.get_frame(cap, each.frame_id) frame = toolbox.compress_frame(frame, *args, **kwargs) split_line = get_split_line(frame) frame_list.append(frame) frame_list.append(split_line) return np.hstack(frame_list)
def to_video_frame(self, *args, **kwargs) -> VideoFrame: # VideoFrame has `data` # SingleClassifierResult has `stage` with toolbox.video_capture(self.video_path) as cap: frame = toolbox.get_frame(cap, self.frame_id) compressed = toolbox.compress_frame(frame, *args, **kwargs) return VideoFrame(self.frame_id, self.timestamp, compressed)
def pick_and_save( self, range_list: typing.List[VideoCutRange], frame_count: int, to_dir: str = None, prune: float = None, # in kwargs # compress_rate: float = None, # target_size: typing.Tuple[int, int] = None, # to_grey: bool = None, *args, **kwargs) -> str: """ pick some frames from range, and save them as files :param range_list: VideoCutRange list :param frame_count: default to 3, and finally you will get 3 frames for each range :param to_dir: will saved to this path :param prune: float, 0-1. if set it 0.9, some stages which are too similar (ssim > 0.9) will be removed :param args: :param kwargs: :return: """ stage_list = list() # build tag and get frames for index, each_range in enumerate(range_list): picked = each_range.pick(frame_count, *args, **kwargs) picked_frames = each_range.get_frames(picked) logger.info(f'pick {picked} in range {each_range}') stage_list.append((str(index), picked_frames)) # prune if prune: stage_list = self._prune(prune, stage_list) # create parent dir if not to_dir: to_dir = toolbox.get_timestamp_str() os.makedirs(to_dir, exist_ok=True) for each_stage_id, each_frame_list in stage_list: # create sub dir each_stage_dir = os.path.join(to_dir, str(each_stage_id)) os.makedirs(each_stage_dir, exist_ok=True) for each_frame_object in each_frame_list: each_frame_path = os.path.join(each_stage_dir, f'{uuid.uuid4()}.png') compressed = toolbox.compress_frame(each_frame_object.frame, **kwargs) cv2.imwrite(each_frame_path, compressed) logger.debug( f'frame [{each_frame_object.frame_id}] saved to {each_frame_path}' ) return to_dir
def convert_video_into_ssim_list( self, video_path: str) -> typing.List[VideoCutRange]: ssim_list = list() with toolbox.video_capture(video_path) as cap: # get video info frame_count = toolbox.get_frame_count(cap) frame_size = toolbox.get_frame_size(cap) logger.debug( f'total frame count: {frame_count}, size: {frame_size}') # load the first two frames _, start = cap.read() start_frame_id = toolbox.get_current_frame_id(cap) toolbox.video_jump(cap, self.step) ret, end = cap.read() end_frame_id = toolbox.get_current_frame_id(cap) # compress start = toolbox.compress_frame(start, compress_rate=self.compress_rate) while ret: end = toolbox.compress_frame(end, compress_rate=self.compress_rate) ssim = toolbox.compare_ssim(start, end) logger.debug( f'ssim between {start_frame_id} & {end_frame_id}: {ssim}') ssim_list.append( VideoCutRange( video_path, start=start_frame_id, end=end_frame_id, ssim=ssim, )) # load the next one start = end start_frame_id, end_frame_id = end_frame_id, end_frame_id + self.step toolbox.video_jump(cap, end_frame_id) ret, end = cap.read() return ssim_list
def thumbnail( self, target_range: VideoCutRange, to_dir: str = None, compress_rate: float = None, is_vertical: bool = None, *_, **__, ) -> np.ndarray: """ build a thumbnail, for easier debug or something else :param target_range: VideoCutRange :param to_dir: your thumbnail will be saved to this path :param compress_rate: float, 0 - 1, about thumbnail's size, default to 0.1 (1/10) :param is_vertical: direction :return: """ if not compress_rate: compress_rate = 0.1 # direction if is_vertical: stack_func = np.vstack def get_split_line(f): return np.zeros((5, f.shape[1])) else: stack_func = np.hstack def get_split_line(f): return np.zeros((f.shape[0], 5)) frame_list = list() with toolbox.video_capture(self.video.path) as cap: toolbox.video_jump(cap, target_range.start) ret, frame = cap.read() count = 1 length = target_range.get_length() while ret and count <= length: frame = toolbox.compress_frame(frame, compress_rate) frame_list.append(frame) frame_list.append(get_split_line(frame)) ret, frame = cap.read() count += 1 merged = stack_func(frame_list) # create parent dir if to_dir: target_path = os.path.join( to_dir, f"thumbnail_{target_range.start}-{target_range.end}.png" ) cv2.imwrite(target_path, merged) logger.debug(f"save thumbnail to {target_path}") return merged
def predict_with_object(self, pic_object: np.ndarray) -> str: """ predict a single object :param pic_object: :return: """ pic_object = toolbox.compress_frame(pic_object, self.compress_rate, self.target_size) pic_object = self.feature_func(pic_object) pic_object = pic_object.reshape(1, -1) return self._model.predict(pic_object)[0]
def to_video_frame(self, *args, **kwargs) -> VideoFrame: # VideoFrame has `data` # SingleClassifierResult has `stage` (data is optional) # already have data if self.data is not None: return VideoFrame(self.frame_id, self.timestamp, self.data) # no data with toolbox.video_capture(self.video_path) as cap: frame = toolbox.get_frame(cap, self.frame_id) compressed = toolbox.compress_frame(frame, *args, **kwargs) return VideoFrame(self.frame_id, self.timestamp, compressed)
def train(self): if not self._model: self._model = LinearSVC() train_data = list() train_label = list() for each_label, each_label_pic_list in self.read(): for each_pic_object in each_label_pic_list: each_pic_object = toolbox.compress_frame(each_pic_object) each_pic_object = self.feature_func(each_pic_object).flatten() train_data.append(each_pic_object) train_label.append(each_label) logger.debug('data ready') self._model.fit(train_data, train_label) logger.debug('train finished')
def train(self): if not self._model: self._model = LinearSVC() train_data = list() train_label = list() for each_label, each_label_pic_list in self.data.items(): for each_pic in each_label_pic_list: logger.debug(f'loading {each_pic} ...') each_pic_object = cv2.imread(each_pic.as_posix()) each_pic_object = toolbox.compress_frame(each_pic_object) each_pic_object = self.feature_func(each_pic_object).flatten() train_data.append(each_pic_object) train_label.append(each_label) logger.debug('data ready') self._model.fit(train_data, train_label) logger.debug('train finished')
def pick_and_save(self, range_list: typing.List[VideoCutRange], frame_count: int, to_dir: str = None, # in kwargs # compress_rate: float = None, # target_size: typing.Tuple[int, int] = None, # to_grey: bool = None, *args, **kwargs) -> str: """ pick some frames from range, and save them as files :param range_list: VideoCutRange list :param frame_count: default to 3, and finally you will get 3 frames for each range :param to_dir: will saved to this path :param args: :param kwargs: :return: """ stage_list = list() for index, each_range in enumerate(range_list): picked = each_range.pick(frame_count, *args, **kwargs) logger.info(f'pick {picked} in range {each_range}') stage_list.append((index, picked)) # create parent dir if not to_dir: to_dir = toolbox.get_timestamp_str() os.makedirs(to_dir, exist_ok=True) for each_stage_id, each_frame_list in stage_list: # create sub dir each_stage_dir = os.path.join(to_dir, str(each_stage_id)) os.makedirs(each_stage_dir, exist_ok=True) with toolbox.video_capture(self.video_path) as cap: for each_frame_id in each_frame_list: each_frame_path = os.path.join(each_stage_dir, f'{uuid.uuid4()}.png') each_frame = toolbox.get_frame(cap, each_frame_id - 1) each_frame = toolbox.compress_frame(each_frame, **kwargs) cv2.imwrite(each_frame_path, each_frame) logger.debug(f'frame [{each_frame_id}] saved to {each_frame_path}') return to_dir
def train(self): """ train your classifier with data. must be called before prediction :return: """ if not self._model: self._model = LinearSVC() train_data = list() train_label = list() for each_label, each_label_pic_list in self.read(): for each_pic_object in each_label_pic_list: each_pic_object = toolbox.compress_frame(each_pic_object, self.compress_rate, self.target_size) each_pic_object = self.feature_func(each_pic_object).flatten() train_data.append(each_pic_object) train_label.append(each_label) logger.debug('data ready') self._model.fit(train_data, train_label) logger.debug('train finished')
def predict_with_object(self, pic_object: np.ndarray) -> str: pic_object = toolbox.compress_frame(pic_object) pic_object = self.feature_func(pic_object) pic_object = pic_object.reshape(1, -1) return self._model.predict(pic_object)[0]
def pick_and_save( self, range_list: typing.List[VideoCutRange], frame_count: int, to_dir: str = None, prune: float = None, meaningful_name: bool = None, # in kwargs # compress_rate: float = None, # target_size: typing.Tuple[int, int] = None, # to_grey: bool = None, *args, **kwargs, ) -> str: """ pick some frames from range, and save them as files :param range_list: VideoCutRange list :param frame_count: default to 3, and finally you will get 3 frames for each range :param to_dir: will saved to this path :param prune: float, 0-1. if set it 0.9, some stages which are too similar (ssim > 0.9) will be removed :param meaningful_name: bool, False by default. if true, image names will become meaningful (with timestamp/id or something else) :param args: :param kwargs: :return: """ stage_list = list() # build tag and get frames for index, each_range in enumerate(range_list): picked = each_range.pick(frame_count, *args, **kwargs) picked_frames = each_range.get_frames(picked) logger.info(f"pick {picked} in range {each_range}") stage_list.append((str(index), picked_frames)) # prune if prune: stage_list = self._prune(prune, stage_list) # create parent dir if not to_dir: to_dir = toolbox.get_timestamp_str() logger.debug(f"try to make dirs: {to_dir}") os.makedirs(to_dir, exist_ok=True) for each_stage_id, each_frame_list in stage_list: # create sub dir each_stage_dir = os.path.join(to_dir, str(each_stage_id)) if os.path.isdir(each_stage_dir): logger.warning(f"sub dir [{each_stage_dir}] already existed") logger.warning( "NOTICE: make sure your data will not be polluted by accident" ) os.makedirs(each_stage_dir, exist_ok=True) # create image files for each_frame_object in each_frame_list: if meaningful_name: # - video name # - frame id # - frame timestamp image_name = ( f"{os.path.basename(os.path.splitext(self.video.path)[0])}" f"_" f"{each_frame_object.frame_id}" f"_" f"{each_frame_object.timestamp}" f".png" ) else: image_name = f"{uuid.uuid4()}.png" each_frame_path = os.path.join(each_stage_dir, image_name) compressed = toolbox.compress_frame(each_frame_object.data, **kwargs) cv2.imwrite(each_frame_path, compressed) logger.debug( f"frame [{each_frame_object.frame_id}] saved to {each_frame_path}" ) return to_dir
def draw( self, classifier_result: ClassifierResult, report_path: str = None, unstable_ranges: typing.List[VideoCutRange] = None, cut_result: VideoCutResult = None, compress_rate: float = None, *_, **__, ): """ draw report file :param classifier_result: classifierResult, output of classifier :param report_path: your report will be there :param unstable_ranges: for marking unstable ranges :param cut_result: more charts would be built :param compress_rate: :return: """ # default: compress_rate if not compress_rate: compress_rate = 0.2 if not unstable_ranges: unstable_ranges = [] # draw line = self._draw_line(classifier_result) bar = self._draw_bar(classifier_result) # merge charts page = Page() page.add(line) page.add(bar) # insert pictures if cut_result: # sim chart sim_line = self._draw_sim(cut_result) page.add(sim_line) # mark range for each in unstable_ranges: classifier_result.mark_range_unstable(each.start, each.end) offset = classifier_result.get_offset() for each in classifier_result.get_stage_range(): middle = each[len(each) // 2] if middle.is_stable(): label = "stable" frame = toolbox.compress_frame(middle.get_data(), compress_rate=compress_rate) else: label = "unstable" frame = np.hstack([ toolbox.compress_frame(i.get_data(), compress_rate=compress_rate) for i in each ]) first, last = each[0], each[-1] self.add_thumbnail( f"{label} range {first.frame_id}({first.timestamp - offset}) - {last.frame_id}({last.timestamp}), " f"duration: {last.timestamp - first.timestamp + offset}", frame, ) # calc time cost cost_dict = classifier_result.calc_changing_cost() # time stamp timestamp = toolbox.get_timestamp_str() # video self.add_extra("video path", classifier_result.video_path) self.add_extra("frame count", str(classifier_result.get_length())) self.add_extra("offset between frames", str(classifier_result.get_offset())) # insert extras template = Template(get_template()) template_content = template.render( chart=Markup(page.render_embed()), thumbnail_list=self.thumbnail_list, extras=self.extra_dict, background_color=constants.BACKGROUND_COLOR, cost_dict=cost_dict, timestamp=timestamp, version_code=__VERSION__, ) # save to file if not report_path: report_path = f"{timestamp}.html" with open(report_path, "w", encoding=constants.CHARSET) as fh: fh.write(template_content) logger.info(f"save report to {report_path}")
def test_compress(): image = toolbox.imread(IMAGE_PATH) frame = toolbox.compress_frame(image, target_size=(100, 100)) assert frame.shape == (100, 100)
def convert_video_into_ssim_list(self, video_path: str, block: int = None, **kwargs) -> typing.List[VideoCutRange]: if not block: block = 1 ssim_list = list() with toolbox.video_capture(video_path) as cap: # get video info frame_count = toolbox.get_frame_count(cap) frame_size = toolbox.get_frame_size(cap) logger.debug(f'total frame count: {frame_count}, size: {frame_size}') # load the first two frames _, start = cap.read() start_frame_id = toolbox.get_current_frame_id(cap) start_frame_time = toolbox.get_current_frame_time(cap) toolbox.video_jump(cap, self.step + 1) ret, end = cap.read() end_frame_id = toolbox.get_current_frame_id(cap) end_frame_time = toolbox.get_current_frame_time(cap) # compress start = toolbox.compress_frame(start, **kwargs) # split func # width > height if frame_size[0] > frame_size[1]: split_func = np.hsplit else: split_func = np.vsplit logger.debug(f'split function: {split_func.__name__}') while ret: end = toolbox.compress_frame(end, **kwargs) ssim = 0 start_part_list = split_func(start, block) end_part_list = split_func(end, block) for part_index, (each_start, each_end) in enumerate(zip(start_part_list, end_part_list)): part_ssim = toolbox.compare_ssim(each_start, each_end) ssim += part_ssim logger.debug(f'part {part_index}: {part_ssim}') ssim = ssim / block logger.debug(f'ssim between {start_frame_id} & {end_frame_id}: {ssim}') ssim_list.append( VideoCutRange( video_path, start=start_frame_id, end=end_frame_id, ssim=[ssim], start_time=start_frame_time, end_time=end_frame_time, ) ) # load the next one start = end start_frame_id, end_frame_id = end_frame_id, end_frame_id + self.step start_frame_time = end_frame_time toolbox.video_jump(cap, end_frame_id) ret, end = cap.read() end_frame_time = toolbox.get_current_frame_time(cap) return ssim_list
def draw( self, classifier_result: ClassifierResult, report_path: str = None, unstable_ranges: typing.List[VideoCutRange] = None, cut_result: VideoCutResult = None, compress_rate: float = None, *_, **__, ): """ draw report file :param classifier_result: classifierResult, output of classifier :param report_path: your report will be there :param unstable_ranges: for marking unstable ranges :param cut_result: more charts would be built :param compress_rate: :return: """ # default: compress_rate if not compress_rate: compress_rate = 0.2 if not unstable_ranges: unstable_ranges = [] # draw line = self._draw_line(classifier_result) bar = self._draw_bar(classifier_result) # merge charts page = Page() page.add(line) page.add(bar) # insert pictures if cut_result: # sim chart sim_line = self._draw_sim(cut_result) page.add(sim_line) # mark range for each in unstable_ranges: classifier_result.mark_range_unstable(each.start, each.end) offset = classifier_result.get_offset() stage_range = classifier_result.get_stage_range() for cur_index in range(len(stage_range)): each = stage_range[cur_index] middle = each[len(each) // 2] if middle.is_stable(): label = self.LABEL_STABLE frame = toolbox.compress_frame(middle.get_data(), compress_rate=compress_rate) else: # todo: looks not good enough. `unspecific` looks a little weird but I have no idea now if middle.stage == constants.UNKNOWN_STAGE_FLAG: label = self.LABEL_UNSPECIFIC else: label = self.LABEL_UNSTABLE # add a frame if cur_index + 1 < len(stage_range): new_each = [*each, stage_range[cur_index + 1][0]] else: new_each = each frame = np.hstack([ toolbox.compress_frame(i.get_data(), compress_rate=compress_rate) for i in new_each ]) first, last = each[0], each[-1] self.add_thumbnail( f"{label} range {first.frame_id}({first.timestamp}) - {last.frame_id}({last.timestamp + offset}), " f"duration: {last.timestamp - first.timestamp + offset}", frame, ) # calc time cost cost_dict = classifier_result.calc_changing_cost() # time stamp timestamp = toolbox.get_timestamp_str() # video self.add_extra("video path", classifier_result.video_path) self.add_extra("frame count", str(classifier_result.get_length())) self.add_extra("offset between frames", str(classifier_result.get_offset())) # insert extras template = Template(get_template()) template_content = template.render( chart=Markup(page.render_embed()), thumbnail_list=self.thumbnail_list, extras=self.extra_dict, background_color=constants.BACKGROUND_COLOR, cost_dict=cost_dict, timestamp=timestamp, version_code=__VERSION__, ) # default: write to current dir default_name = f"{timestamp}.html" if not report_path: report_path = default_name # somewhere specific # existed dir? elif os.path.isdir(report_path): report_path = os.path.join(report_path, default_name) logger.debug(f"trying to save report to {report_path}") # write file with open(report_path, "w", encoding=constants.CHARSET) as fh: fh.write(template_content) logger.info(f"save report to {report_path}")
def do(self, frame: VideoFrame, *_, **__) -> typing.Optional[VideoFrame]: super().do(frame, *_, **__) frame.data = toolbox.compress_frame(frame.data, compress_rate=self.compress_rate, target_size=self.target_size) return frame