def _classify_frame(self,
frame: np.ndarray,
video_cap: cv2.VideoCapture,
threshold: float = None,
*_, **__) -> str:
if not threshold:
threshold = 0.85
frame = toolbox.compress_frame(
frame,
self.compress_rate,
self.target_size,
)
result = list()
for each_stage_name, each_stage_pic_list in self.read(video_cap):
each_result = list()
for target_pic in each_stage_pic_list:
target_pic = toolbox.compress_frame(target_pic, self.compress_rate, self.target_size)
each_pic_ssim = toolbox.compare_ssim(frame, target_pic)
each_result.append(each_pic_ssim)
ssim = max(each_result)
result.append((each_stage_name, ssim))
logger.debug(f'stage [{each_stage_name}]: {ssim}')
result = max(result, key=lambda x: x[1])
if result[1] < threshold:
logger.debug('not a known stage, set it -1')
result = ('-1', result[1])
return result[0]
def _classify_frame(self,
frame: np.ndarray,
threshold: float = None,
*_,
**__) -> str:
if not threshold:
threshold = 0.85
frame = toolbox.compress_frame(frame)
result = list()
for each_stage_name, each_stage_pic_list in self.data.items():
each_result = list()
for each in each_stage_pic_list:
target_pic = cv2.imread(each.as_posix())
target_pic = toolbox.compress_frame(target_pic)
each_pic_ssim = toolbox.compare_ssim(frame, target_pic)
each_result.append(each_pic_ssim)
ssim = max(each_result)
result.append((each_stage_name, ssim))
logger.debug(f'stage [{each_stage_name}]: {ssim}')
result = max(result, key=lambda x: x[1])
if result[1] < threshold:
logger.debug('not a known stage, set it -1')
result = ('-1', result[1])
return result[0]
def pick_and_save(self,
range_list: typing.List[VideoCutRange],
frame_count: int,
to_dir: str = None,
compress_rate: float = None,
*args, **kwargs) -> str:
stage_list = list()
for index, each_range in enumerate(range_list):
picked = each_range.pick(frame_count, *args, **kwargs)
logger.info(f'pick {picked} in range {each_range}')
stage_list.append((index, picked))
# create parent dir
if not to_dir:
to_dir = toolbox.get_timestamp_str()
os.makedirs(to_dir, exist_ok=True)
for each_stage_id, each_frame_list in stage_list:
# create sub dir
each_stage_dir = os.path.join(to_dir, str(each_stage_id))
os.makedirs(each_stage_dir, exist_ok=True)
with toolbox.video_capture(self.video_path) as cap:
for each_frame_id in each_frame_list:
each_frame_path = os.path.join(each_stage_dir, f'{uuid.uuid4()}.png')
each_frame = toolbox.get_frame(cap, each_frame_id - 1)
if compress_rate:
each_frame = toolbox.compress_frame(each_frame, compress_rate)
cv2.imwrite(each_frame_path, each_frame)
logger.debug(f'frame [{each_frame_id}] saved to {each_frame_path}')
return to_dir
def do(
self, frame_id: int, frame: np.ndarray, *_, **__
) -> typing.Optional[np.ndarray]:
super().do(frame_id, frame, *_, **__)
return toolbox.compress_frame(
frame, compress_rate=self.compress_rate, target_size=self.target_size
)
def get_stable_stage_sample(
data_list: typing.List[ClassifierResult], *args, **kwargs
) -> np.ndarray:
last = data_list[0]
picked: typing.List[ClassifierResult] = [last]
for each in data_list:
# ignore unstable stage
if each.stage == UNSTABLE_FLAG:
continue
if last.stage != each.stage:
last = each
picked.append(each)
def get_split_line(f):
return np.zeros((f.shape[0], 5))
with toolbox.video_capture(last.video_path) as cap:
frame_list: typing.List[np.ndarray] = []
for each in picked:
frame = toolbox.get_frame(cap, each.frame_id)
frame = toolbox.compress_frame(frame, *args, **kwargs)
split_line = get_split_line(frame)
frame_list.append(frame)
frame_list.append(split_line)
return np.hstack(frame_list)
def to_video_frame(self, *args, **kwargs) -> VideoFrame:
# VideoFrame has `data`
# SingleClassifierResult has `stage`
with toolbox.video_capture(self.video_path) as cap:
frame = toolbox.get_frame(cap, self.frame_id)
compressed = toolbox.compress_frame(frame, *args, **kwargs)
return VideoFrame(self.frame_id, self.timestamp, compressed)
def pick_and_save(
self,
range_list: typing.List[VideoCutRange],
frame_count: int,
to_dir: str = None,
prune: float = None,
# in kwargs
# compress_rate: float = None,
# target_size: typing.Tuple[int, int] = None,
# to_grey: bool = None,
*args,
**kwargs) -> str:
"""
pick some frames from range, and save them as files
:param range_list: VideoCutRange list
:param frame_count: default to 3, and finally you will get 3 frames for each range
:param to_dir: will saved to this path
:param prune: float, 0-1. if set it 0.9, some stages which are too similar (ssim > 0.9) will be removed
:param args:
:param kwargs:
:return:
"""
stage_list = list()
# build tag and get frames
for index, each_range in enumerate(range_list):
picked = each_range.pick(frame_count, *args, **kwargs)
picked_frames = each_range.get_frames(picked)
logger.info(f'pick {picked} in range {each_range}')
stage_list.append((str(index), picked_frames))
# prune
if prune:
stage_list = self._prune(prune, stage_list)
# create parent dir
if not to_dir:
to_dir = toolbox.get_timestamp_str()
os.makedirs(to_dir, exist_ok=True)
for each_stage_id, each_frame_list in stage_list:
# create sub dir
each_stage_dir = os.path.join(to_dir, str(each_stage_id))
os.makedirs(each_stage_dir, exist_ok=True)
for each_frame_object in each_frame_list:
each_frame_path = os.path.join(each_stage_dir,
f'{uuid.uuid4()}.png')
compressed = toolbox.compress_frame(each_frame_object.frame,
**kwargs)
cv2.imwrite(each_frame_path, compressed)
logger.debug(
f'frame [{each_frame_object.frame_id}] saved to {each_frame_path}'
)
return to_dir
def convert_video_into_ssim_list(
self, video_path: str) -> typing.List[VideoCutRange]:
ssim_list = list()
with toolbox.video_capture(video_path) as cap:
# get video info
frame_count = toolbox.get_frame_count(cap)
frame_size = toolbox.get_frame_size(cap)
logger.debug(
f'total frame count: {frame_count}, size: {frame_size}')
# load the first two frames
_, start = cap.read()
start_frame_id = toolbox.get_current_frame_id(cap)
toolbox.video_jump(cap, self.step)
ret, end = cap.read()
end_frame_id = toolbox.get_current_frame_id(cap)
# compress
start = toolbox.compress_frame(start,
compress_rate=self.compress_rate)
while ret:
end = toolbox.compress_frame(end,
compress_rate=self.compress_rate)
ssim = toolbox.compare_ssim(start, end)
logger.debug(
f'ssim between {start_frame_id} & {end_frame_id}: {ssim}')
ssim_list.append(
VideoCutRange(
video_path,
start=start_frame_id,
end=end_frame_id,
ssim=ssim,
))
# load the next one
start = end
start_frame_id, end_frame_id = end_frame_id, end_frame_id + self.step
toolbox.video_jump(cap, end_frame_id)
ret, end = cap.read()
return ssim_list
def thumbnail(
self,
target_range: VideoCutRange,
to_dir: str = None,
compress_rate: float = None,
is_vertical: bool = None,
*_,
**__,
) -> np.ndarray:
"""
build a thumbnail, for easier debug or something else
:param target_range: VideoCutRange
:param to_dir: your thumbnail will be saved to this path
:param compress_rate: float, 0 - 1, about thumbnail's size, default to 0.1 (1/10)
:param is_vertical: direction
:return:
"""
if not compress_rate:
compress_rate = 0.1
# direction
if is_vertical:
stack_func = np.vstack
def get_split_line(f):
return np.zeros((5, f.shape[1]))
else:
stack_func = np.hstack
def get_split_line(f):
return np.zeros((f.shape[0], 5))
frame_list = list()
with toolbox.video_capture(self.video.path) as cap:
toolbox.video_jump(cap, target_range.start)
ret, frame = cap.read()
count = 1
length = target_range.get_length()
while ret and count <= length:
frame = toolbox.compress_frame(frame, compress_rate)
frame_list.append(frame)
frame_list.append(get_split_line(frame))
ret, frame = cap.read()
count += 1
merged = stack_func(frame_list)
# create parent dir
if to_dir:
target_path = os.path.join(
to_dir, f"thumbnail_{target_range.start}-{target_range.end}.png"
)
cv2.imwrite(target_path, merged)
logger.debug(f"save thumbnail to {target_path}")
return merged
def predict_with_object(self, pic_object: np.ndarray) -> str:
"""
predict a single object
:param pic_object:
:return:
"""
pic_object = toolbox.compress_frame(pic_object, self.compress_rate, self.target_size)
pic_object = self.feature_func(pic_object)
pic_object = pic_object.reshape(1, -1)
return self._model.predict(pic_object)[0]
def to_video_frame(self, *args, **kwargs) -> VideoFrame:
# VideoFrame has `data`
# SingleClassifierResult has `stage` (data is optional)
# already have data
if self.data is not None:
return VideoFrame(self.frame_id, self.timestamp, self.data)
# no data
with toolbox.video_capture(self.video_path) as cap:
frame = toolbox.get_frame(cap, self.frame_id)
compressed = toolbox.compress_frame(frame, *args, **kwargs)
return VideoFrame(self.frame_id, self.timestamp, compressed)
def train(self):
if not self._model:
self._model = LinearSVC()
train_data = list()
train_label = list()
for each_label, each_label_pic_list in self.read():
for each_pic_object in each_label_pic_list:
each_pic_object = toolbox.compress_frame(each_pic_object)
each_pic_object = self.feature_func(each_pic_object).flatten()
train_data.append(each_pic_object)
train_label.append(each_label)
logger.debug('data ready')
self._model.fit(train_data, train_label)
logger.debug('train finished')
def train(self):
if not self._model:
self._model = LinearSVC()
train_data = list()
train_label = list()
for each_label, each_label_pic_list in self.data.items():
for each_pic in each_label_pic_list:
logger.debug(f'loading {each_pic} ...')
each_pic_object = cv2.imread(each_pic.as_posix())
each_pic_object = toolbox.compress_frame(each_pic_object)
each_pic_object = self.feature_func(each_pic_object).flatten()
train_data.append(each_pic_object)
train_label.append(each_label)
logger.debug('data ready')
self._model.fit(train_data, train_label)
logger.debug('train finished')
def pick_and_save(self,
range_list: typing.List[VideoCutRange],
frame_count: int,
to_dir: str = None,
# in kwargs
# compress_rate: float = None,
# target_size: typing.Tuple[int, int] = None,
# to_grey: bool = None,
*args, **kwargs) -> str:
"""
pick some frames from range, and save them as files
:param range_list: VideoCutRange list
:param frame_count: default to 3, and finally you will get 3 frames for each range
:param to_dir: will saved to this path
:param args:
:param kwargs:
:return:
"""
stage_list = list()
for index, each_range in enumerate(range_list):
picked = each_range.pick(frame_count, *args, **kwargs)
logger.info(f'pick {picked} in range {each_range}')
stage_list.append((index, picked))
# create parent dir
if not to_dir:
to_dir = toolbox.get_timestamp_str()
os.makedirs(to_dir, exist_ok=True)
for each_stage_id, each_frame_list in stage_list:
# create sub dir
each_stage_dir = os.path.join(to_dir, str(each_stage_id))
os.makedirs(each_stage_dir, exist_ok=True)
with toolbox.video_capture(self.video_path) as cap:
for each_frame_id in each_frame_list:
each_frame_path = os.path.join(each_stage_dir, f'{uuid.uuid4()}.png')
each_frame = toolbox.get_frame(cap, each_frame_id - 1)
each_frame = toolbox.compress_frame(each_frame, **kwargs)
cv2.imwrite(each_frame_path, each_frame)
logger.debug(f'frame [{each_frame_id}] saved to {each_frame_path}')
return to_dir
def train(self):
"""
train your classifier with data. must be called before prediction
:return:
"""
if not self._model:
self._model = LinearSVC()
train_data = list()
train_label = list()
for each_label, each_label_pic_list in self.read():
for each_pic_object in each_label_pic_list:
each_pic_object = toolbox.compress_frame(each_pic_object, self.compress_rate, self.target_size)
each_pic_object = self.feature_func(each_pic_object).flatten()
train_data.append(each_pic_object)
train_label.append(each_label)
logger.debug('data ready')
self._model.fit(train_data, train_label)
logger.debug('train finished')
def predict_with_object(self, pic_object: np.ndarray) -> str:
pic_object = toolbox.compress_frame(pic_object)
pic_object = self.feature_func(pic_object)
pic_object = pic_object.reshape(1, -1)
return self._model.predict(pic_object)[0]
def pick_and_save(
self,
range_list: typing.List[VideoCutRange],
frame_count: int,
to_dir: str = None,
prune: float = None,
meaningful_name: bool = None,
# in kwargs
# compress_rate: float = None,
# target_size: typing.Tuple[int, int] = None,
# to_grey: bool = None,
*args,
**kwargs,
) -> str:
"""
pick some frames from range, and save them as files
:param range_list: VideoCutRange list
:param frame_count: default to 3, and finally you will get 3 frames for each range
:param to_dir: will saved to this path
:param prune: float, 0-1. if set it 0.9, some stages which are too similar (ssim > 0.9) will be removed
:param meaningful_name: bool, False by default. if true, image names will become meaningful (with timestamp/id or something else)
:param args:
:param kwargs:
:return:
"""
stage_list = list()
# build tag and get frames
for index, each_range in enumerate(range_list):
picked = each_range.pick(frame_count, *args, **kwargs)
picked_frames = each_range.get_frames(picked)
logger.info(f"pick {picked} in range {each_range}")
stage_list.append((str(index), picked_frames))
# prune
if prune:
stage_list = self._prune(prune, stage_list)
# create parent dir
if not to_dir:
to_dir = toolbox.get_timestamp_str()
logger.debug(f"try to make dirs: {to_dir}")
os.makedirs(to_dir, exist_ok=True)
for each_stage_id, each_frame_list in stage_list:
# create sub dir
each_stage_dir = os.path.join(to_dir, str(each_stage_id))
if os.path.isdir(each_stage_dir):
logger.warning(f"sub dir [{each_stage_dir}] already existed")
logger.warning(
"NOTICE: make sure your data will not be polluted by accident"
)
os.makedirs(each_stage_dir, exist_ok=True)
# create image files
for each_frame_object in each_frame_list:
if meaningful_name:
# - video name
# - frame id
# - frame timestamp
image_name = (
f"{os.path.basename(os.path.splitext(self.video.path)[0])}"
f"_"
f"{each_frame_object.frame_id}"
f"_"
f"{each_frame_object.timestamp}"
f".png"
)
else:
image_name = f"{uuid.uuid4()}.png"
each_frame_path = os.path.join(each_stage_dir, image_name)
compressed = toolbox.compress_frame(each_frame_object.data, **kwargs)
cv2.imwrite(each_frame_path, compressed)
logger.debug(
f"frame [{each_frame_object.frame_id}] saved to {each_frame_path}"
)
return to_dir
def draw(
self,
classifier_result: ClassifierResult,
report_path: str = None,
unstable_ranges: typing.List[VideoCutRange] = None,
cut_result: VideoCutResult = None,
compress_rate: float = None,
*_,
**__,
):
"""
draw report file
:param classifier_result: classifierResult, output of classifier
:param report_path: your report will be there
:param unstable_ranges: for marking unstable ranges
:param cut_result: more charts would be built
:param compress_rate:
:return:
"""
# default: compress_rate
if not compress_rate:
compress_rate = 0.2
if not unstable_ranges:
unstable_ranges = []
# draw
line = self._draw_line(classifier_result)
bar = self._draw_bar(classifier_result)
# merge charts
page = Page()
page.add(line)
page.add(bar)
# insert pictures
if cut_result:
# sim chart
sim_line = self._draw_sim(cut_result)
page.add(sim_line)
# mark range
for each in unstable_ranges:
classifier_result.mark_range_unstable(each.start, each.end)
offset = classifier_result.get_offset()
for each in classifier_result.get_stage_range():
middle = each[len(each) // 2]
if middle.is_stable():
label = "stable"
frame = toolbox.compress_frame(middle.get_data(),
compress_rate=compress_rate)
else:
label = "unstable"
frame = np.hstack([
toolbox.compress_frame(i.get_data(),
compress_rate=compress_rate)
for i in each
])
first, last = each[0], each[-1]
self.add_thumbnail(
f"{label} range {first.frame_id}({first.timestamp - offset}) - {last.frame_id}({last.timestamp}), "
f"duration: {last.timestamp - first.timestamp + offset}",
frame,
)
# calc time cost
cost_dict = classifier_result.calc_changing_cost()
# time stamp
timestamp = toolbox.get_timestamp_str()
# video
self.add_extra("video path", classifier_result.video_path)
self.add_extra("frame count", str(classifier_result.get_length()))
self.add_extra("offset between frames",
str(classifier_result.get_offset()))
# insert extras
template = Template(get_template())
template_content = template.render(
chart=Markup(page.render_embed()),
thumbnail_list=self.thumbnail_list,
extras=self.extra_dict,
background_color=constants.BACKGROUND_COLOR,
cost_dict=cost_dict,
timestamp=timestamp,
version_code=__VERSION__,
)
# save to file
if not report_path:
report_path = f"{timestamp}.html"
with open(report_path, "w", encoding=constants.CHARSET) as fh:
fh.write(template_content)
logger.info(f"save report to {report_path}")
def test_compress():
image = toolbox.imread(IMAGE_PATH)
frame = toolbox.compress_frame(image, target_size=(100, 100))
assert frame.shape == (100, 100)
def convert_video_into_ssim_list(self, video_path: str, block: int = None, **kwargs) -> typing.List[VideoCutRange]:
if not block:
block = 1
ssim_list = list()
with toolbox.video_capture(video_path) as cap:
# get video info
frame_count = toolbox.get_frame_count(cap)
frame_size = toolbox.get_frame_size(cap)
logger.debug(f'total frame count: {frame_count}, size: {frame_size}')
# load the first two frames
_, start = cap.read()
start_frame_id = toolbox.get_current_frame_id(cap)
start_frame_time = toolbox.get_current_frame_time(cap)
toolbox.video_jump(cap, self.step + 1)
ret, end = cap.read()
end_frame_id = toolbox.get_current_frame_id(cap)
end_frame_time = toolbox.get_current_frame_time(cap)
# compress
start = toolbox.compress_frame(start, **kwargs)
# split func
# width > height
if frame_size[0] > frame_size[1]:
split_func = np.hsplit
else:
split_func = np.vsplit
logger.debug(f'split function: {split_func.__name__}')
while ret:
end = toolbox.compress_frame(end, **kwargs)
ssim = 0
start_part_list = split_func(start, block)
end_part_list = split_func(end, block)
for part_index, (each_start, each_end) in enumerate(zip(start_part_list, end_part_list)):
part_ssim = toolbox.compare_ssim(each_start, each_end)
ssim += part_ssim
logger.debug(f'part {part_index}: {part_ssim}')
ssim = ssim / block
logger.debug(f'ssim between {start_frame_id} & {end_frame_id}: {ssim}')
ssim_list.append(
VideoCutRange(
video_path,
start=start_frame_id,
end=end_frame_id,
ssim=[ssim],
start_time=start_frame_time,
end_time=end_frame_time,
)
)
# load the next one
start = end
start_frame_id, end_frame_id = end_frame_id, end_frame_id + self.step
start_frame_time = end_frame_time
toolbox.video_jump(cap, end_frame_id)
ret, end = cap.read()
end_frame_time = toolbox.get_current_frame_time(cap)
return ssim_list
def draw(
self,
classifier_result: ClassifierResult,
report_path: str = None,
unstable_ranges: typing.List[VideoCutRange] = None,
cut_result: VideoCutResult = None,
compress_rate: float = None,
*_,
**__,
):
"""
draw report file
:param classifier_result: classifierResult, output of classifier
:param report_path: your report will be there
:param unstable_ranges: for marking unstable ranges
:param cut_result: more charts would be built
:param compress_rate:
:return:
"""
# default: compress_rate
if not compress_rate:
compress_rate = 0.2
if not unstable_ranges:
unstable_ranges = []
# draw
line = self._draw_line(classifier_result)
bar = self._draw_bar(classifier_result)
# merge charts
page = Page()
page.add(line)
page.add(bar)
# insert pictures
if cut_result:
# sim chart
sim_line = self._draw_sim(cut_result)
page.add(sim_line)
# mark range
for each in unstable_ranges:
classifier_result.mark_range_unstable(each.start, each.end)
offset = classifier_result.get_offset()
stage_range = classifier_result.get_stage_range()
for cur_index in range(len(stage_range)):
each = stage_range[cur_index]
middle = each[len(each) // 2]
if middle.is_stable():
label = self.LABEL_STABLE
frame = toolbox.compress_frame(middle.get_data(),
compress_rate=compress_rate)
else:
# todo: looks not good enough. `unspecific` looks a little weird but I have no idea now
if middle.stage == constants.UNKNOWN_STAGE_FLAG:
label = self.LABEL_UNSPECIFIC
else:
label = self.LABEL_UNSTABLE
# add a frame
if cur_index + 1 < len(stage_range):
new_each = [*each, stage_range[cur_index + 1][0]]
else:
new_each = each
frame = np.hstack([
toolbox.compress_frame(i.get_data(),
compress_rate=compress_rate)
for i in new_each
])
first, last = each[0], each[-1]
self.add_thumbnail(
f"{label} range {first.frame_id}({first.timestamp}) - {last.frame_id}({last.timestamp + offset}), "
f"duration: {last.timestamp - first.timestamp + offset}",
frame,
)
# calc time cost
cost_dict = classifier_result.calc_changing_cost()
# time stamp
timestamp = toolbox.get_timestamp_str()
# video
self.add_extra("video path", classifier_result.video_path)
self.add_extra("frame count", str(classifier_result.get_length()))
self.add_extra("offset between frames",
str(classifier_result.get_offset()))
# insert extras
template = Template(get_template())
template_content = template.render(
chart=Markup(page.render_embed()),
thumbnail_list=self.thumbnail_list,
extras=self.extra_dict,
background_color=constants.BACKGROUND_COLOR,
cost_dict=cost_dict,
timestamp=timestamp,
version_code=__VERSION__,
)
# default: write to current dir
default_name = f"{timestamp}.html"
if not report_path:
report_path = default_name
# somewhere specific
# existed dir?
elif os.path.isdir(report_path):
report_path = os.path.join(report_path, default_name)
logger.debug(f"trying to save report to {report_path}")
# write file
with open(report_path, "w", encoding=constants.CHARSET) as fh:
fh.write(template_content)
logger.info(f"save report to {report_path}")
def do(self, frame: VideoFrame, *_, **__) -> typing.Optional[VideoFrame]:
super().do(frame, *_, **__)
frame.data = toolbox.compress_frame(frame.data,
compress_rate=self.compress_rate,
target_size=self.target_size)
return frame
