def thumbnail(
self,
target_range: VideoCutRange,
to_dir: str = None,
compress_rate: float = None,
is_vertical: bool = None,
*_,
**__,
) -> np.ndarray:
"""
build a thumbnail, for easier debug or something else
:param target_range: VideoCutRange
:param to_dir: your thumbnail will be saved to this path
:param compress_rate: float, 0 - 1, about thumbnail's size, default to 0.1 (1/10)
:param is_vertical: direction
:return:
"""
if not compress_rate:
compress_rate = 0.1
# direction
if is_vertical:
stack_func = np.vstack
def get_split_line(f):
return np.zeros((5, f.shape[1]))
else:
stack_func = np.hstack
def get_split_line(f):
return np.zeros((f.shape[0], 5))
frame_list = list()
with toolbox.video_capture(self.video.path) as cap:
toolbox.video_jump(cap, target_range.start)
ret, frame = cap.read()
count = 1
length = target_range.get_length()
while ret and count <= length:
frame = toolbox.compress_frame(frame, compress_rate)
frame_list.append(frame)
frame_list.append(get_split_line(frame))
ret, frame = cap.read()
count += 1
merged = stack_func(frame_list)
# create parent dir
if to_dir:
target_path = os.path.join(
to_dir, f"thumbnail_{target_range.start}-{target_range.end}.png"
)
cv2.imwrite(target_path, merged)
logger.debug(f"save thumbnail to {target_path}")
return merged
def get_range(
self, limit: int = None, unstable_limit: int = None, **kwargs
) -> typing.Tuple[typing.List[VideoCutRange], typing.List[VideoCutRange]]:
"""
return stable_range_list and unstable_range_list
:param limit: ignore some ranges which are too short, 5 means ignore stable ranges which length < 5
:param unstable_limit: ignore some ranges which are too short, 5 means ignore unstable ranges which length < 5
:param kwargs:
threshold: float, 0-1, default to 0.95. decided whether a range is stable. larger => more unstable ranges
range_threshold:
same as threshold, but it decided whether a merged range is stable.
see https://github.com/williamfzc/stagesepx/issues/17 for details
offset:
it will change the way to decided whether two ranges can be merged
before: first_range.end == second_range.start
after: first_range.end + offset >= secord_range.start
:return:
"""
"""
videos have 4 kinds of status:
- stable start + stable end (usually)
- stable start + unstable end
- unstable start + stable end
- unstable start + unstable end
so, unstable range list can be:
- start > 0, end < frame_count
- start = 0, end < frame_count
- start > 0, end = frame_count
- start = 0, end = frame_count
"""
unstable_range_list = self.get_unstable_range(unstable_limit, **kwargs)
# it is not a real frame (not existed)
# just take it as a beginning
# real frame id is started with 1, with non-zero timestamp
video_start_frame_id = 0
video_start_timestamp = 0.0
video_end_frame_id = self.range_list[-1].end
video_end_timestamp = self.range_list[-1].end_time
# stable all the time
if len(unstable_range_list) == 0:
logger.warning(
"no unstable stage detected, seems nothing happened in your video"
)
return (
# stable
[
VideoCutRange(
self.video,
video_start_frame_id,
video_end_frame_id,
[1.0],
[0.0],
[0.0],
video_start_timestamp,
video_end_timestamp,
)
],
# unstable
[],
)
# IMPORTANT: +1 and -1 easily cause error
# end of first stable range == start of first unstable range
first_stable_range_end_id = unstable_range_list[0].start - 1
# start of last stable range == end of last unstable range
end_stable_range_start_id = unstable_range_list[-1].end + 1
# IMPORTANT: len(ssim_list) + 1 == video_end_frame_id
range_list: typing.List[VideoCutRange] = list()
# stable start
if first_stable_range_end_id >= 1:
logger.debug(f"stable start")
range_list.append(
VideoCutRange(
self.video,
video_start_frame_id,
first_stable_range_end_id,
[1.0],
[0.0],
[0.0],
video_start_timestamp,
self.get_target_range_by_id(first_stable_range_end_id).end_time,
)
)
# unstable start
else:
logger.debug("unstable start")
# stable end
if end_stable_range_start_id <= video_end_frame_id:
logger.debug("stable end")
range_list.append(
VideoCutRange(
self.video,
end_stable_range_start_id,
video_end_frame_id,
[1.0],
[0.0],
[0.0],
self.get_target_range_by_id(end_stable_range_start_id).end_time,
video_end_timestamp,
)
)
# unstable end
else:
logger.debug("unstable end")
# diff range
for i in range(len(unstable_range_list) - 1):
range_start_id = unstable_range_list[i].end + 1
range_end_id = unstable_range_list[i + 1].start - 1
# stable range's length is 1
if range_start_id > range_end_id:
range_start_id, range_end_id = range_end_id, range_start_id
range_list.append(
# IMPORTANT: frame's timestamp => end time of this frame
# because frame 0's timestamp is 0.0
# frame {range_start_id} end time - frame {range_end_id} end time
VideoCutRange(
self.video,
range_start_id,
range_end_id,
[1.0],
[0.0],
[0.0],
self.get_target_range_by_id(range_start_id).end_time,
self.get_target_range_by_id(range_end_id).end_time,
)
)
# remove some ranges, which is limit
if limit:
range_list = self._length_filter(range_list, limit)
logger.debug(f"stable range of [{self.video.path}]: {range_list}")
stable_range_list = sorted(range_list, key=lambda x: x.start)
return stable_range_list, unstable_range_list
def loads(cls, content: str) -> "VideoCutResult":
json_dict: dict = json.loads(content)
return cls(
VideoObject(**json_dict["video"]),
[VideoCutRange(**each) for each in json_dict["range_list"]],
)
def _convert_video_into_range_list(
self, video: VideoObject, block: int, window_size: int,
window_coefficient: int) -> typing.List[VideoCutRange]:
step = self.step
video_length = video.frame_count
class _Window(object):
def __init__(self):
self.start = 1
self.size = window_size
self.end = self.start + window_size * step
def load_data(self) -> typing.List[VideoFrame]:
cur = self.start
result = []
video_operator = video.get_operator()
while cur <= self.end:
frame = video_operator.get_frame_by_id(cur)
result.append(frame)
cur += step
# at least 2
if len(result) < 2:
last = video_operator.get_frame_by_id(self.end)
result.append(last)
return result
def shift(self) -> bool:
logger.debug(f"window before: {self.start}, {self.end}")
self.start += step
self.end += step
if self.start >= video_length:
# out of range
return False
# window end
if self.end >= video_length:
self.end = video_length
logger.debug(f"window after: {self.start}, {self.end}")
return True
def _float_merge(float_list: typing.List[float]) -> float:
# the first, the largest.
length = len(float_list)
result = 0.0
denominator = 0.0
for i, each in enumerate(float_list):
weight = pow(length - i, window_coefficient)
denominator += weight
result += each * weight
logger.debug(f"calc: {each} x {weight}")
final = result / denominator
logger.debug(f"calc final: {final} from {result} / {denominator}")
return final
range_list: typing.List[VideoCutRange] = list()
logger.info(
f"total frame count: {video_length}, size: {video.frame_size}")
window = _Window()
while True:
frame_list = window.load_data()
frame_list = [self._apply_hook(each) for each in frame_list]
# window loop
ssim_list = []
mse_list = []
psnr_list = []
cur_frame = frame_list[0]
first_target_frame = frame_list[1]
cur_frame_list = self.pic_split(cur_frame.data, block)
for each in frame_list[1:]:
each_frame_list = self.pic_split(each.data, block)
ssim, mse, psnr = self.compare_frame_list(
cur_frame_list, each_frame_list)
ssim_list.append(ssim)
mse_list.append(mse)
psnr_list.append(psnr)
logger.debug(
f"between {cur_frame.frame_id} & {each.frame_id}: ssim={ssim}; mse={mse}; psnr={psnr}"
)
ssim = _float_merge(ssim_list)
mse = _float_merge(mse_list)
psnr = _float_merge(psnr_list)
range_list.append(
VideoCutRange(
video,
start=cur_frame.frame_id,
end=first_target_frame.frame_id,
ssim=[ssim],
mse=[mse],
psnr=[psnr],
start_time=cur_frame.timestamp,
end_time=first_target_frame.timestamp,
))
continue_flag = window.shift()
if not continue_flag:
break
return range_list
def _convert_video_into_range_list(self,
video: VideoObject,
block: int = None,
*args,
**kwargs) -> typing.List[VideoCutRange]:
if not block:
block = 2
range_list: typing.List[VideoCutRange] = list()
with toolbox.video_capture(video.path) as cap:
logger.debug(
f'total frame count: {video.frame_count}, size: {video.frame_size}'
)
# load the first two frames
_, start = cap.read()
start_frame_id = toolbox.get_current_frame_id(cap)
start_frame_time = toolbox.get_current_frame_time(cap)
toolbox.video_jump(cap, self.step + 1)
ret, end = cap.read()
end_frame_id = toolbox.get_current_frame_id(cap)
end_frame_time = toolbox.get_current_frame_time(cap)
# hook
start = self._apply_hook(start_frame_id, start)
# check block
if not self.is_block_valid(start, block):
logger.warning(
'array split does not result in an equal division, set block to 1'
)
block = 1
while ret:
# hook
end = self._apply_hook(end_frame_id, end, *args, **kwargs)
logger.debug(
f'computing {start_frame_id}({start_frame_time}) & {end_frame_id}({end_frame_time}) ...'
)
start_part_list = self.pic_split(start, block)
end_part_list = self.pic_split(end, block)
# find the min ssim and the max mse / psnr
ssim = 1.
mse = 0.
psnr = 0.
for part_index, (each_start, each_end) in enumerate(
zip(start_part_list, end_part_list)):
part_ssim = toolbox.compare_ssim(each_start, each_end)
if part_ssim < ssim:
ssim = part_ssim
# mse is very sensitive
part_mse = toolbox.calc_mse(each_start, each_end)
if part_mse > mse:
mse = part_mse
part_psnr = toolbox.calc_psnr(each_start, each_end)
if part_psnr > psnr:
psnr = part_psnr
logger.debug(
f'part {part_index}: ssim={part_ssim}; mse={part_mse}; psnr={part_psnr}'
)
logger.debug(
f'between {start_frame_id} & {end_frame_id}: ssim={ssim}; mse={mse}; psnr={psnr}'
)
range_list.append(
VideoCutRange(
video,
start=start_frame_id,
end=end_frame_id,
ssim=[ssim],
mse=[mse],
psnr=[psnr],
start_time=start_frame_time,
end_time=end_frame_time,
))
# load the next one
start = end
start_frame_id, end_frame_id = end_frame_id, end_frame_id + self.step
start_frame_time = end_frame_time
toolbox.video_jump(cap, end_frame_id)
ret, end = cap.read()
end_frame_time = toolbox.get_current_frame_time(cap)
return range_list
def _convert_video_into_range_list(self,
video: VideoObject,
block: int = None,
*args,
**kwargs) -> typing.List[VideoCutRange]:
range_list: typing.List[VideoCutRange] = list()
logger.info(
f"total frame count: {video.frame_count}, size: {video.frame_size}"
)
# load the first two frames
video_operator = video.get_operator()
cur_frame = video_operator.get_frame_by_id(1)
next_frame = video_operator.get_frame_by_id(1 + self.step)
# hook
cur_frame.data = self._apply_hook(cur_frame.frame_id, cur_frame.data)
# check block
if not block:
block = 2
if not self.is_block_valid(cur_frame.data, block):
logger.warning(
"array split does not result in an equal division, set block to 1"
)
block = 1
while True:
# hook
next_frame.data = self._apply_hook(next_frame.frame_id,
next_frame.data, *args,
**kwargs)
logger.debug(
f"computing {cur_frame.frame_id}({cur_frame.timestamp}) & {next_frame.frame_id}({next_frame.timestamp}) ..."
)
start_part_list = self.pic_split(cur_frame.data, block)
end_part_list = self.pic_split(next_frame.data, block)
# find the min ssim and the max mse / psnr
ssim = 1.0
mse = 0.0
psnr = 0.0
for part_index, (each_start, each_end) in enumerate(
zip(start_part_list, end_part_list)):
part_ssim = toolbox.compare_ssim(each_start, each_end)
if part_ssim < ssim:
ssim = part_ssim
# mse is very sensitive
part_mse = toolbox.calc_mse(each_start, each_end)
if part_mse > mse:
mse = part_mse
part_psnr = toolbox.calc_psnr(each_start, each_end)
if part_psnr > psnr:
psnr = part_psnr
logger.debug(
f"part {part_index}: ssim={part_ssim}; mse={part_mse}; psnr={part_psnr}"
)
logger.debug(
f"between {cur_frame.frame_id} & {next_frame.frame_id}: ssim={ssim}; mse={mse}; psnr={psnr}"
)
range_list.append(
VideoCutRange(
video,
start=cur_frame.frame_id,
end=next_frame.frame_id,
ssim=[ssim],
mse=[mse],
psnr=[psnr],
start_time=cur_frame.timestamp,
end_time=next_frame.timestamp,
))
# load the next one
cur_frame = next_frame
next_frame = video_operator.get_frame_by_id(next_frame.frame_id +
self.step)
if next_frame is None:
break
return range_list
def loads(cls, content: str) -> 'VideoCutResult':
json_dict: dict = json.loads(content)
return cls(
VideoObject(**json_dict['video']),
[VideoCutRange(**each) for each in json_dict['range_list']]
)