def save_image(fname, img, codec='bmp', pix_fmt='', lib_opts={}):
"""Saves the given image to disk in the format requested.
:param fname: The filename where to save the image.
:param img: The :class:`ffpyplayer.pic.Image` to save.
:param codec: The codec to pass to
:class:`ffpyplayer.writer.MediaWriter` that determines the image
type. Defaults to 'bmp'.
:param pix_fmt: The pixel format into which to convert the image before
saving. If empty, the original pixel format is used. If the
codec doesn't support the image format, we first convert it to the
closest supported format.
:param lib_opts: Any additional `lib_opts` options to pass to
:class:`ffpyplayer.writer.MediaWriter`.
:return: The estimated size of the image on disk.
"""
fmt = img.get_pixel_format()
w, h = img.get_size()
if not codec:
codec = get_format_codec(fname)
ofmt = get_supported_pixfmts(codec, fmt)[0]
else:
ofmt = get_supported_pixfmts(codec, pix_fmt or fmt)[0]
if ofmt != fmt:
sws = SWScale(w, h, fmt, ofmt=ofmt)
img = sws.scale(img)
fmt = ofmt
out_opts = {'pix_fmt_in': fmt, 'width_in': w, 'height_in': h,
'frame_rate': (30, 1), 'codec': codec}
writer = MediaWriter(fname, [out_opts], lib_opts=lib_opts)
size = writer.write_frame(img=img, pts=0, stream=0)
writer.close()
return size
def test_write_smaller_than_frame_rate(tmp_path, fmt):
from ffpyplayer.writer import MediaWriter
fname = str(tmp_path / 'test_frame.') + fmt[0]
w, h = 64, 64
out_opts = {
'pix_fmt_in': 'rgb24',
'width_in': w,
'height_in': h,
'codec': 'rawvideo',
'pix_fmt_out': 'yuv420p',
'frame_rate': (30, 1)
}
writer = MediaWriter(fname, [out_opts], fmt=fmt[1])
img = get_image(w, h)
if fmt[0] == 'avi':
with pytest.raises(Exception):
for i in range(20):
writer.write_frame(img=img, pts=i / 300, stream=0)
else:
for i in range(20):
writer.write_frame(img=img, pts=i / 300, stream=0)
writer.close()
def test_write_larger_than_frame_rate(tmp_path, fmt):
from ffpyplayer.writer import MediaWriter
fname = str(tmp_path / 'test_frame.') + fmt[0]
w, h = 64, 64
out_opts = {
'pix_fmt_in': 'gray',
'width_in': w,
'height_in': h,
'codec': 'rawvideo',
'frame_rate': (15, 1)
}
writer = MediaWriter(fname, [out_opts], fmt=fmt[1])
timestamps = []
image_vals = []
for i in range(20):
timestamps.append(i)
image_vals.append(i * 5)
writer.write_frame(img=get_gray_image_with_val(w, h, i * 5),
pts=i,
stream=0)
writer.close()
verify_frames(fname, timestamps, image_vals)
def test_play(self):
import ffpyplayer.tests.common
from ffpyplayer.writer import MediaWriter
from ffpyplayer.tools import get_supported_pixfmts, get_supported_framerates
from ffpyplayer.pic import Image
w, h = 640, 480
out_opts = {
'pix_fmt_in': 'rgb24', 'width_in': w, 'height_in': h,
'codec': 'libx264', 'frame_rate': (5, 1)}
lib_opts = {'preset':'slow', 'crf':'22'}
metadata = {'title':'Singing in the sun', 'author':'Rat', 'genre':'Animal sounds'}
writer = MediaWriter(fname, [out_opts] * 2, fmt='mp4',
width_out=w/2, height_out=h/2, pix_fmt_out='yuv420p',
lib_opts=lib_opts, metadata=metadata)
# Construct images
size = w * h * 3
buf = bytearray([int(x * 255 / size) for x in range(size)])
img = Image(plane_buffers=[buf], pix_fmt='rgb24', size=(w, h))
buf = bytearray([int((size - x) * 255 / size) for x in range(size)])
img2 = Image(plane_buffers=[buf], pix_fmt='rgb24', size=(w, h))
for i in range(20):
writer.write_frame(img=img, pts=i / 5., stream=0) # stream 1
writer.write_frame(img=img2, pts=i / 5., stream=1) # stream 2
def video_file(tmp_path_factory):
from ffpyplayer.writer import MediaWriter
from ffpyplayer.pic import Image
fname = str(tmp_path_factory.mktemp('data') / 'test_video.avi')
w, h = 64, 64
size = w * h
out_opts = {
'pix_fmt_in': 'gray',
'width_in': w,
'height_in': h,
'codec': 'rawvideo',
'frame_rate': (2997, 100)
}
buf = bytearray([int(x * 255 / size) for x in range(size)])
buf2 = bytearray([0] * size)
img = Image(plane_buffers=[buf, buf2], pix_fmt='gray', size=(w, h))
writer = MediaWriter(fname, [out_opts])
for i in range(20):
writer.write_frame(img=img, pts=i / 29.97, stream=0)
writer.close()
return fname
def save_video_thumbnail(source, output):
"""Saves thumbnail of the given video under the given name"""
player = MediaPlayer(source, ff_opts={'ss': 1.0})
frame, val = None, None
while not frame:
frame, val = player.get_frame(force_refresh=True)
player.close_player()
if val == 'eof':
return None
elif frame is None:
return None
else:
img = frame[0]
pixel_format = img.get_pixel_format()
img_size = img.get_size()
thumb_size = 256, int(img_size[1] * 256 / img_size[0])
codec = 'tiff'
output_format = get_supported_pixfmts(codec, pixel_format)[0]
#resize and convert into the best pixel format
sws = SWScale(img_size[0], img_size[1], pixel_format, thumb_size[0],
thumb_size[1], output_format)
thumbnail = sws.scale(img)
streams = [{
'pix_fmt_in': output_format,
'width_in': thumb_size[0],
'height_in': thumb_size[1],
'codec': codec,
'frame_rate': (30, 1)
}]
writer = MediaWriter(output,
streams,
lib_opts={'compression_algo': 'lzw'})
writer.write_frame(img=thumbnail, pts=0, stream=0)
writer.close()
def test_write_streams(tmp_path):
from ffpyplayer.writer import MediaWriter
from ffpyplayer.tools import get_supported_pixfmts, get_supported_framerates
from ffpyplayer.pic import Image
from ffpyplayer.tools import get_codecs
fname = str(tmp_path / 'test_video.avi')
lib_opts = {}
codec = 'rawvideo'
if 'libx264' in get_codecs(encode=True, video=True):
codec = 'libx264'
lib_opts = {'preset': 'slow', 'crf': '22'}
w, h = 640, 480
out_opts = {
'pix_fmt_in': 'rgb24',
'width_in': w,
'height_in': h,
'codec': codec,
'frame_rate': (5, 1)
}
metadata = {
'title': 'Singing in the sun',
'author': 'Rat',
'genre': 'Animal sounds'
}
writer = MediaWriter(fname, [out_opts] * 2,
fmt='mp4',
width_out=w / 2,
height_out=h / 2,
pix_fmt_out='yuv420p',
lib_opts=lib_opts,
metadata=metadata)
# Construct images
size = w * h * 3
buf = bytearray([int(x * 255 / size) for x in range(size)])
img = Image(plane_buffers=[buf], pix_fmt='rgb24', size=(w, h))
buf = bytearray([int((size - x) * 255 / size) for x in range(size)])
img2 = Image(plane_buffers=[buf], pix_fmt='rgb24', size=(w, h))
for i in range(20):
writer.write_frame(img=img, pts=i / 5., stream=0) # stream 1
writer.write_frame(img=img2, pts=i / 5., stream=1) # stream 2
writer.close()
def generate_movie(self,
filename,
out_fmt='yuv420p',
codec='libx264',
lib_opts={'crf': '0'},
video_fmt='mp4',
start=None,
end=None,
canvas_size=(0, 0),
canvas_size_hint=(1, 1),
projector_pos=(0, 0),
projector_pos_hint=(None, None),
paint_funcs=(),
stimulation_transparency=1.,
lum=1.,
speed=1.,
hidden_shapes=None):
from kivy.graphics import (Canvas, Translate, Fbo, ClearColor,
ClearBuffers, Scale)
from kivy.core.window import Window
rate = float(self.view_controller.frame_rate)
rate_int = int(rate)
if rate != rate_int:
raise ValueError('Frame rate should be integer')
orig_w, orig_h = (self.view_controller.screen_width,
self.view_controller.screen_height)
canvas_w, canvas_h = canvas_size
cv_hint_w, cv_hint_h = canvas_size_hint
w = int(canvas_w if cv_hint_w is None else orig_w * cv_hint_w)
h = int(canvas_h if cv_hint_h is None else orig_h * cv_hint_h)
projector_x, projector_y = projector_pos
projector_hint_x, projector_hint_y = projector_pos_hint
x = int(projector_x if projector_hint_x is None else orig_w *
projector_hint_x)
y = int(projector_y if projector_hint_y is None else orig_h *
projector_hint_y)
Window.size = w, h
intensities = self.shapes_intensity
n = len(intensities[next(iter(intensities.keys()))])
if start is not None:
start = int(start * rate)
if start >= n:
raise Exception('Start time is after the end of the data')
else:
start = 0
if end is not None:
end = int(math.ceil(end * rate)) + 1
if end <= start:
raise Exception('End time is before or at the start time')
else:
end = n
stream = {
'pix_fmt_in': 'rgba',
'pix_fmt_out': out_fmt,
'width_in': w,
'height_in': h,
'width_out': w,
'height_out': h,
'codec': codec,
'frame_rate': (int(speed * rate_int), 1)
}
writer = MediaWriter(filename, [stream],
fmt=video_fmt,
lib_opts=lib_opts)
fbo = Fbo(size=(w, h), with_stencilbuffer=True)
with fbo:
ClearColor(0, 0, 0, 1)
ClearBuffers()
Scale(1, -1, 1)
Translate(0, -h, 0)
config = {
'canvas': fbo,
'pos': (x, y),
'size': (w, h),
'orig_size': (orig_w, orig_h),
'rate': rate
}
paint_funcs = [func(config) for func in paint_funcs]
paint_funcs = [func for func in paint_funcs if func is not None]
fbo.draw()
img = Image(plane_buffers=[fbo.pixels], pix_fmt='rgba', size=(w, h))
writer.write_frame(img, 0.)
fbo.add(Translate(x, y))
shape_views = self.stage_factory.get_shapes_gl_color_instructions(
fbo, 'stage_replay')
fbo.add(Translate(-x, -y))
pbar = tqdm(total=(end - 1 - start) / rate,
file=sys.stdout,
unit='second',
unit_scale=1)
# all shapes listed in intensities must be in shape_views. However,
# we don't want to show shapes not given values in intensities or if
# they are to be hidden
unused_shapes = set(shape_views) - set(intensities)
unused_shapes.update(set(hidden_shapes or []))
for name in unused_shapes:
if name in shape_views:
shape_views[name].rgba = 0, 0, 0, 0
for i in range(start, end):
pbar.update(1 / rate)
for name, intensity in intensities.items():
r, g, b, a = intensity[i]
if name in unused_shapes:
a = 0
shape_views[name].rgba = \
r * lum, g * lum, b * lum, a * stimulation_transparency
try:
for func in paint_funcs:
func(i)
except EndOfDataException:
break
fbo.draw()
img = Image(plane_buffers=[fbo.pixels],
pix_fmt='rgba',
size=(w, h))
writer.write_frame(img, (i - start + 1) / (rate * speed))
pbar.close()
def record_thread_run(self, filename):
queue = self.image_queue
recorder = None
t0 = None
last_t = None
while self.record_state != 'stopping':
item = queue.get()
if item == 'eof':
break
img, metadata = item
if recorder is None:
try:
self.setattr_in_kivy_thread('ts_record', clock())
t0 = metadata['t']
iw, ih = img.get_size()
ipix_fmt = img.get_pixel_format()
(ifmt, iw, ih, irate), (opix_fmt, ow, oh, orate) = \
self.compute_recording_opts(ipix_fmt, iw, ih)
self.setattr_in_kivy_thread(
'metadata_record_used',
VideoMetadata(opix_fmt, ow, oh, orate))
orate = Fraction(orate)
if orate >= 1.:
orate = Fraction(orate.denominator, orate.numerator)
orate = orate.limit_denominator(2 ** 30 - 1)
orate = (orate.denominator, orate.numerator)
else:
orate = orate.limit_denominator(2 ** 30 - 1)
orate = (orate.numerator, orate.denominator)
stream = {
'pix_fmt_in': ipix_fmt, 'pix_fmt_out': opix_fmt,
'width_in': iw, 'height_in': ih, 'width_out': ow,
'height_out': oh, 'codec': 'rawvideo',
'frame_rate': orate}
recorder = MediaWriter(filename, [stream])
except Exception as e:
self.exception(e)
Clock.schedule_once(self.complete_stop)
return
Clock.schedule_once(self.complete_start)
try:
self.setattr_in_kivy_thread(
'size_recorded',
recorder.write_frame(img, metadata['t'] - t0))
self.increment_in_kivy_thread('frames_recorded')
except Exception as e:
self.exception(e)
self.increment_in_kivy_thread('frames_skipped')
if recorder is not None:
try:
recorder.close()
except Exception as e:
self.exception(e)
Clock.schedule_once(self.complete_stop)
video = './../video/1x01.mkv'
save_dir = './../frame/1x01/'
# create image
w, h = 1280, 720
fmt = 'rgb24'
codec = 'png' # we'll encode it using the tiff codec
out_opts = {
'pix_fmt_in': fmt,
'width_in': w,
'height_in': h,
'frame_rate': (30, 1),
'codec': codec
}
player = MediaPlayer(video)
val = ''
while val != 'eof':
frame, val = player.get_frame()
if val != 'eof' and frame is not None:
img, t = frame
frame_file = 'frame' + str(int(t)) + '.png'
file = save_dir + frame_file
print(str(t))
buf = img.to_bytearray()
img = Image(plane_buffers=[buf[0]], pix_fmt=fmt, size=(w, h))
writer = MediaWriter(file, [out_opts])
writer.write_frame(img=img, pts=0, stream=0)
writer.close()
buffer_size = 3686400 # get form fb_fix_screeninfo.smem_len
h = buffer_size / line_length
codec = 'png'
file = open(sys.argv[1], "rb")
buf = bytearray(file.read())
img = Image(plane_buffers=[buf], pix_fmt=fmt, size=(w, h))
# make sure the output codec supports the input pixel format type
# otherwise, convert it to the best pixel format
ofmt = get_supported_pixfmts(codec, fmt)[0]
if ofmt != fmt:
sws = SWScale(w, h, fmt, ofmt=ofmt)
img = sws.scale(img)
fmt = ofmt
out_opts = {
'pix_fmt_in': fmt,
'width_in': w,
'height_in': h,
'frame_rate': (30, 1),
'codec': codec
}
writer = MediaWriter('output.' + codec, [out_opts])
writer.write_frame(img=img, pts=0, stream=0)
writer.close()
file.close()
def main(_):
with tf.Session() as sess:
config = get_config(FLAGS)
env = MyEnvironment(config)
agent = Agent(config, env, sess)
scale = 1
# 1. first probe file, get metadata
in_file = config.input_name
out_file = config.output_name
convert_num = -1
ff_opts = {
'out_fmt': 'yuv444p',
'framedrop': False,
'an': True,
'sn': True,
}
player = MediaPlayer(in_file, ff_opts=ff_opts)
# must wait for probe result, strange
while player.get_metadata()['src_vid_size'] == (0, 0):
time.sleep(0.01)
meta = player.get_metadata()
width = meta['src_vid_size'][0]
height = meta['src_vid_size'][1]
width_out = width * scale
height_out = height * scale
out_opts = {
'pix_fmt_in': 'yuv444p',
'pix_fmt_out': 'yuv420p',
'width_in': width_out,
'height_in': height_out,
'frame_rate': meta['frame_rate'],
'codec': 'libx264',
#'acpect': '4:3',
}
lib_opts = {
# config for BT.2020 HDR10
# 'x265-params': 'range=pc:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc:crf=15',
# config for x264 to encode video
'x264-params': 'crf=15',
}
writer = MediaWriter(out_file, [out_opts],
lib_opts=lib_opts,
overwrite=True)
frame_count = 0
start_timestamp = 0
while True:
frame, val = player.get_frame()
if val == 'eof':
print('end of video')
break
elif frame is None:
time.sleep(0.01)
else:
t1 = time.time() * 1000
img, t = frame
if frame_count == 0:
start_timestamp = t
bufs = img.to_bytearray()
assert len(bufs) >= 3
Y = np.frombuffer(bufs[0], dtype=np.uint8)
U = np.frombuffer(bufs[1], dtype=np.uint8)
V = np.frombuffer(bufs[2], dtype=np.uint8)
input_YUV = cv2.merge([Y, U, V])
img = cv2.cvtColor(input_YUV, cv2.COLOR_YUV2RGB)
img = np.array(img).reshape(height, width, 3)
outputImg = agent.test_video(img)
out = np.array(outputImg).reshape(height_out * width_out, 1, 3)
YUV = cv2.cvtColor(out, cv2.COLOR_RGB2YUV)
(Y, U, V) = cv2.split(YUV)
bufs = []
bufs.append(Y.tobytes())
bufs.append(U.tobytes())
bufs.append(V.tobytes())
outputImg = Image(plane_buffers=bufs,
pix_fmt='yuv444p',
size=(width_out, height_out))
t = t - start_timestamp
writer.write_frame(img=outputImg, pts=t, stream=0)
t2 = time.time() * 1000
frame_count += 1
if (frame_count % 30 == 0):
print('convert frame # ', frame_count)
#print('--pts:', t)
if frame_count >= convert_num > 0:
break
# if frame_count >= 1800:
# break
# print("time: ", time.time()*1000-tt)
player.close_player()
writer.close()
def generate_movie(
self, filename, out_fmt='yuv420p', codec='libx264',
lib_opts={'crf': '0'}, video_fmt='mp4', start=None, end=None,
canvas_size=(0, 0),
canvas_size_hint=(1, 1), projector_pos=(0, 0),
projector_pos_hint=(None, None), paint_funcs=(),
stimulation_transparency=1., lum=1., speed=1.):
from kivy.graphics import (
Canvas, Translate, Fbo, ClearColor, ClearBuffers, Scale)
from kivy.core.window import Window
rate = float(self.view_controller.frame_rate)
rate_int = int(rate)
if rate != rate_int:
raise ValueError('Frame rate should be integer')
orig_w, orig_h = (
self.view_controller.screen_width,
self.view_controller.screen_height)
canvas_w, canvas_h = canvas_size
cv_hint_w, cv_hint_h = canvas_size_hint
w = int(canvas_w if cv_hint_w is None else orig_w * cv_hint_w)
h = int(canvas_h if cv_hint_h is None else orig_h * cv_hint_h)
projector_x, projector_y = projector_pos
projector_hint_x, projector_hint_y = projector_pos_hint
x = int(projector_x if projector_hint_x is None else
orig_w * projector_hint_x)
y = int(projector_y if projector_hint_y is None else
orig_h * projector_hint_y)
Window.size = w, h
intensities = self.shapes_intensity
n = len(intensities[next(iter(intensities.keys()))])
if start is not None:
start = int(start * rate)
if start >= n:
raise Exception('Start time is after the end of the data')
else:
start = 0
if end is not None:
end = int(math.ceil(end * rate)) + 1
if end <= start:
raise Exception('End time is before or at the start time')
else:
end = n
stream = {
'pix_fmt_in': 'rgba', 'pix_fmt_out': out_fmt,
'width_in': w, 'height_in': h, 'width_out': w,
'height_out': h, 'codec': codec,
'frame_rate': (int(speed * rate_int), 1)}
writer = MediaWriter(
filename, [stream], fmt=video_fmt, lib_opts=lib_opts)
fbo = Fbo(size=(w, h), with_stencilbuffer=True)
with fbo:
ClearColor(0, 0, 0, 1)
ClearBuffers()
Scale(1, -1, 1)
Translate(0, -h, 0)
config = {'canvas': fbo, 'pos': (x, y), 'size': (w, h),
'orig_size': (orig_w, orig_h), 'rate': rate}
paint_funcs = [func(config) for func in paint_funcs]
paint_funcs = [func for func in paint_funcs if func is not None]
fbo.draw()
img = Image(plane_buffers=[fbo.pixels], pix_fmt='rgba', size=(w, h))
writer.write_frame(img, 0.)
fbo.add(Translate(x, y))
shape_views = self.stage_factory.get_shapes_gl_color_instructions(
fbo, 'stage_replay')
fbo.add(Translate(-x, -y))
pbar = tqdm(
total=(end - 1 - start) / rate, file=sys.stdout, unit='second',
unit_scale=1)
for i in range(start, end):
pbar.update(1 / rate)
for name, intensity in intensities.items():
r, g, b, a = intensity[i]
if not r and not g and not b:
a = 0
shape_views[name].rgba = \
r * lum, g * lum, b * lum, a * stimulation_transparency
try:
for func in paint_funcs:
func(i)
except EndOfDataException:
break
fbo.draw()
img = Image(plane_buffers=[fbo.pixels], pix_fmt='rgba', size=(w, h))
writer.write_frame(img, (i - start + 1) / (rate * speed))
pbar.close()