def test_haldclut_graph(self):
raise SkipTest()
graph = Graph()
img = Image.open(fate_suite("png1/lena-rgb24.png"))
frame = VideoFrame.from_image(img)
img_source = graph.add_buffer(frame)
hald_img = Image.open("hald_7.png")
hald_frame = VideoFrame.from_image(hald_img)
hald_source = graph.add_buffer(hald_frame)
hald_filter = graph.add("haldclut")
sink = graph.add("buffersink")
img_source.link(0, hald_filter, 0)
hald_source.link(0, hald_filter, 1)
hald_filter.link(0, sink, 0)
graph.config()
self.assertIs(img_source.outputs[0].linked_to, hald_filter.inputs[0])
self.assertIs(hald_source.outputs[0].linked_to, hald_filter.inputs[1])
self.assertIs(hald_filter.outputs[0].linked_to, sink.inputs[0])
hald_source.push(hald_frame)
img_source.push(frame)
frame = sink.pull()
self.assertIsInstance(frame, VideoFrame)
frame.to_image().save(self.sandboxed("filtered.png"))
async def recv(self):
frame = await self.track.recv()
self.counter += 1
if self.transform == 'edges':
# perform edge detection
img = frame.to_ndarray(format='bgr24')
img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)
# rebuild a VideoFrame, preserving timing information
new_frame = VideoFrame.from_ndarray(img, format='bgr24')
new_frame.pts = frame.pts
new_frame.time_base = frame.time_base
return new_frame
elif self.transform == 'rotate':
# rotate image
img = frame.to_ndarray(format='bgr24')
rows, cols, _ = img.shape
M = cv2.getRotationMatrix2D((cols / 2, rows / 2), frame.time * 45,
1)
img = cv2.warpAffine(img, M, (cols, rows))
# rebuild a VideoFrame, preserving timing information
new_frame = VideoFrame.from_ndarray(img, format='bgr24')
new_frame.pts = frame.pts
new_frame.time_base = frame.time_base
return new_frame
else:
return frame
def _YUVtoVFrame(Y, U, V, format=None):
assert Y.ndim == 2, "Y.ndim must be 2."
assert U.ndim == 2, "U.ndim must be 2."
assert V.ndim == 2, "V.ndim must be 2."
assert Y.shape[0] % 2 == 0, "Y.shape[0] must be even."
assert Y.shape[1] % 2 == 0, "Y.shape[1] must be even."
assert U.shape == V.shape, "V.shape must be equal to U.shape."
H, W = Y.shape
h, w = U.shape
if 2 * h == H and 2 * w == W:
if format is None:
format = "yuv420p"
A = numpy.concatenate((Y.reshape(H * W), U.reshape(h * w),
V.reshape(h * w))).reshape(3 * H // 2, W)
return VideoFrame.from_ndarray(A, format=format)
elif h == H and 2 * w == W:
if format is None:
format = "yuyv422"
UV = numpy.concatenate(
(U, V), axis=1).reshape(-1, 2, w).swapaxes(1, 2).reshape(-1, W)
A = numpy.moveaxis((Y, UV), 0, 2)
return VideoFrame.from_ndarray(A, format=format)
def test_reformat_pts(self):
frame = VideoFrame(640, 480, 'rgb24')
frame.pts = 123
frame.time_base = '456/1' # Just to be different.
frame = frame.reformat(320, 240)
self.assertEqual(frame.pts, 123)
self.assertEqual(frame.time_base, 456)
def test_to_image_rgb24(self):
sizes = [
(318, 238),
(320, 240),
(500, 500),
]
for width, height in sizes:
frame = VideoFrame(width, height, format='rgb24')
# fill video frame data
for plane in frame.planes:
ba = bytearray(plane.buffer_size)
pos = 0
for row in range(height):
for i in range(plane.line_size):
ba[pos] = i % 256
pos += 1
plane.update(ba)
# construct expected image data
expected = bytearray(height * width * 3)
pos = 0
for row in range(height):
for i in range(width * 3):
expected[pos] = i % 256
pos += 1
img = frame.to_image()
self.assertEqual(img.size, (width, height))
self.assertEqual(img.tobytes(), expected)
async def recv(self):
frame = await self.track.recv()
if self.transform == "cartoon":
img = frame.to_ndarray(format="bgr24")
# prepare color
img_color = cv2.pyrDown(cv2.pyrDown(img))
for _ in range(6):
img_color = cv2.bilateralFilter(img_color, 9, 9, 7)
img_color = cv2.pyrUp(cv2.pyrUp(img_color))
# prepare edges
img_edges = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
img_edges = cv2.adaptiveThreshold(
cv2.medianBlur(img_edges, 7),
255,
cv2.ADAPTIVE_THRESH_MEAN_C,
cv2.THRESH_BINARY,
9,
2,
)
img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)
# combine color and edges
img = cv2.bitwise_and(img_color, img_edges)
# rebuild a VideoFrame, preserving timing information
new_frame = VideoFrame.from_ndarray(img, format="bgr24")
new_frame.pts = frame.pts
new_frame.time_base = frame.time_base
return new_frame
elif self.transform == "edges":
# perform edge detection
img = frame.to_ndarray(format="bgr24")
img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)
# rebuild a VideoFrame, preserving timing information
new_frame = VideoFrame.from_ndarray(img, format="bgr24")
new_frame.pts = frame.pts
new_frame.time_base = frame.time_base
return new_frame
elif self.transform == "rotate":
# rotate image
img = frame.to_ndarray(format="bgr24")
rows, cols, _ = img.shape
M = cv2.getRotationMatrix2D((cols / 2, rows / 2), frame.time * 45,
1)
img = cv2.warpAffine(img, M, (cols, rows))
# rebuild a VideoFrame, preserving timing information
new_frame = VideoFrame.from_ndarray(img, format="bgr24")
new_frame.pts = frame.pts
new_frame.time_base = frame.time_base
return new_frame
else:
return frame
# end of file
def create_video_frame(self, width, height, pts, format='yuv420p', time_base=VIDEO_TIME_BASE):
"""
Create a single blank video frame.
"""
frame = VideoFrame(width=width, height=height, format=format)
for p in frame.planes:
p.update(bytes(p.buffer_size))
frame.pts = pts
frame.time_base = time_base
return frame
def test_basic_to_nd_array(self):
frame = VideoFrame(640, 480, 'rgb24')
with warnings.catch_warnings(record=True) as recorded:
array = frame.to_nd_array()
self.assertEqual(array.shape, (480, 640, 3))
# check deprecation warning
self.assertEqual(len(recorded), 1)
self.assertEqual(recorded[0].category, AttributeRenamedWarning)
self.assertEqual(
str(recorded[0].message),
'VideoFrame.to_nd_array is deprecated; please use VideoFrame.to_ndarray.'
)
async def recv(self) -> Frame:
"""
Receive the next :class:`~av.video.frame.VideoFrame`.
The base implementation just reads a 640x480 green frame at 30fps,
subclass :class:`VideoStreamTrack` to provide a useful implementation.
"""
pts, time_base = await self.next_timestamp()
frame = VideoFrame(width=640, height=480)
for p in frame.planes:
p.update(bytes(p.buffer_size))
frame.pts = pts
frame.time_base = time_base
return frame
async def recv(self):
global im
pts, time_base = await self.next_timestamp()
# create video frame
if im == None:
frame = VideoFrame.from_ndarray(self.img, format="bgr24")
frame.pts = pts
frame.time_base = time_base
else:
frame = VideoFrame.from_ndarray(im, format="bgr24")
frame.pts = pts
frame.time_base = time_base
return frame
def test_reformat_pixel_format_align(self):
height = 480
for width in range(2, 258, 2):
frame_yuv = VideoFrame(width, height, "yuv420p")
for plane in frame_yuv.planes:
plane.update(b"\xff" * plane.buffer_size)
expected_rgb = numpy.zeros(shape=(height, width, 3),
dtype=numpy.uint8)
expected_rgb[:, :, 0] = 255
expected_rgb[:, :, 1] = 124
expected_rgb[:, :, 2] = 255
frame_rgb = frame_yuv.reformat(format="rgb24")
self.assertNdarraysEqual(frame_rgb.to_ndarray(), expected_rgb)
def recv(self, frame: av.VideoFrame) -> av.VideoFrame:
img = frame.to_ndarray(format="bgr24")
# perform edge detection
img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)
return av.VideoFrame.from_ndarray(img, format="bgr24")
async def recv(self):
frame = await self.track.recv()
self.transform = "edges"
if self.transform == "edges":
# perform edge detection
img = frame.to_ndarray(format="bgr24")
# img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)
#
# # rebuild a VideoFrame, preserving timing information
# new_frame = VideoFrame.from_ndarray(img, format="bgr24")
# new_frame.pts = frame.pts
# new_frame.time_base = frame.time_base
logo = cv2.imread('watermark.png')
logo = cv2.resize(logo, (100, 60)) # width, height
# Create a mask of logo
img2gray = cv2.cvtColor(logo, cv2.COLOR_BGR2GRAY)
ret, mask = cv2.threshold(img2gray, 1, 255, cv2.THRESH_BINARY)
# Region of Image (ROI), where we want to insert logo
roi = img[
-60 - 10:-10, -100 - 10:
-10] # height box, shift bottom:top, width box, shift right:left
# NOTE it seems that shift values must be equal
# Set an index of where the mask is
roi[np.where(mask)] = 0
roi += logo
# rebuild a VideoFrame, preserving timing information
new_frame = VideoFrame.from_ndarray(img, format="bgr24")
new_frame.pts = frame.pts
new_frame.time_base = frame.time_base
return new_frame
else:
return frame
def write_rgb_rotate(output):
if not Image:
raise SkipTest()
output.metadata['title'] = 'container'
output.metadata['key'] = 'value'
stream = output.add_stream("mpeg4", 24)
stream.width = WIDTH
stream.height = HEIGHT
stream.pix_fmt = "yuv420p"
for frame_i in range(DURATION):
frame = VideoFrame(WIDTH, HEIGHT, 'rgb24')
image = Image.new('RGB', (WIDTH, HEIGHT), (
int(255 *
(0.5 + 0.5 * math.sin(frame_i / DURATION * 2 * math.pi))),
int(255 * (0.5 + 0.5 * math.sin(frame_i / DURATION * 2 * math.pi +
2 / 3 * math.pi))),
int(255 * (0.5 + 0.5 * math.sin(frame_i / DURATION * 2 * math.pi +
4 / 3 * math.pi))),
))
frame.planes[0].update(image.tobytes())
for packet in stream.encode(frame):
output.mux(packet)
for packet in stream.encode(None):
output.mux(packet)
# Done!
output.close()
async def recv(self):
pts, time_base = await self.next_timestamp()
ret, frame = self.cap.read()
if not ret:
return
self.tex = self.ctx.texture((self.width, self.height),
components=3,
data=frame.tobytes())
self.tex.use()
self.ctx.clear(1.0, 1.0, 1.0)
self.vao = self.ctx.simple_vertex_array(self.prog,
self.ctx.buffer(self.vertices),
'in_vert')
self.vao.render(mode=6)
img_buf = Image.frombytes('RGB', (self.width, self.height),
self.fbo.read(components=3))
image_out = np.array(img_buf.convert('RGB'))
new_frame = VideoFrame.from_ndarray(image_out, format="bgr24")
new_frame.pts = pts
new_frame.time_base = time_base
return new_frame
def test_stream_index(self):
output = av.open(self.sandboxed('output.mov'), 'w')
vstream = output.add_stream('mpeg4', 24)
vstream.pix_fmt = 'yuv420p'
vstream.width = 320
vstream.height = 240
astream = output.add_stream('mp2', 48000)
astream.channels = 2
astream.format = 's16'
self.assertEqual(vstream.index, 0)
self.assertEqual(astream.index, 1)
vframe = VideoFrame(320, 240, 'yuv420p')
vpacket = vstream.encode(vframe)[0]
self.assertIs(vpacket.stream, vstream)
self.assertEqual(vpacket.stream_index, 0)
for i in range(10):
aframe = AudioFrame('s16', 'stereo', samples=astream.frame_size)
aframe.rate = 48000
apackets = astream.encode(aframe)
if apackets:
apacket = apackets[0]
break
self.assertIs(apacket.stream, astream)
self.assertEqual(apacket.stream_index, 1)
async def recv(self):
print(self.position)
frame = await self.track.recv()
img = frame.to_ndarray(format="bgr24")
logo = cv2.imread('watermark.png')
logo = cv2.resize(logo, (100, 60)) # width, height
# Create a mask of logo
img2gray = cv2.cvtColor(logo, cv2.COLOR_BGR2GRAY)
ret, mask = cv2.threshold(img2gray, 1, 255, cv2.THRESH_BINARY)
# Region of Image (ROI), where we want to insert logo
roi = img[
-60 - 10:-10, -100 - 10:
-10] # height box, shift bottom:top, width box, shift right:left
# NOTE it seems that shift values must be equal
# Set an index of where the mask is
roi[np.where(mask)] = 0
roi += logo
xy1 = [self.powidth, self.poheight]
xy2 = [self.powidth2, self.poheight2]
# xy1 = [160, 120]
# xy2 = [480, 360]
img = cv2.rectangle(img, (xy1[0], xy1[1]), (xy2[0], xy2[1]),
(0, 0, 255), 1, cv2.LINE_4)
new_frame = VideoFrame.from_ndarray(img, format="bgr24")
new_frame.pts = frame.pts
new_frame.time_base = frame.time_base
return new_frame
def test_stream_index(self):
with av.open(self.sandboxed("output.mov"), "w") as output:
vstream = output.add_stream("mpeg4", 24)
vstream.pix_fmt = "yuv420p"
vstream.width = 320
vstream.height = 240
astream = output.add_stream("mp2", 48000)
astream.channels = 2
astream.format = "s16"
self.assertEqual(vstream.index, 0)
self.assertEqual(astream.index, 1)
vframe = VideoFrame(320, 240, "yuv420p")
vpacket = vstream.encode(vframe)[0]
self.assertIs(vpacket.stream, vstream)
self.assertEqual(vpacket.stream_index, 0)
for i in range(10):
if astream.frame_size != 0:
frame_size = astream.frame_size
else:
# decoder didn't indicate constant frame size
frame_size = 1000
aframe = AudioFrame("s16", "stereo", samples=frame_size)
aframe.rate = 48000
apackets = astream.encode(aframe)
if apackets:
apacket = apackets[0]
break
self.assertIs(apacket.stream, astream)
self.assertEqual(apacket.stream_index, 1)
async def recv(self):
self.data_bgr = await self.camera_device.get_latest_frame()
frame = VideoFrame.from_ndarray(self.data_bgr, format='bgr24')
pts, time_base = await self.next_timestamp()
frame.pts = pts
frame.time_base = time_base
return frame
async def recv(self):
if self.track:
frame = await self.track.recv()
img = None
try:
# process video frame
frame_img = frame.to_ndarray(format='bgr24')
if isinstance(self.__frame_transformer, FrameTransformer):
img = self.__frame_transformer.transform(frame_img, self.frame_idx)
else:
img = self.__frame_transformer(frame_img, self.frame_idx)
except Exception as ex:
logger.error(ex)
if img is None and self.last_img is None:
img = frame.to_ndarray(format='bgr24')
elif img is None:
img = self.last_img
else:
self.last_img = img
self.frame_idx += 1
else:
img = np.zeros((640, 480, 3))
# rebuild a VideoFrame, preserving timing information
new_frame = VideoFrame.from_ndarray(img, format='bgr24')
new_frame.pts = frame.pts
new_frame.time_base = frame.time_base
return new_frame
def transform(self, frame: av.VideoFrame) -> np.ndarray:
global model
# Target area where the hand gestures should be.
img = frame.to_ndarray(format="bgr24")
cv2.rectangle(img, (self.x_, 0), (CROP_SIZE + self.x_, CROP_SIZE),
(0, 255, 0), 3)
#cv2.rectangle(img, (100, 5), (100, 400), (0, 200, 0), 3)
# Preprocessing the frame before input to the model.
cropped_image = img[0:CROP_SIZE, 0:CROP_SIZE]
resized_frame = cv2.resize(cropped_image, (IMAGE_SIZE, IMAGE_SIZE))
reshaped_frame = (np.array(resized_frame)).reshape(
(1, IMAGE_SIZE, IMAGE_SIZE, 3))
frame_for_model = data_generator.standardize(
np.float64(reshaped_frame))
# Predicting the frame.
prediction = np.array(model.predict(frame_for_model))
self.predicted_class = classes[
prediction.argmax()] # Selecting the max confidence index.
self.predict()
cv2.putText(img, self.word, (10, 450), 1, 2, (255, 255, 0), 2,
cv2.LINE_AA)
return img
async def recv(self):
pts, time_base = await self.next_timestamp()
# read cameras
ret1, frame1 = self.cap1.read()
ret2, frame2 = self.cap2.read()
if not ret1 and not ret2:
return
self.tex1.write(data=frame1)
self.tex2.write(data=frame2)
# update rotation parameters
self.prog['yaw'].value = 0.0
self.prog['pitch'].value = 0.0
self.prog['roll'].value = 0.0
# render
self.ctx.clear(1.0, 1.0, 1.0)
self.vao.render(mode=6)
self.fbo.read_into(self.frame, components=3)
# output
final_frame = np.array(
Image.fromarray(self.frame.astype(np.uint8)).resize(
self.output_size))
new_frame = VideoFrame.from_ndarray(final_frame, format="bgr24")
new_frame.pts = pts
new_frame.time_base = time_base
return new_frame
async def recv(self):
pts, time_base = await self.next_timestamp()
global frame
video_frame = VideoFrame.from_ndarray(frame, format="bgr24")
video_frame.pts = pts
video_frame.time_base = time_base
return video_frame
def __init__(self):
super().__init__() # don't forget this!
self.counter = 0
height, width = 480, 640
# generate flag
data_bgr = np.hstack([
self._create_rectangle(width=213, height=480,
color=(255, 0, 0)), # blue
self._create_rectangle(width=214,
height=480,
color=(255, 255, 255)), # white
self._create_rectangle(width=213, height=480,
color=(0, 0, 255)), # red
])
# shrink and center it
M = np.float32([[0.5, 0, width / 4], [0, 0.5, height / 4]])
data_bgr = cv2.warpAffine(data_bgr, M, (width, height))
# compute animation
omega = 2 * np.pi / height
id_x = np.tile(np.array(range(width), dtype=np.float32), (height, 1))
id_y = np.tile(np.array(range(height), dtype=np.float32),
(width, 1)).transpose()
self.frames = []
for k in range(30):
phase = 2 * k * np.pi / 30
map_x = id_x + 10 * np.cos(omega * id_x + phase)
map_y = id_y + 10 * np.sin(omega * id_x + phase)
self.frames.append(
VideoFrame.from_ndarray(cv2.remap(data_bgr, map_x, map_y,
cv2.INTER_LINEAR),
format="bgr24"))
def read_frame(self):
# this is where we read a frame from the camera
# this can be changed to a function that takes a frame in as a parameter, puts it in the buffer and then
# allows that buffer to be used in the async send
ret, frame = self.vid.read()
if ret:
self.last_frame = VideoFrame.from_ndarray(frame, format="bgr24")
def test_rgb24_planes(self):
frame = VideoFrame(640, 480, 'rgb24')
self.assertEqual(len(frame.planes), 1)
self.assertEqual(frame.planes[0].width, 640)
self.assertEqual(frame.planes[0].height, 480)
self.assertEqual(frame.planes[0].line_size, 640 * 3)
self.assertEqual(frame.planes[0].buffer_size, 640 * 480 * 3)
def generatePreview(self, n):
try:
return next(self.zonecopy.iterFrames(n))
except StopIteration:
return VideoFrame(width=self.filtercopy.width,
height=self.filtercopy.height)
async def recv(self):
pts, time_base = await self.next_timestamp()
_, img = self.cap.read()
frame = VideoFrame.from_ndarray(img, format="bgr24")
frame.pts = pts
frame.time_base = time_base
return frame
async def recv(self):
pts, time_base = await self.next_timestamp()
frame = VideoFrame.from_ndarray(self.data_bgr, format='bgr24')
frame.pts = pts
frame.time_base = time_base
return frame
def test_reformat_colourspace(self):
# This is allowed.
frame = VideoFrame(640, 480, 'rgb24')
frame.reformat(src_colorspace=None, dst_colorspace='smpte240')
# I thought this was not allowed, but it seems to be.
frame = VideoFrame(640, 480, 'yuv420p')
frame.reformat(src_colorspace=None, dst_colorspace='smpte240')
async def recv(self):
pts, time_base = await self.next_timestamp()
# Send a new frame
img = await self.framereader.getFrame()
new_frame = VideoFrame.from_ndarray(img, format="bgr24")
new_frame.pts = pts
new_frame.time_base = time_base
return new_frame
def test_haldclut_graph(self):
raise SkipTest()
graph = Graph()
img = Image.open(fate_suite('png1/lena-rgb24.png'))
frame = VideoFrame.from_image(img)
img_source = graph.add_buffer(frame)
hald_img = Image.open('hald_7.png')
hald_frame = VideoFrame.from_image(hald_img)
hald_source = graph.add_buffer(hald_frame)
try:
hald_filter = graph.add('haldclut')
except ValueError:
# Not in Libav.
raise SkipTest()
sink = graph.add('buffersink')
img_source.link(0, hald_filter, 0)
hald_source.link(0, hald_filter, 1)
hald_filter.link(0, sink, 0)
graph.config()
self.assertIs(img_source.outputs[0].linked_to, hald_filter.inputs[0])
self.assertIs(hald_source.outputs[0].linked_to, hald_filter.inputs[1])
self.assertIs(hald_filter.outputs[0].linked_to, sink.inputs[0])
hald_source.push(hald_frame)
img_source.push(frame)
frame = sink.pull()
self.assertIsInstance(frame, VideoFrame)
frame.to_image().save(self.sandboxed('filtered.png'))
async def recv(self):
if self._startTime is None and self._startedCallback is not None:
self._startedCallback()
try:
img = await self._frameSubscription.get()
except SubscriptionClosed:
self._log.debug(
"Video track finished. raising MediaStreamError to shut down connection"
)
self.stop()
raise MediaStreamError
except:
self._log.exception("Got unknown error. Crashing video stream")
self.stop()
raise MediaStreamError
if self._startTime is None:
self._startTime = time.time()
new_frame = VideoFrame.from_ndarray(img, format="bgr24")
new_frame.time_base = VIDEO_TIME_BASE
# https://en.wikipedia.org/wiki/Presentation_timestamp
if self._fps is None:
# We assume that the frames arrive as fast as they are created.
new_frame.pts = int((time.time() - self._startTime) * VIDEO_CLOCK_RATE)
else:
# We have a target frame rate. Here, we do something similar to the audioSubscription
self._frameNumber += 1
perfectFrameNumber = int((time.time() - self._startTime) * self._fps)
if self._canSkip:
if perfectFrameNumber - self._fps * 1 > self._frameNumber:
self._log.warn(
"Received video frame is over 1 second behind optimal timestamp! Skipping frame forward! Use canSkip=False to disable this correction"
)
self._frameNumber = perfectFrameNumber
new_frame.pts = int(self._frameNumber * VIDEO_CLOCK_RATE / self._fps)
if perfectFrameNumber + self._fps * 2 < self._frameNumber:
# If the audio stream is over 2 seconds ahead, we wait 1 second before continuing
self._log.debug(
"Stream is over 2 seconds ahead. Sleeping for 1 second."
)
await asyncio.sleep(1)
self._log.debug("Writing frame %s", new_frame)
return new_frame
def test_encoding_with_pts(self):
path = self.sandboxed('video_with_pts.mov')
output = av.open(path, 'w')
stream = output.add_stream('libx264', 24)
stream.width = WIDTH
stream.height = HEIGHT
stream.pix_fmt = "yuv420p"
for i in range(DURATION):
frame = VideoFrame(WIDTH, HEIGHT, 'rgb24')
frame.pts = i * 2000
frame.time_base = Fraction(1, 48000)
for packet in stream.encode(frame):
self.assertEqual(packet.time_base, Fraction(1, 24))
output.mux(packet)
for packet in stream.encode(None):
self.assertEqual(packet.time_base, Fraction(1, 24))
output.mux(packet)
output.close()
