def do_transform_ip(self, buffer):
if self.pts_offset is None:
self.pts_offset = self.first_pts - buffer.pts
buffer.pts += self.pts_offset
Gst.debug("do_transform_ip: timestamp: %s" %
(Gst.TIME_ARGS(buffer.pts)))
return Gst.FlowReturn.OK
def scale_value_changed_cb(self, scale):
# see seek.c:seek_cb
real = long(scale.get_value() * self.p_duration / 100) # in ns
Gst.debug('value changed, perform seek to %r' % real)
self.player.seek(real)
# allow for a preroll
self.player.get_state(timeout=50*Gst.MSECOND,full=True) # 50 ms
def _consume_done(self, config, ghostPad, recv_rtp_socket,
recv_rtcp_socket):
Gst.info('%s _consume_done %s' % (self.name, config))
#
desc = getConsumerPipelineDesc(config)
Gst.debug('%s _produce_done desc=%s' % (self.name, desc))
bin = Gst.parse_bin_from_description(desc, False)
self.add(bin)
rtpbin = bin.get_by_name('rtpbin')
# setup sockets
bin.get_by_name('rtp_udpsrc').set_property('socket', recv_rtp_socket)
bin.get_by_name('rtcp_udpsrc').set_property('socket', recv_rtcp_socket)
#
# bin.set_state(Gst.State.PAUSED)
# link ghost pad
src_pad = bin.get_by_name('sink').get_static_pad('src')
tmp_pad = ghostPad.get_target()
ghostPad.set_target(src_pad)
self.remove_pad(tmp_pad)
#
bin.set_state(Gst.State.PLAYING)
#
self.emit('consumer-added', config['consumerId'])
#
self.mediasoup.resumeConsumer(config['transportId'],
config['consumerId'],
self._resume_consumer_done,
self._on_error, config)
def _sinkl_event(self, pad, parent, event):
Gst.debug("event %s" % event)
Gst.debug("event type %s" % event.type)
if event.type == Gst.EventType.CAPS:
caps = event.parse_caps()
Gst.info("event caps %s" % caps)
return self.setcaps_srcv(parent, caps)
return self.srcvpad.push_event(event)
def _srcv_event(self, pad, parent, event):
Gst.debug("event %s" % event)
Gst.debug("event type %s" % event.type)
if event.type == Gst.EventType.QOS:
info = event.parse_qos()
Gst.info("QOS %s" % (str(info)))
else:
return self.sinklpad.push_event(event) and self.sinkrpad.push_event(event)
def do_state_changed(self, oldstate, newstate, pending):
Gst.debug('%s do_state_changed oldstate=%s newstate=%s pending=%s' %
(self.name, oldstate, newstate, pending))
if oldstate in (Gst.State.READY,
Gst.State.PAUSED) and newstate == Gst.State.NULL:
if self.mediasoup:
self.mediasoup.stop()
self.mediasoup = None
if self.signaling:
self.signaling.stop()
self.signaling = None
def _sinkr_chain(self, pad, parent, buf_r):
"""
Chain function
"""
Gst.debug("R Buffer R %s ts=%s sz=%s" \
% (buf_r, Gst.TIME_ARGS(buf_r.pts), buf_r.get_size()))
self._bufs_r.append(buf_r)
#self.process_video(parent)
res = Gst.FlowReturn.OK if len(self._bufs_r) < 1000 else Gst.FlowReturn.FLUSHING
return res
def add_region(self,
x,
y,
w,
h,
label_id: int,
confidence: float = 0.0,
region_tensor: Gst.Structure = None,
normalized: bool = False) -> RegionOfInterest:
if normalized:
x = int(x * self.video_info().width)
y = int(y * self.video_info().height)
w = int(w * self.video_info().width)
h = int(h * self.video_info().height)
if not self.__is_bounded(x, y, w, h):
x_init, y_init, w_init, h_init = x, y, w, h
x, y, w, h = self.__clip(x, y, w, h)
Gst.debug(
"ROI coordinates [x, y, w, h] are out of image borders and will be clipped: [{}, {}, {}, {}] -> "
"[{}, {}, {}, {}]".format(x_init, y_init, w_init, h_init, x, y,
w, h))
label = self.__get_label_by_label_id(region_tensor, label_id)
video_roi_meta = GstVideo.buffer_add_video_region_of_interest_meta(
self.__buffer, label, x, y, w, h)
if not region_tensor:
region_tensor = Gst.Structure.new_empty("detection")
else:
region_tensor.set_name(
"detection") # make sure we're about to add detection Tensor
region_tensor.set_value('label_id', label_id)
region_tensor.set_value('confidence', confidence)
region_tensor.set_value('x_min', x / self.video_info().width)
region_tensor.set_value('x_max', (x + w) / self.video_info().width)
region_tensor.set_value('y_min', y / self.video_info().height)
region_tensor.set_value('y_max', (y + h) / self.video_info().height)
self.__regions.append(
RegionOfInterest(
ctypes.cast(
hash(video_roi_meta),
ctypes.POINTER(VideoRegionOfInterestMeta)).contents))
self.__regions[-1].add_tensor(tensor=region_tensor)
return self.__regions[-1]
def seek(self, location):
"""
@param location: time to seek to, in nanoseconds
"""
Gst.debug("seeking to %r" % location)
event = Gst.event_new_seek(1.0, Gst.FORMAT_TIME,
Gst.SEEK_FLAG_FLUSH | Gst.SEEK_FLAG_ACCURATE,
Gst.SEEK_TYPE_SET, location,
Gst.SEEK_TYPE_NONE, 0)
res = self.player.send_event(event)
if res:
#Gst.info("setting new stream time to 0")
self.player.set_new_stream_time(0L)
else:
Gst.error("seek to %r failed" % location)
def scale_button_press_cb(self, widget, event):
# see seek.c:start_seek
Gst.debug('starting seek')
self.seekmove = True
self.was_playing = self.player.is_playing()
if self.was_playing:
self.pause_resume()
# don't timeout-update position during seek
if self.update_id != -1:
GObject.source_remove(self.update_id)
self.update_id = -1
# make sure we get changed notifies
if self.changed_id == -1:
self.changed_id = self.seeker.connect('value-changed',
self.scale_value_changed_cb)
def scale_button_release_cb(self, widget, event):
# see seek.cstop_seek
self.seeker.disconnect(self.changed_id)
self.changed_id = -1
self.seekmove = False
if self.seek_timeout_id != -1:
GObject.source_remove(self.seek_timeout_id)
self.seek_timeout_id = -1
else:
Gst.debug('released slider, setting back to playing')
if self.was_playing:
self.pause_resume()
if self.update_id != -1:
self.error('Had a previous update timeout id')
else:
self.update_id = GObject.timeout_add(self.UPDATE_INTERVAL,
self.update_scale_cb)
def chain_function(pad, parent, buf):
Gst.debug('%s chain_function caps: %s pts: %f' %
(pad.name, pad.get_current_caps(), buf.pts * 1e-9))
caps = pad.get_current_caps()
if caps:
structure = caps.get_structure(0)
Gst.info('%s event_function caps=%s' % (pad.name, caps))
kind = structure.get_name().split('/')[0]
appData = json.loads(self.app_data)
# producer
config = {
'kind': kind,
'producerId': structure.get_string('producer-id'),
'text_overlay': self.text_overlay,
'time_overlay': self.time_overlay,
'clock_overlay': self.clock_overlay,
}
if kind == 'audio':
config.update(DEFAULT_AUDIO_CONFIG)
config['codec'] = self.audio_codec
config['bitrate'] = self.audio_bitrate
else:
config.update(DEFAULT_VIDEO_CONFIG)
config['server_ip'] = self.server_ip
config['codec'] = self.video_codec
config['bitrate'] = self.video_bitrate
config['hw'] = self.hw
config['gop'] = self.gop
config['simulcast'] = self.simulcast
config['width'] = structure['width']
config['height'] = structure['height']
config['framerate'] = structure['framerate'].num or 30
appData['width'] = config['width']
appData['height'] = config['height']
appData['framerate'] = config['framerate']
appData['maxBitrate'] = config['bitrate']
self.mediasoup.produce(config, appData, self._produce_done,
self._on_error,
self._on_producer_removed, ghostPad)
return Gst.FlowReturn.OK
def process_video(self, parent):
while True:
def popleft_or_none(x):
try:
return x.popleft()
except IndexError:
return
Gst.info("Buffers outstanding % 3d / % 3d" % (len(self._bufs_l), len(self._bufs_r)))
buf_l0 = popleft_or_none(self._bufs_l)
buf_l1 = popleft_or_none(self._bufs_l)
buf_r0 = popleft_or_none(self._bufs_r)
buf_r1 = popleft_or_none(self._bufs_r)
def app_if(x, y):
if y is not None:
x.appendleft(y)
if buf_l0 is None:
Gst.info("No L buffers")
app_if(self._bufs_r, buf_r1)
app_if(self._bufs_r, buf_r0)
return
if buf_r0 is None:
Gst.info("No R buffers")
app_if(self._bufs_l, buf_l1)
app_if(self._bufs_l, buf_l0)
return
if buf_l1 is None:
app_if(self._bufs_l, buf_l0)
app_if(self._bufs_r, buf_r1)
app_if(self._bufs_r, buf_r0)
return
if buf_r1 is None:
app_if(self._bufs_r, buf_r0)
app_if(self._bufs_l, buf_l1)
app_if(self._bufs_l, buf_l0)
return
Gst.debug("Buffer L0 %s ts=%s sz=%s" \
% (buf_l0, Gst.TIME_ARGS(buf_l0.pts), buf_l0.get_size()))
Gst.debug("Buffer L1 %s ts=%s sz=%s" \
% (buf_l1, Gst.TIME_ARGS(buf_l1.pts), buf_l1.get_size()))
Gst.debug("Buffer R0 %s ts=%s sz=%s" \
% (buf_r0, Gst.TIME_ARGS(buf_r0.pts), buf_r0.get_size()))
Gst.debug("Buffer R1 %s ts=%s sz=%s" \
% (buf_r1, Gst.TIME_ARGS(buf_r1.pts), buf_r1.get_size()))
if buf_l0.pts >= buf_r0.pts and buf_l0.pts <= buf_r0.pts + buf_r0.duration:
Gst.debug("a")
buf_l = buf_l0
buf_r = buf_r0
app_if(self._bufs_l, buf_l1)
app_if(self._bufs_r, buf_r1)
elif buf_r0.pts >= buf_l0.pts and buf_r0.pts <= buf_l0.pts + buf_l0.duration:
Gst.debug("b")
buf_l = buf_l0
buf_r = buf_r0
app_if(self._bufs_l, buf_l1)
app_if(self._bufs_r, buf_r1)
elif buf_r0.pts + buf_r0.duration < buf_l0.pts:
Gst.debug("c")
app_if(self._bufs_l, buf_l1)
app_if(self._bufs_l, buf_l0)
app_if(self._bufs_r, buf_r1)
Gst.info("Next time for R")
continue
elif buf_l0.pts + buf_l0.duration < buf_r0.pts:
Gst.debug("d")
app_if(self._bufs_r, buf_r1)
app_if(self._bufs_r, buf_r0)
app_if(self._bufs_l, buf_l1)
Gst.info("Next time for L")
continue
else:
Gst.info("Not implemented!")
Gst.debug("Pushing new buffer for t=%s" % Gst.TIME_ARGS(buf_l.pts))
srch, srcw, srcd = self._srch, self._srcw, self._srcd
dsth, dstw = self._dsth, self._dstw
tz = list()
res, mil = buf_l.map(Gst.MapFlags.READ)
assert res
res, mir = buf_r.map(Gst.MapFlags.READ)
assert res
src0_d = mil.data
src1_d = mir.data
img_l = np.fromstring(src0_d, dtype=np.uint8)[:srch*srcw*srcd].reshape((srch, srcw, srcd))
img_r = np.fromstring(src1_d, dtype=np.uint8)[:srch*srcw*srcd].reshape((srch, srcw, srcd))
buf_r.unmap(mir)
buf_l.unmap(mil)
v = np.hstack((img_l, img_r))
tz.append(time.time())
buf_out = Gst.Buffer.new_allocate(None, dsth*dstw*4, None)
buf_out.dts = buf_l.dts
buf_out.pts = buf_l.pts
buf_out.duration = buf_l.duration
tz.append(time.time())
buf_out.fill(0, v.tobytes())
#time.sleep(0.01) # artificial computation time
#buf_out = Gst.Buffer.new_wrapped(v.data)#, 1280*480*4)
#res, mio = buf_out.map(Gst.MapFlags.WRITE)
#print(dir(mio))
#dst_d = mio.data
#dst_d[:] = v
#buf_out.unmap(mio)
tz.append(time.time())
tz.append(time.time())
self.srcvpad.push(buf_out)
tz.append(time.time())
Gst.debug("t_alloc %.3f t_cp %.3f t_push %.3f" \
% (tz[1] - tz[0], tz[2] - tz[1], tz[-1] - tz[-2]))
def setcaps_srcv(self, parent, caps):
othercaps = self.srcvpad.get_allowed_caps()
Gst.debug("other caps %s" % othercaps)
self._srcw = caps[0]["width"]
self._srch = caps[0]["height"]
self._srcd = 4
if 1:
outcaps = Gst.Caps('%s' % othercaps)
out_s = outcaps[0]
Gst.debug("outcasp 0 %s" % (out_s))
Gst.debug(" %s" % dir(out_s))
fr = caps[0]["framerate"]
out_s.set_value("framerate", fr)
Gst.debug("out_s %s" % out_s.to_string())
Gst.debug(" dir %s" % dir(outcaps))
#outcaps.set_structure(0, out_s)
outcaps = Gst.Caps.from_string(out_s.to_string())
Gst.debug("pad is %s" % self.srcvpad.__class__)
Gst.debug("pad can %s" % dir(self.srcvpad))
self._dstw = outcaps[0]["width"]
self._dsth = outcaps[0]["height"]
res = self.srcvpad.push_event(Gst.Event.new_caps(outcaps))
Gst.info("srcv caps %s" % outcaps)
self.srcvpad.use_fixed_caps()
return res
