async def handle_offer_for_webcam(self, request, ws):
offer = RTCSessionDescription(sdp=request["sdp"], type=request["type"])
pc = RTCPeerConnection()
pc_id = "PeerConnection(%s)" % uuid.uuid4()
self.webrtc.pcs.add(pc)
self.pcs[ws] = pc
def log_info(msg, *args):
self.logger.info(pc_id + " " + msg, *args)
#log_info("Receiving for %s", request.remote)
@pc.on("iceconnectionstatechange")
async def on_iceconnectionstatechange():
log_info("ICE connection state is %s", pc.iceConnectionState)
if pc.iceConnectionState == "failed":
await pc.close()
self.webrtc.pcs.discard(pc)
# handle offer
await pc.setRemoteDescription(offer)
for t in pc.getTransceivers():
# if t.kind == "audio" and player.audio:
# pc.addTrack(player.audio)
if t.kind == "video":
print(self.webrtc.local_video)
pc.addTrack(self.webrtc.local_video)
# send answer
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
answer = {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}
answer_message = { "type": "ANSWER", "payload": {"message":answer} }
log_info("Negotiation Answer- %s", answer_message)
await ws.send_str(json.dumps(answer_message))
class Handler:
def __init__(self,
handlerId: str,
channel: Channel,
loop: asyncio.AbstractEventLoop,
getTrack,
addRemoteTrack,
getRemoteTrack,
configuration: Optional[RTCConfiguration] = None) -> None:
self._handlerId = handlerId
self._channel = channel
self._pc = RTCPeerConnection(configuration or None)
# dictionary of sending transceivers mapped by given localId
self._sendTransceivers = dict() # type: Dict[str, RTCRtpTransceiver]
# dictionary of dataChannelds mapped by internal id
self._dataChannels = dict() # type: Dict[str, RTCDataChannel]
# function returning a sending track given a player id and a kind
self._getTrack = getTrack
# function to store a receiving track
self._addRemoteTrack = addRemoteTrack
# function returning a receiving track
self._getRemoteTrack = getRemoteTrack
@self._pc.on("track") # type: ignore
def on_track(track) -> None:
Logger.debug(
f"handler: ontrack [kind:{track.kind}, id:{track.id}]")
# store it
self._addRemoteTrack(track)
@self._pc.on("signalingstatechange") # type: ignore
async def on_signalingstatechange() -> None:
Logger.debug(
f"handler: signalingstatechange [state:{self._pc.signalingState}]"
)
await self._channel.notify(self._handlerId, "signalingstatechange",
self._pc.signalingState)
@self._pc.on("icegatheringstatechange") # type: ignore
async def on_icegatheringstatechange() -> None:
Logger.debug(
f"handler: icegatheringstatechange [state:{self._pc.iceGatheringState}]"
)
await self._channel.notify(self._handlerId,
"icegatheringstatechange",
self._pc.iceGatheringState)
@self._pc.on("iceconnectionstatechange") # type: ignore
async def on_iceconnectionstatechange() -> None:
Logger.debug(
f"handler: iceconnectionstatechange [state:{self._pc.iceConnectionState}]"
)
await self._channel.notify(self._handlerId,
"iceconnectionstatechange",
self._pc.iceConnectionState)
async def checkDataChannelsBufferedAmount() -> None:
while True:
await asyncio.sleep(1)
for dataChannelId, dataChannel in self._dataChannels.items():
await self._channel.notify(dataChannelId, "bufferedamount",
dataChannel.bufferedAmount)
self._dataChannelsBufferedAmountTask = loop.create_task(
checkDataChannelsBufferedAmount())
async def close(self) -> None:
# stop the periodic task
self._dataChannelsBufferedAmountTask.cancel()
# close peerconnection
await self._pc.close()
def dump(self) -> Any:
result = {
"id": self._handlerId,
"signalingState": self._pc.signalingState,
"iceConnectionState": self._pc.iceConnectionState,
"iceGatheringState": self._pc.iceGatheringState,
"transceivers": [],
"sendTransceivers": []
}
for transceiver in self._pc.getTransceivers():
transceiverInfo = {
"mid": transceiver.mid,
"stopped": transceiver.stopped,
"kind": transceiver.kind,
"currentDirection": transceiver.currentDirection,
"direction": transceiver.direction,
"sender": {
"trackId":
transceiver.sender.track.id
if transceiver.sender.track else None
},
"receiver": {
"trackId":
transceiver.receiver.track.id
if transceiver.receiver.track else None
}
}
result["transceivers"].append(transceiverInfo)
for localId, transceiver in self._sendTransceivers.items():
sendTransceiverInfo = {"localId": localId, "mid": transceiver.mid}
result["sendTransceivers"].append(sendTransceiverInfo)
return result
async def processRequest(self, request: Request) -> Any:
if request.method == "handler.getLocalDescription":
localDescription = self._pc.localDescription
if (localDescription is not None):
return {
"type": localDescription.type,
"sdp": localDescription.sdp
}
else:
return None
elif request.method == "handler.createOffer":
offer = await self._pc.createOffer()
return {"type": offer.type, "sdp": offer.sdp}
elif request.method == "handler.createAnswer":
answer = await self._pc.createAnswer()
return {"type": answer.type, "sdp": answer.sdp}
elif request.method == "handler.setLocalDescription":
data = request.data
if isinstance(data, RTCSessionDescription):
raise TypeError("request data not a RTCSessionDescription")
description = RTCSessionDescription(**data)
await self._pc.setLocalDescription(description)
elif request.method == "handler.setRemoteDescription":
data = request.data
if isinstance(data, RTCSessionDescription):
raise TypeError("request data not a RTCSessionDescription")
description = RTCSessionDescription(**data)
await self._pc.setRemoteDescription(description)
elif request.method == "handler.getMid":
data = request.data
localId = data.get("localId")
if localId is None:
raise TypeError("missing data.localId")
# raise on purpose if the key is not found
transceiver = self._sendTransceivers[localId]
return transceiver.mid
elif request.method == "handler.addTrack":
data = request.data
localId = data.get("localId")
if localId is None:
raise TypeError("missing data.localId")
kind = data["kind"]
playerId = data.get("playerId")
recvTrackId = data.get("recvTrackId")
# sending a track got from a MediaPlayer
if playerId:
track = self._getTrack(playerId, kind)
transceiver = self._pc.addTransceiver(track)
# sending a track which is a remote/receiving track
elif recvTrackId:
track = self._getRemoteTrack(recvTrackId, kind)
transceiver = self._pc.addTransceiver(track)
else:
raise TypeError("missing data.playerId or data.recvTrackId")
# store transceiver in the dictionary
self._sendTransceivers[localId] = transceiver
elif request.method == "handler.removeTrack":
data = request.data
localId = data.get("localId")
if localId is None:
raise TypeError("missing data.localId")
transceiver = self._sendTransceivers[localId]
transceiver.direction = "inactive"
transceiver.sender.replaceTrack(None)
# NOTE: do not remove transceiver from the dictionary
elif request.method == "handler.replaceTrack":
data = request.data
localId = data.get("localId")
if localId is None:
raise TypeError("missing data.localId")
kind = data["kind"]
playerId = data.get("playerId")
recvTrackId = data.get("recvTrackId")
transceiver = self._sendTransceivers[localId]
# sending a track got from a MediaPlayer
if playerId:
track = self._getTrack(playerId, kind)
# sending a track which is a remote/receiving track
elif recvTrackId:
track = self._getRemoteTrack(recvTrackId, kind)
else:
raise TypeError("missing data.playerId or data.recvTrackId")
transceiver.sender.replaceTrack(track)
elif request.method == "handler.getTransportStats":
result = {}
stats = await self._pc.getStats()
for key in stats:
type = stats[key].type
if type == "inbound-rtp":
result[key] = self._serializeInboundStats(stats[key])
elif type == "outbound-rtp":
result[key] = self._serializeOutboundStats(stats[key])
elif type == "remote-inbound-rtp":
result[key] = self._serializeRemoteInboundStats(stats[key])
elif type == "remote-outbound-rtp":
result[key] = self._serializeRemoteOutboundStats(
stats[key])
elif type == "transport":
result[key] = self._serializeTransportStats(stats[key])
return result
elif request.method == "handler.getSenderStats":
data = request.data
mid = data.get("mid")
if mid is None:
raise TypeError("missing data.mid")
transceiver = self._getTransceiverByMid(mid)
sender = transceiver.sender
result = {}
stats = await sender.getStats()
for key in stats:
type = stats[key].type
if type == "outbound-rtp":
result[key] = self._serializeOutboundStats(stats[key])
elif type == "remote-inbound-rtp":
result[key] = self._serializeRemoteInboundStats(stats[key])
elif type == "transport":
result[key] = self._serializeTransportStats(stats[key])
return result
elif request.method == "handler.getReceiverStats":
data = request.data
mid = data.get("mid")
if mid is None:
raise TypeError("missing data.mid")
transceiver = self._getTransceiverByMid(mid)
receiver = transceiver.receiver
result = {}
stats = await receiver.getStats()
for key in stats:
type = stats[key].type
if type == "inbound-rtp":
result[key] = self._serializeInboundStats(stats[key])
elif type == "remote-outbound-rtp":
result[key] = self._serializeRemoteOutboundStats(
stats[key])
elif type == "transport":
result[key] = self._serializeTransportStats(stats[key])
return result
elif request.method == "handler.createDataChannel":
internal = request.internal
dataChannelId = internal.get("dataChannelId")
data = request.data
id = data.get("id")
ordered = data.get("ordered")
maxPacketLifeTime = data.get("maxPacketLifeTime")
maxRetransmits = data.get("maxRetransmits")
label = data.get("label")
protocol = data.get("protocol")
dataChannel = self._pc.createDataChannel(
negotiated=True,
id=id,
ordered=ordered,
maxPacketLifeTime=maxPacketLifeTime,
maxRetransmits=maxRetransmits,
label=label,
protocol=protocol)
# store datachannel in the dictionary
self._dataChannels[dataChannelId] = dataChannel
@dataChannel.on("open") # type: ignore
async def on_open() -> None:
await self._channel.notify(dataChannelId, "open")
@dataChannel.on("closing") # type: ignore
async def on_closing() -> None:
await self._channel.notify(dataChannelId, "closing")
@dataChannel.on("close") # type: ignore
async def on_close() -> None:
# NOTE: After calling dataChannel.close() aiortc emits "close" event
# on the dataChannel. Probably it shouldn't do it. So caution.
try:
del self._dataChannels[dataChannelId]
await self._channel.notify(dataChannelId, "close")
except KeyError:
pass
@dataChannel.on("message") # type: ignore
async def on_message(message) -> None:
if isinstance(message, str):
await self._channel.notify(dataChannelId, "message",
message)
if isinstance(message, bytes):
message_bytes = base64.b64encode(message)
await self._channel.notify(dataChannelId, "binary",
str(message_bytes))
@dataChannel.on("bufferedamountlow") # type: ignore
async def on_bufferedamountlow() -> None:
await self._channel.notify(dataChannelId, "bufferedamountlow")
return {
"streamId":
dataChannel.id,
"ordered":
dataChannel.ordered,
"maxPacketLifeTime":
dataChannel.maxPacketLifeTime,
"maxRetransmits":
dataChannel.maxRetransmits,
"label":
dataChannel.label,
"protocol":
dataChannel.protocol,
# status fields
"readyState":
dataChannel.readyState,
"bufferedAmount":
dataChannel.bufferedAmount,
"bufferedAmountLowThreshold":
dataChannel.bufferedAmountLowThreshold
}
else:
raise TypeError("unknown request method")
async def processNotification(self, notification: Notification) -> None:
if notification.event == "enableTrack":
Logger.warning("handler: enabling track not implemented")
elif notification.event == "disableTrack":
Logger.warning("handler: disabling track not implemented")
elif notification.event == "datachannel.send":
internal = notification.internal
dataChannelId = internal.get("dataChannelId")
if dataChannelId is None:
raise TypeError("missing internal.dataChannelId")
data = notification.data
dataChannel = self._dataChannels[dataChannelId]
dataChannel.send(data)
# Good moment to update bufferedAmount in Node.js side
await self._channel.notify(dataChannelId, "bufferedamount",
dataChannel.bufferedAmount)
elif notification.event == "datachannel.sendBinary":
internal = notification.internal
dataChannelId = internal.get("dataChannelId")
if dataChannelId is None:
raise TypeError("missing internal.dataChannelId")
data = notification.data
dataChannel = self._dataChannels[dataChannelId]
dataChannel.send(base64.b64decode(data))
# Good moment to update bufferedAmount in Node.js side
await self._channel.notify(dataChannelId, "bufferedamount",
dataChannel.bufferedAmount)
elif notification.event == "datachannel.close":
internal = notification.internal
dataChannelId = internal.get("dataChannelId")
if dataChannelId is None:
raise TypeError("missing internal.dataChannelId")
dataChannel = self._dataChannels.get(dataChannelId)
if dataChannel is None:
return
# NOTE: After calling dataChannel.close() aiortc emits "close" event
# on the dataChannel. Probably it shouldn't do it. So caution.
try:
del self._dataChannels[dataChannelId]
except KeyError:
pass
dataChannel.close()
elif notification.event == "datachannel.setBufferedAmountLowThreshold":
internal = notification.internal
dataChannelId = internal.get("dataChannelId")
if dataChannelId is None:
raise TypeError("missing internal.dataChannelId")
value = notification.data
dataChannel = self._dataChannels[dataChannelId]
dataChannel.bufferedAmountLowThreshold = value
else:
raise TypeError("unknown notification event")
"""
Helper functions
"""
def _getTransceiverByMid(self, mid: str) -> Optional[RTCRtpTransceiver]:
return next(filter(lambda x: x.mid == mid, self._pc.getTransceivers()),
None)
def _serializeInboundStats(self, stats: RTCStatsReport) -> Dict[str, Any]:
return {
# RTCStats
"timestamp": stats.timestamp.timestamp(),
"type": stats.type,
"id": stats.id,
# RTCStreamStats
"ssrc": stats.ssrc,
"kind": stats.kind,
"transportId": stats.transportId,
# RTCReceivedRtpStreamStats
"packetsReceived": stats.packetsReceived,
"packetsLost": stats.packetsLost,
"jitter": stats.jitter
}
def _serializeOutboundStats(self, stats: RTCStatsReport) -> Dict[str, Any]:
return {
# RTCStats
"timestamp": stats.timestamp.timestamp(),
"type": stats.type,
"id": stats.id,
# RTCStreamStats
"ssrc": stats.ssrc,
"kind": stats.kind,
"transportId": stats.transportId,
# RTCSentRtpStreamStats
"packetsSent": stats.packetsSent,
"bytesSent": stats.bytesSent,
# RTCOutboundRtpStreamStats
"trackId": stats.trackId
}
def _serializeRemoteInboundStats(self,
stats: RTCStatsReport) -> Dict[str, Any]:
return {
# RTCStats
"timestamp": stats.timestamp.timestamp(),
"type": stats.type,
"id": stats.id,
# RTCStreamStats
"ssrc": stats.ssrc,
"kind": stats.kind,
"transportId": stats.transportId,
# RTCReceivedRtpStreamStats
"packetsReceived": stats.packetsReceived,
"packetsLost": stats.packetsLost,
"jitter": stats.jitter,
# RTCRemoteInboundRtpStreamStats
"roundTripTime": stats.roundTripTime,
"fractionLost": stats.fractionLost
}
def _serializeRemoteOutboundStats(self,
stats: RTCStatsReport) -> Dict[str, Any]:
return {
# RTCStats
"timestamp": stats.timestamp.timestamp(),
"type": stats.type,
"id": stats.id,
# RTCStreamStats
"ssrc": stats.ssrc,
"kind": stats.kind,
"transportId": stats.transportId,
# RTCSentRtpStreamStats
"packetsSent": stats.packetsSent,
"bytesSent": stats.bytesSent,
# RTCRemoteOutboundRtpStreamStats
"remoteTimestamp": stats.remoteTimestamp.timestamp()
}
def _serializeTransportStats(self,
stats: RTCStatsReport) -> Dict[str, Any]:
return {
# RTCStats
"timestamp": stats.timestamp.timestamp(),
"type": stats.type,
"id": stats.id,
# RTCTransportStats
"packetsSent": stats.packetsSent,
"packetsReceived": stats.packetsReceived,
"bytesSent": stats.bytesSent,
"bytesReceived": stats.bytesReceived,
"iceRole": stats.iceRole,
"dtlsState": stats.dtlsState
}
async def offer(request):
params = await request.json()
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
#logger.info(offer)
pc = RTCPeerConnection()
pc_id = "PeerConnection(%s)" % uuid.uuid4()
pcs.add(pc)
def log_info(msg, *args):
logger.info(pc_id + " " + msg, *args)
log_info("Created for %s", request.remote)
@pc.on("datachannel")
def on_datachannel(channel):
@channel.on("message")
def on_message(message):
if isinstance(message, str) and message.startswith("ping"):
channel.send("pong" + message[4:])
@pc.on("connectionstatechange")
async def on_connectionstatechange():
log_info("Connection state is %s", pc.connectionState)
if pc.connectionState == "failed":
await pc.close()
pcs.discard(pc)
# Otetaan video ja audio vastaan striimin välittävältä palvelimelta
# Tässä toistetaan video / audio -tiedosto, mitä tarkoittaa jatkuvan
# streamin tapauksessa?
@pc.on("track")
def on_track(track):
log_info("Track %s received", track.kind)
if track.kind == "audio":
pc.addTrack(player.audio)
recorder.addTrack(track)
elif track.kind == "video":
create_broadcast(track)
pc.addTrack(track)
#pc.addTrack(track)#relay.subscribe(broadcast))
@track.on("ended")
async def on_ended():
log_info("Track %s ended", track.kind)
#await pc.close()
#pcs.discard(pc)
broadcast_ended()
coros = [pc.close() for pc in pcs]
await asyncio.gather(*coros)
pcs.clear()
# handle offer
await pc.setRemoteDescription(offer)
# Tämä ajetaan kun clientistä on "Listen for..." valittuna
# addTrack menee clienttiin ja siihen laitetaan relay broadcastista
if params["listen_video"]:
log_info("Kuuntelu")
for t in pc.getTransceivers():
log_info("Kuuntelu %s", t.kind)
# Tarkasta onko "broadcast" olemassa
if t.kind == "video" and broadcast:
pc.addTrack(relay.subscribe(broadcast))
capabilities = RTCRtpSender.getCapabilities('video')
preferences = list(
filter(lambda x: x.name == 'H264', capabilities.codecs))
print(preferences)
transc = pc.getTransceivers()[0]
transc.setCodecPreferences(preferences)
# send answer
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
print(json.dumps({"type": pc.localDescription.type}))
return web.Response(
content_type="application/json",
text=json.dumps({
"sdp": pc.localDescription.sdp,
"type": pc.localDescription.type
}),
)
async def _process_offer(
mode: WebRtcMode,
pc: RTCPeerConnection,
offer: RTCSessionDescription,
player_factory: Optional[MediaPlayerFactory],
in_recorder_factory: Optional[MediaRecorderFactory],
out_recorder_factory: Optional[MediaRecorderFactory],
video_transformer: Optional[VideoTransformerBase],
video_receiver: Optional[VideoReceiver],
async_transform: bool,
callback: Callable[[Union[RTCSessionDescription, Exception]], None],
):
try:
player = None
if player_factory:
player = player_factory()
in_recorder = None
if in_recorder_factory:
in_recorder = in_recorder_factory()
out_recorder = None
if out_recorder_factory:
out_recorder = out_recorder_factory()
@pc.on("iceconnectionstatechange")
async def on_iceconnectionstatechange():
logger.info("ICE connection state is %s", pc.iceConnectionState)
if pc.iceConnectionState == "failed":
await pc.close()
if mode == WebRtcMode.SENDRECV:
@pc.on("track")
def on_track(input_track):
logger.info("Track %s received", input_track.kind)
output_track = None
if input_track.kind == "audio":
if player and player.audio:
logger.info("Add player to audio track")
output_track = player.audio
else:
# Transforming audio is not supported yet.
output_track = input_track # passthrough
elif input_track.kind == "video":
if player and player.video:
logger.info("Add player to video track")
output_track = player.video
elif video_transformer:
VideoTrack = (
AsyncVideoTransformTrack
if async_transform
else VideoTransformTrack
)
logger.info(
"Add a input video track %s to "
"output track with video_transformer %s",
input_track,
VideoTrack,
)
local_video = VideoTrack(
track=input_track, video_transformer=video_transformer
)
logger.info("Add the video track with transfomer to %s", pc)
output_track = local_video
else:
output_track = input_track
if not output_track:
raise Exception(
"Neither a player nor a transformer is created. "
"Either factory must be set."
)
pc.addTrack(output_track)
if out_recorder:
logger.info("Track %s is added to out_recorder", output_track.kind)
out_recorder.addTrack(output_track)
if in_recorder:
logger.info("Track %s is added to in_recorder", input_track.kind)
in_recorder.addTrack(input_track)
@input_track.on("ended")
async def on_ended():
logger.info("Track %s ended", input_track.kind)
if in_recorder:
await in_recorder.stop()
if out_recorder:
await out_recorder.stop()
elif mode == WebRtcMode.SENDONLY:
@pc.on("track")
def on_track(input_track):
logger.info("Track %s received", input_track.kind)
if input_track.kind == "audio":
# Not supported yet
pass
elif input_track.kind == "video":
if video_receiver:
logger.info(
"Add a track %s to receiver %s", input_track, video_receiver
)
video_receiver.addTrack(input_track)
if in_recorder:
logger.info("Track %s is added to in_recorder", input_track.kind)
in_recorder.addTrack(input_track)
@input_track.on("ended")
async def on_ended():
logger.info("Track %s ended", input_track.kind)
if video_receiver:
video_receiver.stop()
if in_recorder:
await in_recorder.stop()
await pc.setRemoteDescription(offer)
if mode == WebRtcMode.RECVONLY:
for t in pc.getTransceivers():
output_track = None
if t.kind == "audio":
if player and player.audio:
output_track = player.audio
# pc.addTrack(player.audio)
elif t.kind == "video":
if player and player.video:
# pc.addTrack(player.video)
output_track = player.video
if output_track:
pc.addTrack(output_track)
# NOTE: Recording is not supported in this mode
# because connecting player to recorder does not work somehow;
# it generates unplayable movie files.
if video_receiver and video_receiver.hasTrack():
video_receiver.start()
if in_recorder:
await in_recorder.start()
if out_recorder:
await out_recorder.start()
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
callback(pc.localDescription)
except Exception as e:
logger.debug("Error occurred in process_offer")
logger.debug(e)
callback(e)
async def offer(request):
params = await request.json()
print("got offer request: \n" + params["sdp"])
print("offer type: " + params["type"])
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
pc = RTCPeerConnection()
pc_id = "PeerConnection(%s)" % uuid.uuid4()
pcs.add(pc)
print("Created for %s", request.remote)
@pc.on("datachannel")
def on_datachannel(channel):
global io_channel
io_channel = channel
@channel.on("message")
def on_message(message):
if isinstance(message, str):
if message.startswith("ch"):
new_channels = list(
map(lambda x: int(x),
message.split(" ", 1)[1].split()))
new_channels = {
"1": new_channels[0],
"2": new_channels[1],
"3": new_channels[2],
"4": new_channels[3],
}
global channels
channels = new_channels
@pc.on("iceconnectionstatechange")
async def on_iceconnectionstatechange():
print("ICE connection state is %s", pc.iceConnectionState)
if pc.iceConnectionState == "failed":
await pc.close()
pcs.discard(pc)
elif pc.iceConnectionState == "closed":
subprocess.Popen(
"ps x | awk {'{print $1}'} | awk 'NR > 1' | xargs kill",
shell=True)
# handle offer
await pc.setRemoteDescription(offer)
for t in pc.getTransceivers():
if t.kind == "video":
pc.addTrack(VideoTrack())
print("added track")
# send answer
answer = await pc.createAnswer()
print("answer: " + str(answer))
await pc.setLocalDescription(answer)
print("sdp" + pc.localDescription.sdp)
return web.Response(
content_type="application/json",
text=json.dumps({
"sdp": pc.localDescription.sdp,
"type": pc.localDescription.type
}),
)
async def _process_offer(
mode: WebRtcMode,
pc: RTCPeerConnection,
offer: RTCSessionDescription,
player_factory: Optional[MediaPlayerFactory],
video_transformer: Optional[VideoTransformerBase],
video_receiver: Optional[VideoReceiver],
async_transform: bool,
callback: Callable[[Union[RTCSessionDescription, Exception]], None],
):
try:
player = None
if player_factory:
player = player_factory()
@pc.on("iceconnectionstatechange")
async def on_iceconnectionstatechange():
logger.info("ICE connection state is %s", pc.iceConnectionState)
if pc.iceConnectionState == "failed":
await pc.close()
if mode == WebRtcMode.SENDRECV:
@pc.on("track")
def on_track(track):
logger.info("Track %s received", track.kind)
if track.kind == "audio":
if player and player.audio:
pc.addTrack(player.audio)
elif track.kind == "video":
if player and player.video:
logger.info("Add player to video track")
pc.addTrack(player.video)
elif video_transformer:
VideoTrack = (AsyncVideoTransformTrack if
async_transform else VideoTransformTrack)
logger.info(
"Add a input video track %s to "
"another track with video_transformer %s",
track,
VideoTrack,
)
local_video = VideoTrack(
track=track, video_transformer=video_transformer)
logger.info(
"Add the video track with transfomer to %s", pc)
pc.addTrack(local_video)
elif mode == WebRtcMode.SENDONLY:
@pc.on("track")
def on_track(track):
logger.info("Track %s received", track.kind)
if track.kind == "audio":
# Not supported yet
pass
elif track.kind == "video":
if video_receiver:
logger.info("Add a track %s to receiver %s", track,
video_receiver)
video_receiver.addTrack(track)
@track.on("ended")
async def on_ended():
logger.info("Track %s ended", track.kind)
if video_receiver:
video_receiver.stop()
await pc.setRemoteDescription(offer)
if mode == WebRtcMode.RECVONLY:
for t in pc.getTransceivers():
if t.kind == "audio":
if player and player.audio:
pc.addTrack(player.audio)
elif t.kind == "video":
if player and player.video:
pc.addTrack(player.video)
if video_receiver and video_receiver.hasTrack():
video_receiver.start()
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
callback(pc.localDescription)
except Exception as e:
logger.debug("Error occurred in process_offer")
logger.debug(e)
callback(e)
async def offer(request):
if len(pcs) < MAX_CONNECTION_NUM:
params = await request.json()
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
idNum = params["idNum"]
# print("idNum(offer) : %d" % idNum)
# print("offer func")
# print("path : %s" % request.rel_url.path)
# print(request.rel_url.path == "/offerFilter")
camFilter = (request.rel_url.path == "/offerFilter")
indexNum = getIndexFromId(idNum)
if indexNum >= 0:
if userList[indexNum][0] == request.remote:
pc = RTCPeerConnection()
pcs.add(pc)
userList[indexNum][2] = pc
userList[indexNum][4] = camFilter
# print(pc)
# print(vars(request))
@pc.on("iceconnectionstatechange")
async def on_connectionstatechange():
print("Connection state is %s" % pc.connectionState)
if pc.connectionState == "failed":
deleteAnObj(pc)
await pc.close()
pcs.discard(pc)
owner = False
if userList[indexNum][1] == 0:
owner = True
# open media source
audio, video = create_local_tracks(args.play_from, owner = owner, camFilter = camFilter)
await pc.setRemoteDescription(offer)
for t in pc.getTransceivers():
if t.kind == "audio" and audio:
pc.addTrack(audio)
elif t.kind == "video" and video:
pc.addTrack(video)
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
return web.Response(
content_type="application/json",
text=json.dumps(
{"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}
),
)
else:
print("request remote : %s" % request.remote)
return web.Response(content_type="text/html", text="busy")
else:
print("indexNum error : %d" % indexNum)
return web.Response(content_type="text/html", text="busy")
else:
return web.Response(content_type="text/html", text="busy")
async def offer(request):
#cap rtc peers at <X>
if len(pcs) > pcs_max_size:
return
params = await request.json()
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
uuid = str(params["uuid"])
pc = next((x for x in pcs if x.uuid == uuid), None)
if pc is None:
pc = RTCPeerConnection()
# configuration=RTCConfiguration(
# iceServers=[
# RTCIceServer(urls=["stun:stun.l.google:19302"]),
# RTCIceServer(urls=["turn:0.peerjs.com:3478"], username="******", credential="peerjsp"),
# RTCIceServer(urls=["turn:turn.bistri.com:80"], username="******", credential="homeo"),
# RTCIceServer(urls=["turn:turn.anyfirewall.com:443"], username="******", credential="webrtc")
# ]))
setattr(pc, 'uuid', uuid)
pcs.append(pc)
@pc.on("datachannel")
def on_datachannel(channel):
print("Channel id: {}".format(channel.id))
#print("queue position: {}".format(pcs.index(pc)))
#This transport monitor is necessary, otherwise
#a closed connection breaks the server on the next connection
async def monitor():
while True:
if channel.transport.transport.state == "closed":
await pc.close()
js = getattr(pc, 'js', None)
if js is not None:
jsdev = next((x for x in jslist if x.dev == js),
None)
if jsdev is not None:
print("Freeing joystick:{}".format(jsdev.idx))
jslist[jslist.index(jsdev)] = jsdev._replace(
locked=False)
try:
pcs.remove(pc)
except ValueError:
pass
break
await asyncio.sleep(5)
asyncio.ensure_future(monitor())
@channel.on("close")
def on_close():
print("Close Channel id: {}".format(channel.id))
js = getattr(pc, 'js', None)
if js is not None:
jsdev = next((x for x in jslist if x.dev == js), None)
if jsdev is not None:
print("Freeing joystick:{}".format(jsdev.idx))
jslist[jslist.index(jsdev)] = jsdev._replace(
locked=False)
setattr(pc, 'js', None)
@channel.on("message")
async def on_message(message):
#we use the 1 a second ping as heartbeat (and query)
if isinstance(message, str) and message.startswith("ping"):
try:
#if at the top of the queue and not assigned
if pcs.index(pc) < len(jslist):
if not hasattr(pc, 'js'):
# if they don't have a js, and one is available
jsdev = next(
(x for x in jslist if x.locked == False),
None)
if jsdev is not None:
setattr(pc, 'js', jsdev.dev)
print("Assigning joystick:{}".format(
jsdev.idx))
jslist[jslist.index(
jsdev)] = jsdev._replace(locked=True)
#trigger to start a direct webrtc video feed (low latency)
channel.send("start: Player {}".format(
jsdev.idx))
else:
# if they do have a joystick, and people are waiting
if (len(pcs) > len(jslist)):
#set future timestamp if not set
if not hasattr(pc, 'ts'):
#lets do 60 seconds plus a bit of hidden stopped time
setattr(pc, 'ts', int(time()) + 60 + 5)
remaining_time = pc.ts - int(time())
if remaining_time < 0:
channel.send("stop")
await asyncio.sleep(0.1)
channel.close()
for t in pc.getTransceivers():
await t.stop()
await pc.close()
js = getattr(pc, 'js', None)
if js is not None:
jsdev = next((x for x in jslist
if x.dev == js),
None)
if jsdev is not None:
print("Freeing joystick:{}".
format(jsdev.idx))
jslist[jslist.index(
jsdev)] = jsdev._replace(
locked=False)
try:
pcs.remove(pc)
except ValueError:
pass
else:
# 5 second hidden game over time
channel.send("Time: {}".format(
max(0, remaining_time - 5)))
else:
channel.send("Time: --")
else:
#send their position in the waitlist
channel.send("Waitlist: {} of {}".format(
pcs.index(pc) + 1 - len(jslist),
len(pcs) - len(jslist)))
except ConnectionError:
pass
elif isinstance(message,
str) and message.startswith("controller: "):
#if in the last "hidden" 5 seconds of a turn, ignore buttons
if hasattr(pc, 'ts') and (pc.ts - int(time()) < 5):
return
vals = message[12:].split(',')
if getattr(pc, 'js', None) is not None:
jsupdate_vals(pc.js, vals)
elif isinstance(message, str) and message.startswith("key"):
#if in the last "hidden" 5 seconds of a turn, ignore buttons
if hasattr(pc, 'ts') and (pc.ts - int(time()) < 5):
return
direction = message[3]
key = message[7]
dval = 1 if direction == "d" else 0
keys = ['z', 'x', 's', 'u', 'l', 'd', 'r']
kpos = keys.index(key)
if getattr(pc, 'js', None) is not None:
if (kpos != 2):
pc.js.emit(events[kpos], int(dval))
#pico8 hack to use start button to restart pico8 back to main menu
elif (kpos == 2 and int(dval) == 1
and pico8proc is not None):
pico8proc.stop()
pico8proc.is_started = False
pico8proc.start()
pc.js.flush()
elif isinstance(message,
str) and message.startswith("gamectl: switch"):
#special game control just for pico-8 (for now)
gameidx = (gameidx + 1) % len(gamelist)
nextgame = gamelist[gameidx]
try:
os.symlink('pico8/data_{}.pod'.format(nextgame),
'pico8/data_tmp.pod')
os.replace('pico8/data_tmp.pod', 'pico8/data.pod')
except FileExistsError:
pass
#sendkeys CTRL+R 'reload' (super ugly CTRL down, then R, then up, up)
#even uglier, need to send a couple of times to ensure it gets caught
for _ in range(6):
self.send_sdl_event(0x400000e0, 0xe0, 1, kmod=kmod)
self.send_sdl_event(ord('r'), 0x15, 1, kmod=kmod)
self.send_sdl_event(0x400000e0, 0xe0, 0, kmod=kmod)
self.send_sdl_event(ord('r'), 0x15, 0, kmod=kmod)
@pc.on("iceconnectionstatechange")
async def on_iceconnectionstatechange():
print("ICE connection state is %s" % pc.iceConnectionState)
if pc.iceConnectionState == "failed":
await pc.close()
js = getattr(pc, 'js', None)
if js is not None:
jsdev = next((x for x in jslist if x.dev == js), None)
if jsdev is not None:
print("Freeing joystick:{}".format(jsdev.idx))
jslist[jslist.index(jsdev)] = jsdev._replace(
locked=False)
try:
pcs.remove(pc)
except ValueError:
pass
# handle offer
await pc.setRemoteDescription(offer)
for t in pc.getTransceivers():
if t.kind == "video":
pc.addTrack(VideoImageTrack())
# send answer
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
return web.Response(
content_type="application/json",
text=json.dumps({
"sdp": pc.localDescription.sdp,
"type": pc.localDescription.type
}),
)
async def _process_offer_coro(
mode: WebRtcMode,
pc: RTCPeerConnection,
offer: RTCSessionDescription,
relay: MediaRelay,
source_video_track: Optional[MediaStreamTrack],
source_audio_track: Optional[MediaStreamTrack],
in_recorder: Optional[MediaRecorder],
out_recorder: Optional[MediaRecorder],
video_processor: Optional[VideoProcessorBase],
audio_processor: Optional[AudioProcessorBase],
video_receiver: Optional[VideoReceiver],
audio_receiver: Optional[AudioReceiver],
async_processing: bool,
sendback_video: bool,
sendback_audio: bool,
on_track_created: Callable[[TrackType, MediaStreamTrack], None],
):
if mode == WebRtcMode.SENDRECV:
@pc.on("track")
def on_track(input_track):
logger.info("Track %s received", input_track.kind)
if input_track.kind == "video":
on_track_created("input:video", input_track)
elif input_track.kind == "audio":
on_track_created("input:audio", input_track)
output_track = None
if input_track.kind == "audio":
if source_audio_track:
logger.info("Set %s as an input audio track",
source_audio_track)
output_track = source_audio_track
elif audio_processor:
AudioTrack = (AsyncAudioProcessTrack
if async_processing else AudioProcessTrack)
logger.info(
"Set %s as an input audio track with audio_processor %s",
input_track,
AudioTrack,
)
output_track = AudioTrack(
track=relay.subscribe(input_track),
processor=audio_processor,
)
else:
output_track = input_track # passthrough
elif input_track.kind == "video":
if source_video_track:
logger.info("Set %s as an input video track",
source_video_track)
output_track = source_video_track
elif video_processor:
VideoTrack = (AsyncVideoProcessTrack
if async_processing else VideoProcessTrack)
logger.info(
"Set %s as an input video track with video_processor %s",
input_track,
VideoTrack,
)
output_track = VideoTrack(
track=relay.subscribe(input_track),
processor=video_processor,
)
else:
output_track = input_track
if (output_track.kind == "video"
and sendback_video) or (output_track.kind == "audio"
and sendback_audio):
logger.info(
"Add a track %s of kind %s to %s",
output_track,
output_track.kind,
pc,
)
pc.addTrack(relay.subscribe(output_track))
else:
logger.info("Block a track %s of kind %s", output_track,
output_track.kind)
if out_recorder:
logger.info("Track %s is added to out_recorder",
output_track.kind)
out_recorder.addTrack(relay.subscribe(output_track))
if in_recorder:
logger.info("Track %s is added to in_recorder",
input_track.kind)
in_recorder.addTrack(relay.subscribe(input_track))
if output_track.kind == "video":
on_track_created("output:video", output_track)
elif output_track.kind == "audio":
on_track_created("output:audio", output_track)
@input_track.on("ended")
async def on_ended():
logger.info("Track %s ended", input_track.kind)
if in_recorder:
await in_recorder.stop()
if out_recorder:
await out_recorder.stop()
elif mode == WebRtcMode.SENDONLY:
@pc.on("track")
def on_track(input_track):
logger.info("Track %s received", input_track.kind)
if input_track.kind == "video":
on_track_created("input:video", input_track)
elif input_track.kind == "audio":
on_track_created("input:audio", input_track)
if input_track.kind == "audio":
if audio_receiver:
logger.info("Add a track %s to receiver %s", input_track,
audio_receiver)
audio_receiver.addTrack(input_track)
elif input_track.kind == "video":
if video_receiver:
logger.info("Add a track %s to receiver %s", input_track,
video_receiver)
video_receiver.addTrack(input_track)
if in_recorder:
logger.info("Track %s is added to in_recorder",
input_track.kind)
in_recorder.addTrack(input_track)
@input_track.on("ended")
async def on_ended():
logger.info("Track %s ended", input_track.kind)
if video_receiver:
video_receiver.stop()
if audio_receiver:
audio_receiver.stop()
if in_recorder:
await in_recorder.stop()
await pc.setRemoteDescription(offer)
if mode == WebRtcMode.RECVONLY:
for t in pc.getTransceivers():
output_track = None
if t.kind == "audio":
if source_audio_track:
if audio_processor:
AudioTrack = (AsyncAudioProcessTrack if
async_processing else AudioProcessTrack)
logger.info(
"Set %s as an input audio track with audio_processor %s",
source_audio_track,
AudioTrack,
)
output_track = AudioTrack(track=source_audio_track,
processor=audio_processor)
else:
output_track = source_audio_track # passthrough
elif t.kind == "video":
if source_video_track:
if video_processor:
VideoTrack = (AsyncVideoProcessTrack if
async_processing else VideoProcessTrack)
logger.info(
"Set %s as an input video track with video_processor %s",
source_video_track,
VideoTrack,
)
output_track = VideoTrack(track=source_video_track,
processor=video_processor)
else:
output_track = source_video_track # passthrough
if output_track:
logger.info("Add a track %s to %s", output_track, pc)
pc.addTrack(relay.subscribe(output_track))
# NOTE: Recording is not supported in this mode
# because connecting player to recorder does not work somehow;
# it generates unplayable movie files.
if output_track.kind == "video":
on_track_created("output:video", output_track)
elif output_track.kind == "audio":
on_track_created("output:audio", output_track)
if video_receiver and video_receiver.hasTrack():
video_receiver.start()
if audio_receiver and audio_receiver.hasTrack():
audio_receiver.start()
if in_recorder:
await in_recorder.start()
if out_recorder:
await out_recorder.start()
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
return pc.localDescription
def test_connect_datachannel(self):
pc1 = RTCPeerConnection()
pc1_data_messages = []
pc1_states = track_states(pc1)
pc2 = RTCPeerConnection()
pc2_data_channels = []
pc2_data_messages = []
pc2_states = track_states(pc2)
@pc2.on('datachannel')
def on_datachannel(channel):
self.assertEqual(channel.readyState, 'open')
pc2_data_channels.append(channel)
@channel.on('message')
def on_message(message):
pc2_data_messages.append(message)
if isinstance(message, str):
channel.send('string-echo: ' + message)
else:
channel.send(b'binary-echo: ' + message)
# create data channel
dc = pc1.createDataChannel('chat', protocol='bob')
self.assertEqual(dc.label, 'chat')
self.assertEqual(dc.protocol, 'bob')
self.assertEqual(dc.readyState, 'connecting')
# send messages
dc.send('hello')
dc.send('')
dc.send(b'\x00\x01\x02\x03')
dc.send(b'')
dc.send(LONG_DATA)
with self.assertRaises(ValueError) as cm:
dc.send(1234)
self.assertEqual(str(cm.exception),
"Cannot send unsupported data type: <class 'int'>")
@dc.on('message')
def on_message(message):
pc1_data_messages.append(message)
# create offer
offer = run(pc1.createOffer())
self.assertEqual(offer.type, 'offer')
self.assertTrue('m=application ' in offer.sdp)
self.assertFalse('a=candidate:' in offer.sdp)
run(pc1.setLocalDescription(offer))
self.assertEqual(pc1.iceConnectionState, 'new')
self.assertEqual(pc1.iceGatheringState, 'complete')
self.assertTrue('m=application ' in pc1.localDescription.sdp)
self.assertTrue('a=candidate:' in pc1.localDescription.sdp)
self.assertTrue('a=sctpmap:5000 webrtc-datachannel 65535' in
pc1.localDescription.sdp)
self.assertTrue('a=fingerprint:sha-256' in pc1.localDescription.sdp)
self.assertTrue('a=setup:actpass' in pc1.localDescription.sdp)
# handle offer
run(pc2.setRemoteDescription(pc1.localDescription))
self.assertEqual(pc2.remoteDescription, pc1.localDescription)
self.assertEqual(len(pc2.getReceivers()), 0)
self.assertEqual(len(pc2.getSenders()), 0)
self.assertEqual(len(pc2.getTransceivers()), 0)
# create answer
answer = run(pc2.createAnswer())
self.assertEqual(answer.type, 'answer')
self.assertTrue('m=application ' in answer.sdp)
self.assertFalse('a=candidate:' in answer.sdp)
run(pc2.setLocalDescription(answer))
self.assertEqual(pc2.iceConnectionState, 'checking')
self.assertEqual(pc2.iceGatheringState, 'complete')
self.assertTrue('m=application ' in pc2.localDescription.sdp)
self.assertTrue('a=candidate:' in pc2.localDescription.sdp)
self.assertTrue('a=sctpmap:5000 webrtc-datachannel 65535' in
pc2.localDescription.sdp)
self.assertTrue('a=fingerprint:sha-256' in pc2.localDescription.sdp)
self.assertTrue('a=setup:active' in pc2.localDescription.sdp)
# handle answer
run(pc1.setRemoteDescription(pc2.localDescription))
self.assertEqual(pc1.remoteDescription, pc2.localDescription)
self.assertEqual(pc1.iceConnectionState, 'checking')
# check outcome
run(asyncio.sleep(1))
self.assertEqual(pc1.iceConnectionState, 'completed')
self.assertEqual(pc2.iceConnectionState, 'completed')
self.assertEqual(dc.readyState, 'open')
# check pc2 got a datachannel
self.assertEqual(len(pc2_data_channels), 1)
self.assertEqual(pc2_data_channels[0].label, 'chat')
self.assertEqual(pc2_data_channels[0].protocol, 'bob')
# check pc2 got messages
run(asyncio.sleep(1))
self.assertEqual(pc2_data_messages, [
'hello',
'',
b'\x00\x01\x02\x03',
b'',
LONG_DATA,
])
# check pc1 got replies
self.assertEqual(pc1_data_messages, [
'string-echo: hello',
'string-echo: ',
b'binary-echo: \x00\x01\x02\x03',
b'binary-echo: ',
b'binary-echo: ' + LONG_DATA,
])
# close data channel
dc.close()
self.assertEqual(dc.readyState, 'closed')
# close
run(pc1.close())
run(pc2.close())
self.assertEqual(pc1.iceConnectionState, 'closed')
self.assertEqual(pc2.iceConnectionState, 'closed')
# check state changes
self.assertEqual(pc1_states['iceConnectionState'],
['new', 'checking', 'completed', 'closed'])
self.assertEqual(pc1_states['iceGatheringState'],
['new', 'gathering', 'complete'])
self.assertEqual(pc1_states['signalingState'],
['stable', 'have-local-offer', 'stable', 'closed'])
self.assertEqual(pc2_states['iceConnectionState'],
['new', 'checking', 'completed', 'closed'])
self.assertEqual(pc2_states['iceGatheringState'],
['new', 'gathering', 'complete'])
self.assertEqual(pc2_states['signalingState'],
['stable', 'have-remote-offer', 'stable', 'closed'])
def test_connect_audio_mid_changes(self):
pc1 = RTCPeerConnection()
pc1_states = track_states(pc1)
pc2 = RTCPeerConnection()
pc2_states = track_states(pc2)
self.assertEqual(pc1.iceConnectionState, 'new')
self.assertEqual(pc1.iceGatheringState, 'new')
self.assertIsNone(pc1.localDescription)
self.assertIsNone(pc1.remoteDescription)
self.assertEqual(pc2.iceConnectionState, 'new')
self.assertEqual(pc2.iceGatheringState, 'new')
self.assertIsNone(pc2.localDescription)
self.assertIsNone(pc2.remoteDescription)
# add audio tracks immediately
pc1.addTrack(AudioStreamTrack())
pc1.getTransceivers()[0].mid = 'sdparta_0' # pretend we're Firefox!
self.assertEqual(mids(pc1), ['sdparta_0'])
pc2.addTrack(AudioStreamTrack())
self.assertEqual(mids(pc2), ['audio'])
# create offer
offer = run(pc1.createOffer())
self.assertEqual(offer.type, 'offer')
self.assertTrue('m=audio ' in offer.sdp)
self.assertFalse('a=candidate:' in offer.sdp)
run(pc1.setLocalDescription(offer))
self.assertEqual(pc1.iceConnectionState, 'new')
self.assertEqual(pc1.iceGatheringState, 'complete')
self.assertTrue('m=audio ' in pc1.localDescription.sdp)
self.assertTrue('a=candidate:' in pc1.localDescription.sdp)
self.assertTrue('a=sendrecv' in pc1.localDescription.sdp)
self.assertTrue('a=fingerprint:sha-256' in pc1.localDescription.sdp)
self.assertTrue('a=setup:actpass' in pc1.localDescription.sdp)
self.assertTrue('a=mid:sdparta_0' in pc1.localDescription.sdp)
# handle offer
run(pc2.setRemoteDescription(pc1.localDescription))
self.assertEqual(pc2.remoteDescription, pc1.localDescription)
self.assertEqual(len(pc2.getReceivers()), 1)
self.assertEqual(len(pc2.getSenders()), 1)
self.assertEqual(mids(pc2), ['sdparta_0'])
# create answer
answer = run(pc2.createAnswer())
self.assertEqual(answer.type, 'answer')
self.assertTrue('m=audio ' in answer.sdp)
self.assertFalse('a=candidate:' in answer.sdp)
run(pc2.setLocalDescription(answer))
self.assertEqual(pc2.iceConnectionState, 'checking')
self.assertEqual(pc2.iceGatheringState, 'complete')
self.assertTrue('m=audio ' in pc2.localDescription.sdp)
self.assertTrue('a=candidate:' in pc2.localDescription.sdp)
self.assertTrue('a=sendrecv' in pc1.localDescription.sdp)
self.assertTrue('a=fingerprint:sha-256' in pc2.localDescription.sdp)
self.assertTrue('a=setup:active' in pc2.localDescription.sdp)
self.assertTrue('a=mid:sdparta_0' in pc2.localDescription.sdp)
# handle answer
run(pc1.setRemoteDescription(pc2.localDescription))
self.assertEqual(pc1.remoteDescription, pc2.localDescription)
self.assertEqual(pc1.iceConnectionState, 'checking')
# check outcome
run(asyncio.sleep(1))
self.assertEqual(pc1.iceConnectionState, 'completed')
self.assertEqual(pc2.iceConnectionState, 'completed')
# close
run(pc1.close())
run(pc2.close())
self.assertEqual(pc1.iceConnectionState, 'closed')
self.assertEqual(pc2.iceConnectionState, 'closed')
# check state changes
self.assertEqual(pc1_states['iceConnectionState'],
['new', 'checking', 'completed', 'closed'])
self.assertEqual(pc1_states['iceGatheringState'],
['new', 'gathering', 'complete'])
self.assertEqual(pc1_states['signalingState'],
['stable', 'have-local-offer', 'stable', 'closed'])
self.assertEqual(pc2_states['iceConnectionState'],
['new', 'checking', 'completed', 'closed'])
self.assertEqual(pc2_states['iceGatheringState'],
['new', 'gathering', 'complete'])
self.assertEqual(pc2_states['signalingState'],
['stable', 'have-remote-offer', 'stable', 'closed'])
class RTCConnection(SubscriptionProducerConsumer):
_log = logging.getLogger("rtcbot.RTCConnection")
def __init__(
self,
defaultChannelOrdered=True,
loop=None,
rtcConfiguration=RTCConfiguration(
[RTCIceServer(urls="stun:stun.l.google.com:19302")]),
video_codec_preference=None,
audio_codec_preference=None,
):
super().__init__(
directPutSubscriptionType=asyncio.Queue,
defaultSubscriptionType=asyncio.Queue,
logger=self._log,
)
self._loop = loop
if self._loop is None:
self._loop = asyncio.get_event_loop()
self._dataChannels = {}
# These allow us to easily signal when the given events happen
self._dataChannelSubscriber = SubscriptionProducer(
logger=self._log.getChild("dataChannelSubscriber"))
self._rtc = RTCPeerConnection(configuration=rtcConfiguration)
self._rtc.on("datachannel", self._onDatachannel)
self._rtc.on("iceconnectionstatechange",
self._onIceConnectionStateChange)
self._rtc.on("track", self._onTrack)
self._hasRemoteDescription = False
self._defaultChannelOrdered = defaultChannelOrdered
self._videoHandler = ConnectionVideoHandler(self._rtc)
self._video_codec_preference = video_codec_preference
self._audioHandler = ConnectionAudioHandler(self._rtc)
self._audio_codec_preference = audio_codec_preference
# When the video/audio handlers get closed, close the entire connection
self._videoHandler.onClose(self.close)
self._audioHandler.onClose(self.close)
async def getLocalDescription(self, description=None):
"""
Gets the description to send on. Creates an initial description
if no remote description was passed, and creates a response if
a remote was given,
"""
for t in self._rtc.getTransceivers():
if t.kind == "video" and self._video_codec_preference is not None:
t.setCodecPreferences(self._video_codec_preference)
elif t.kind == "audio" and self._audio_codec_preference is not None:
t.setCodecPreferences(self._audio_codec_preference)
if self._hasRemoteDescription or description is not None:
# This means that we received an offer - either the remote description
# was already set, or we passed in a description. In either case,
# instead of initializing a new connection, we prepare a response
if not self._hasRemoteDescription:
await self.setRemoteDescription(description)
self._log.debug("Creating response to connection offer")
try:
answer = await self._rtc.createAnswer()
except AttributeError:
self._log.exception(
"\n>>> Looks like the offer didn't include the necessary info to set up audio/video. See RTCConnection.video.offerToReceive(). <<<\n\n"
)
raise
await self._rtc.setLocalDescription(answer)
return {
"sdp": self._rtc.localDescription.sdp,
"type": self._rtc.localDescription.type,
}
# There was no remote description, which means that we are initializing the
# connection.
# Before starting init, we create a default data channel for the connection
self._log.debug("Setting up default data channel")
channel = DataChannel(
self._rtc.createDataChannel("default",
ordered=self._defaultChannelOrdered))
# Subscribe the default channel directly to our own inputs and outputs.
# We have it listen to our own self._get, and write to our self._put_nowait
channel.putSubscription(NoClosedSubscription(self._get))
channel.subscribe(self._put_nowait)
channel.onReady(lambda: self._setReady(channel.ready))
channel.onClose(self.close)
self._dataChannels[channel.name] = channel
# Make sure we offer to receive video and audio if if isn't set up yet with
# all the receiving transceivers
if len(self.video._senders) < self.video._offerToReceive:
self._log.debug("Offering to receive video")
for i in range(self.video._offerToReceive -
len(self.video._senders)):
self._rtc.addTransceiver("video", "recvonly")
if len(self.audio._senders) < self.audio._offerToReceive:
self._log.debug("Offering to receive audio")
for i in range(self.audio._offerToReceive -
len(self.audio._senders)):
self._rtc.addTransceiver("audio", "recvonly")
self._log.debug("Creating new connection offer")
offer = await self._rtc.createOffer()
await self._rtc.setLocalDescription(offer)
return {
"sdp": self._rtc.localDescription.sdp,
"type": self._rtc.localDescription.type,
}
async def setRemoteDescription(self, description):
self._log.debug("Setting remote connection description")
await self._rtc.setRemoteDescription(
RTCSessionDescription(**description))
self._hasRemoteDescription = True
async def _onIceConnectionStateChange(self):
self._log.debug("ICE state: %s", self._rtc.iceConnectionState)
if self._rtc.iceConnectionState == "failed":
self._setError(self._rtc.iceConnectionState)
await self.close()
def _onDatachannel(self, channel):
"""
When a data channel comes in, adds it to the data channels, and sets up its messaging and stuff.
"""
channel = DataChannel(channel)
self._log.debug("Got channel: %s", channel.name)
if channel.name == "default":
# Subscribe the default channel directly to our own inputs and outputs.
# We have it listen to our own self._get, and write to our self._put_nowait
channel.putSubscription(NoClosedSubscription(self._get))
channel.subscribe(self._put_nowait)
channel.onReady(lambda: self._setReady(channel.ready))
channel.onClose(self.close)
else:
self._dataChannelSubscriber.put_nowait(channel)
self._dataChannels[channel.name] = channel
def _onTrack(self, track):
self._log.debug("Received %s track from connection", track.kind)
if track.kind == "audio":
self._audioHandler._onTrack(track)
elif track.kind == "video":
self._videoHandler._onTrack(track)
def onDataChannel(self, callback=None):
"""
Acts as a subscriber...
"""
return self._dataChannelSubscriber.subscribe(callback)
def addDataChannel(self, name, ordered=True):
"""
Adds a data channel to the connection. Note that the RTCConnection adds a "default" channel
automatically, which you can subscribe to directly.
"""
self._log.debug("Adding data channel to connection")
if name in self._dataChannels or name == "default":
raise KeyError("Data channel %s already exists", name)
dc = DataChannel(self._rtc.createDataChannel(name, ordered=ordered))
self._dataChannels[name] = dc
return dc
def getDataChannel(self, name):
"""
Returns the data channel with the given name. Please note that the "default" channel is considered special,
and is not returned.
"""
if name == "default":
raise KeyError(
"Default channel not available for 'get'. Use the RTCConnection's subscribe and put_nowait methods for access to it."
)
return self._dataChannels[name]
@property
def video(self):
"""
Convenience function - you can subscribe to it to get video frames once they show up
"""
return self._videoHandler
@property
def audio(self):
"""
Convenience function - you can subscribe to it to get audio once a stream is received
"""
return self._audioHandler
def close(self):
"""
If the loop is running, returns a future that will close the connection. Otherwise, runs
the loop temporarily to complete closing.
"""
if self.closed:
if self._loop.is_running():
async def donothing():
pass
return asyncio.ensure_future(donothing())
return None
self._log.debug("Closing connection")
super().close()
# And closes all tracks
self.video.close()
self.audio.close()
for dc in self._dataChannels:
self._dataChannels[dc].close()
self._dataChannelSubscriber.close()
if self._loop.is_running():
self._log.debug("Loop is running - close will return a future!")
return asyncio.ensure_future(self._rtc.close())
else:
self._loop.run_until_complete(self._rtc.close())
return None
def send(self, msg):
"""
Send is an alias for put_nowait - makes it easier for people new to rtcbot to understand
what is going on
"""
self.put_nowait(msg)
class Camera(PiResource):
async def handel(self, actionType, data):
if actionType == ActionType.SHOT:
await self.___shot(data)
if actionType == ActionType.RECORD:
await self.___record(data)
if actionType == ActionType.STREAM:
await self.___stream(data)
if actionType == ActionType.TEARDOWN:
await self.___teardown(data)
pass
async def available(self):
self.___available = True
async def close(self):
print("close Camera")
await self.___teardown({})
self.___available = False
pass
___pc = None
___player = None
___options = {"framerate": "25", "video_size": "640x480"}
def __init__(self, param):
self.___param = param
self.___available = True
async def ___shot(self, data):
print("SHOT ", "Camera")
return None
async def ___record(self, data):
print("RECORD ", "Camera")
return None
async def ___stream(self, data):
if self.___available and data:
offer = RTCSessionDescription(sdp=data["sdp"], type=data["type"])
self.___pc = RTCPeerConnection()
@self.___pc.on("iceconnectionstatechange")
async def on_iceconnectionstatechange():
print(self.___pc.iceConnectionState)
if self.___pc.iceConnectionState == "failed":
await self.___teardown({})
await self.___pc.setRemoteDescription(offer)
player = MediaPlayer("/dev/video0",
format="v4l2",
options=self.___options)
for t in self.___pc.getTransceivers():
if t.kind == "audio" and player.audio:
self.___pc.addTrack(player.audio)
elif t.kind == "video" and player.video:
self.___pc.addTrack(player.video)
answer = await self.___pc.createAnswer()
await self.___pc.setLocalDescription(answer)
print("STREAM ", "Camera")
res = {
"sdp": self.___pc.localDescription.sdp,
"type": self.___pc.localDescription.type,
"action": "STREAM"
}
self.___param["notify"](res)
self.___available = False
async def ___teardown(self, data):
print("TEARDOWN ", "Camera")
if self.___pc:
await self.___pc.close()
self.___available = True
async def offer(request):
params = await request.json()
print(params)
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
pc = RTCPeerConnection()
client_id = client_manager.create_new_client(pc)
# data = [True]
#recog_worker_thread = threading.Thread(target=recog_worker,args=(client_manager.get_client(client_id),data,))
#recog_worker_thread.start()
@pc.on("iceconnectionstatechange")
async def on_iceconnectionstatechange():
print("ICE connection state is %s" % pc.iceConnectionState)
if pc.iceConnectionState == "failed":
await client_manager.remove_client(
client_id) #this already handles pc closing
faceregtrack = FacialRecognitionTrack(face_detect,
client_manager.get_client(client_id))
@pc.on("datachannel")
def on_datachannel(channel):
@channel.on("message")
def on_message(message):
client = client_manager.get_client(client_id)
if "$register$" in message: #this is some poorman message handling :)
new_subject = message.split("$register$")[1]
print("Registering for subject: " + new_subject)
client.toggle_register_mode(new_subject)
channel.send("$register$") #ack back
elif "$recognize$" in message:
print("Turned on recognition mode")
client.toggle_recognition_mode()
channel.send("$recognize$") #ack back
@pc.on("close")
async def on_close(track):
print("Connection with client: " + str(client_id) + " closed")
#data[0] = False
#recog_worker_thread.join()
await client_manager.remove_client(client_id)
@pc.on("track")
def on_track(track):
print("Track %s received" % track.kind)
if track.kind == "video":
faceregtrack.update(track)
await pc.setRemoteDescription(offer)
for t in pc.getTransceivers():
if t.kind == "video":
pc.addTrack(faceregtrack)
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
print("Connection with client formed")
return web.Response(
content_type="application/json",
text=json.dumps({
"sdp": pc.localDescription.sdp,
"type": pc.localDescription.type
}),
)