async def handle_offer_for_webcam(self, request, ws): offer = RTCSessionDescription(sdp=request["sdp"], type=request["type"]) pc = RTCPeerConnection() pc_id = "PeerConnection(%s)" % uuid.uuid4() self.webrtc.pcs.add(pc) self.pcs[ws] = pc def log_info(msg, *args): self.logger.info(pc_id + " " + msg, *args) #log_info("Receiving for %s", request.remote) @pc.on("iceconnectionstatechange") async def on_iceconnectionstatechange(): log_info("ICE connection state is %s", pc.iceConnectionState) if pc.iceConnectionState == "failed": await pc.close() self.webrtc.pcs.discard(pc) # handle offer await pc.setRemoteDescription(offer) for t in pc.getTransceivers(): # if t.kind == "audio" and player.audio: # pc.addTrack(player.audio) if t.kind == "video": print(self.webrtc.local_video) pc.addTrack(self.webrtc.local_video) # send answer answer = await pc.createAnswer() await pc.setLocalDescription(answer) answer = {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type} answer_message = { "type": "ANSWER", "payload": {"message":answer} } log_info("Negotiation Answer- %s", answer_message) await ws.send_str(json.dumps(answer_message))
class Handler: def __init__(self, handlerId: str, channel: Channel, loop: asyncio.AbstractEventLoop, getTrack, addRemoteTrack, getRemoteTrack, configuration: Optional[RTCConfiguration] = None) -> None: self._handlerId = handlerId self._channel = channel self._pc = RTCPeerConnection(configuration or None) # dictionary of sending transceivers mapped by given localId self._sendTransceivers = dict() # type: Dict[str, RTCRtpTransceiver] # dictionary of dataChannelds mapped by internal id self._dataChannels = dict() # type: Dict[str, RTCDataChannel] # function returning a sending track given a player id and a kind self._getTrack = getTrack # function to store a receiving track self._addRemoteTrack = addRemoteTrack # function returning a receiving track self._getRemoteTrack = getRemoteTrack @self._pc.on("track") # type: ignore def on_track(track) -> None: Logger.debug( f"handler: ontrack [kind:{track.kind}, id:{track.id}]") # store it self._addRemoteTrack(track) @self._pc.on("signalingstatechange") # type: ignore async def on_signalingstatechange() -> None: Logger.debug( f"handler: signalingstatechange [state:{self._pc.signalingState}]" ) await self._channel.notify(self._handlerId, "signalingstatechange", self._pc.signalingState) @self._pc.on("icegatheringstatechange") # type: ignore async def on_icegatheringstatechange() -> None: Logger.debug( f"handler: icegatheringstatechange [state:{self._pc.iceGatheringState}]" ) await self._channel.notify(self._handlerId, "icegatheringstatechange", self._pc.iceGatheringState) @self._pc.on("iceconnectionstatechange") # type: ignore async def on_iceconnectionstatechange() -> None: Logger.debug( f"handler: iceconnectionstatechange [state:{self._pc.iceConnectionState}]" ) await self._channel.notify(self._handlerId, "iceconnectionstatechange", self._pc.iceConnectionState) async def checkDataChannelsBufferedAmount() -> None: while True: await asyncio.sleep(1) for dataChannelId, dataChannel in self._dataChannels.items(): await self._channel.notify(dataChannelId, "bufferedamount", dataChannel.bufferedAmount) self._dataChannelsBufferedAmountTask = loop.create_task( checkDataChannelsBufferedAmount()) async def close(self) -> None: # stop the periodic task self._dataChannelsBufferedAmountTask.cancel() # close peerconnection await self._pc.close() def dump(self) -> Any: result = { "id": self._handlerId, "signalingState": self._pc.signalingState, "iceConnectionState": self._pc.iceConnectionState, "iceGatheringState": self._pc.iceGatheringState, "transceivers": [], "sendTransceivers": [] } for transceiver in self._pc.getTransceivers(): transceiverInfo = { "mid": transceiver.mid, "stopped": transceiver.stopped, "kind": transceiver.kind, "currentDirection": transceiver.currentDirection, "direction": transceiver.direction, "sender": { "trackId": transceiver.sender.track.id if transceiver.sender.track else None }, "receiver": { "trackId": transceiver.receiver.track.id if transceiver.receiver.track else None } } result["transceivers"].append(transceiverInfo) for localId, transceiver in self._sendTransceivers.items(): sendTransceiverInfo = {"localId": localId, "mid": transceiver.mid} result["sendTransceivers"].append(sendTransceiverInfo) return result async def processRequest(self, request: Request) -> Any: if request.method == "handler.getLocalDescription": localDescription = self._pc.localDescription if (localDescription is not None): return { "type": localDescription.type, "sdp": localDescription.sdp } else: return None elif request.method == "handler.createOffer": offer = await self._pc.createOffer() return {"type": offer.type, "sdp": offer.sdp} elif request.method == "handler.createAnswer": answer = await self._pc.createAnswer() return {"type": answer.type, "sdp": answer.sdp} elif request.method == "handler.setLocalDescription": data = request.data if isinstance(data, RTCSessionDescription): raise TypeError("request data not a RTCSessionDescription") description = RTCSessionDescription(**data) await self._pc.setLocalDescription(description) elif request.method == "handler.setRemoteDescription": data = request.data if isinstance(data, RTCSessionDescription): raise TypeError("request data not a RTCSessionDescription") description = RTCSessionDescription(**data) await self._pc.setRemoteDescription(description) elif request.method == "handler.getMid": data = request.data localId = data.get("localId") if localId is None: raise TypeError("missing data.localId") # raise on purpose if the key is not found transceiver = self._sendTransceivers[localId] return transceiver.mid elif request.method == "handler.addTrack": data = request.data localId = data.get("localId") if localId is None: raise TypeError("missing data.localId") kind = data["kind"] playerId = data.get("playerId") recvTrackId = data.get("recvTrackId") # sending a track got from a MediaPlayer if playerId: track = self._getTrack(playerId, kind) transceiver = self._pc.addTransceiver(track) # sending a track which is a remote/receiving track elif recvTrackId: track = self._getRemoteTrack(recvTrackId, kind) transceiver = self._pc.addTransceiver(track) else: raise TypeError("missing data.playerId or data.recvTrackId") # store transceiver in the dictionary self._sendTransceivers[localId] = transceiver elif request.method == "handler.removeTrack": data = request.data localId = data.get("localId") if localId is None: raise TypeError("missing data.localId") transceiver = self._sendTransceivers[localId] transceiver.direction = "inactive" transceiver.sender.replaceTrack(None) # NOTE: do not remove transceiver from the dictionary elif request.method == "handler.replaceTrack": data = request.data localId = data.get("localId") if localId is None: raise TypeError("missing data.localId") kind = data["kind"] playerId = data.get("playerId") recvTrackId = data.get("recvTrackId") transceiver = self._sendTransceivers[localId] # sending a track got from a MediaPlayer if playerId: track = self._getTrack(playerId, kind) # sending a track which is a remote/receiving track elif recvTrackId: track = self._getRemoteTrack(recvTrackId, kind) else: raise TypeError("missing data.playerId or data.recvTrackId") transceiver.sender.replaceTrack(track) elif request.method == "handler.getTransportStats": result = {} stats = await self._pc.getStats() for key in stats: type = stats[key].type if type == "inbound-rtp": result[key] = self._serializeInboundStats(stats[key]) elif type == "outbound-rtp": result[key] = self._serializeOutboundStats(stats[key]) elif type == "remote-inbound-rtp": result[key] = self._serializeRemoteInboundStats(stats[key]) elif type == "remote-outbound-rtp": result[key] = self._serializeRemoteOutboundStats( stats[key]) elif type == "transport": result[key] = self._serializeTransportStats(stats[key]) return result elif request.method == "handler.getSenderStats": data = request.data mid = data.get("mid") if mid is None: raise TypeError("missing data.mid") transceiver = self._getTransceiverByMid(mid) sender = transceiver.sender result = {} stats = await sender.getStats() for key in stats: type = stats[key].type if type == "outbound-rtp": result[key] = self._serializeOutboundStats(stats[key]) elif type == "remote-inbound-rtp": result[key] = self._serializeRemoteInboundStats(stats[key]) elif type == "transport": result[key] = self._serializeTransportStats(stats[key]) return result elif request.method == "handler.getReceiverStats": data = request.data mid = data.get("mid") if mid is None: raise TypeError("missing data.mid") transceiver = self._getTransceiverByMid(mid) receiver = transceiver.receiver result = {} stats = await receiver.getStats() for key in stats: type = stats[key].type if type == "inbound-rtp": result[key] = self._serializeInboundStats(stats[key]) elif type == "remote-outbound-rtp": result[key] = self._serializeRemoteOutboundStats( stats[key]) elif type == "transport": result[key] = self._serializeTransportStats(stats[key]) return result elif request.method == "handler.createDataChannel": internal = request.internal dataChannelId = internal.get("dataChannelId") data = request.data id = data.get("id") ordered = data.get("ordered") maxPacketLifeTime = data.get("maxPacketLifeTime") maxRetransmits = data.get("maxRetransmits") label = data.get("label") protocol = data.get("protocol") dataChannel = self._pc.createDataChannel( negotiated=True, id=id, ordered=ordered, maxPacketLifeTime=maxPacketLifeTime, maxRetransmits=maxRetransmits, label=label, protocol=protocol) # store datachannel in the dictionary self._dataChannels[dataChannelId] = dataChannel @dataChannel.on("open") # type: ignore async def on_open() -> None: await self._channel.notify(dataChannelId, "open") @dataChannel.on("closing") # type: ignore async def on_closing() -> None: await self._channel.notify(dataChannelId, "closing") @dataChannel.on("close") # type: ignore async def on_close() -> None: # NOTE: After calling dataChannel.close() aiortc emits "close" event # on the dataChannel. Probably it shouldn't do it. So caution. try: del self._dataChannels[dataChannelId] await self._channel.notify(dataChannelId, "close") except KeyError: pass @dataChannel.on("message") # type: ignore async def on_message(message) -> None: if isinstance(message, str): await self._channel.notify(dataChannelId, "message", message) if isinstance(message, bytes): message_bytes = base64.b64encode(message) await self._channel.notify(dataChannelId, "binary", str(message_bytes)) @dataChannel.on("bufferedamountlow") # type: ignore async def on_bufferedamountlow() -> None: await self._channel.notify(dataChannelId, "bufferedamountlow") return { "streamId": dataChannel.id, "ordered": dataChannel.ordered, "maxPacketLifeTime": dataChannel.maxPacketLifeTime, "maxRetransmits": dataChannel.maxRetransmits, "label": dataChannel.label, "protocol": dataChannel.protocol, # status fields "readyState": dataChannel.readyState, "bufferedAmount": dataChannel.bufferedAmount, "bufferedAmountLowThreshold": dataChannel.bufferedAmountLowThreshold } else: raise TypeError("unknown request method") async def processNotification(self, notification: Notification) -> None: if notification.event == "enableTrack": Logger.warning("handler: enabling track not implemented") elif notification.event == "disableTrack": Logger.warning("handler: disabling track not implemented") elif notification.event == "datachannel.send": internal = notification.internal dataChannelId = internal.get("dataChannelId") if dataChannelId is None: raise TypeError("missing internal.dataChannelId") data = notification.data dataChannel = self._dataChannels[dataChannelId] dataChannel.send(data) # Good moment to update bufferedAmount in Node.js side await self._channel.notify(dataChannelId, "bufferedamount", dataChannel.bufferedAmount) elif notification.event == "datachannel.sendBinary": internal = notification.internal dataChannelId = internal.get("dataChannelId") if dataChannelId is None: raise TypeError("missing internal.dataChannelId") data = notification.data dataChannel = self._dataChannels[dataChannelId] dataChannel.send(base64.b64decode(data)) # Good moment to update bufferedAmount in Node.js side await self._channel.notify(dataChannelId, "bufferedamount", dataChannel.bufferedAmount) elif notification.event == "datachannel.close": internal = notification.internal dataChannelId = internal.get("dataChannelId") if dataChannelId is None: raise TypeError("missing internal.dataChannelId") dataChannel = self._dataChannels.get(dataChannelId) if dataChannel is None: return # NOTE: After calling dataChannel.close() aiortc emits "close" event # on the dataChannel. Probably it shouldn't do it. So caution. try: del self._dataChannels[dataChannelId] except KeyError: pass dataChannel.close() elif notification.event == "datachannel.setBufferedAmountLowThreshold": internal = notification.internal dataChannelId = internal.get("dataChannelId") if dataChannelId is None: raise TypeError("missing internal.dataChannelId") value = notification.data dataChannel = self._dataChannels[dataChannelId] dataChannel.bufferedAmountLowThreshold = value else: raise TypeError("unknown notification event") """ Helper functions """ def _getTransceiverByMid(self, mid: str) -> Optional[RTCRtpTransceiver]: return next(filter(lambda x: x.mid == mid, self._pc.getTransceivers()), None) def _serializeInboundStats(self, stats: RTCStatsReport) -> Dict[str, Any]: return { # RTCStats "timestamp": stats.timestamp.timestamp(), "type": stats.type, "id": stats.id, # RTCStreamStats "ssrc": stats.ssrc, "kind": stats.kind, "transportId": stats.transportId, # RTCReceivedRtpStreamStats "packetsReceived": stats.packetsReceived, "packetsLost": stats.packetsLost, "jitter": stats.jitter } def _serializeOutboundStats(self, stats: RTCStatsReport) -> Dict[str, Any]: return { # RTCStats "timestamp": stats.timestamp.timestamp(), "type": stats.type, "id": stats.id, # RTCStreamStats "ssrc": stats.ssrc, "kind": stats.kind, "transportId": stats.transportId, # RTCSentRtpStreamStats "packetsSent": stats.packetsSent, "bytesSent": stats.bytesSent, # RTCOutboundRtpStreamStats "trackId": stats.trackId } def _serializeRemoteInboundStats(self, stats: RTCStatsReport) -> Dict[str, Any]: return { # RTCStats "timestamp": stats.timestamp.timestamp(), "type": stats.type, "id": stats.id, # RTCStreamStats "ssrc": stats.ssrc, "kind": stats.kind, "transportId": stats.transportId, # RTCReceivedRtpStreamStats "packetsReceived": stats.packetsReceived, "packetsLost": stats.packetsLost, "jitter": stats.jitter, # RTCRemoteInboundRtpStreamStats "roundTripTime": stats.roundTripTime, "fractionLost": stats.fractionLost } def _serializeRemoteOutboundStats(self, stats: RTCStatsReport) -> Dict[str, Any]: return { # RTCStats "timestamp": stats.timestamp.timestamp(), "type": stats.type, "id": stats.id, # RTCStreamStats "ssrc": stats.ssrc, "kind": stats.kind, "transportId": stats.transportId, # RTCSentRtpStreamStats "packetsSent": stats.packetsSent, "bytesSent": stats.bytesSent, # RTCRemoteOutboundRtpStreamStats "remoteTimestamp": stats.remoteTimestamp.timestamp() } def _serializeTransportStats(self, stats: RTCStatsReport) -> Dict[str, Any]: return { # RTCStats "timestamp": stats.timestamp.timestamp(), "type": stats.type, "id": stats.id, # RTCTransportStats "packetsSent": stats.packetsSent, "packetsReceived": stats.packetsReceived, "bytesSent": stats.bytesSent, "bytesReceived": stats.bytesReceived, "iceRole": stats.iceRole, "dtlsState": stats.dtlsState }
async def offer(request): params = await request.json() offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) #logger.info(offer) pc = RTCPeerConnection() pc_id = "PeerConnection(%s)" % uuid.uuid4() pcs.add(pc) def log_info(msg, *args): logger.info(pc_id + " " + msg, *args) log_info("Created for %s", request.remote) @pc.on("datachannel") def on_datachannel(channel): @channel.on("message") def on_message(message): if isinstance(message, str) and message.startswith("ping"): channel.send("pong" + message[4:]) @pc.on("connectionstatechange") async def on_connectionstatechange(): log_info("Connection state is %s", pc.connectionState) if pc.connectionState == "failed": await pc.close() pcs.discard(pc) # Otetaan video ja audio vastaan striimin välittävältä palvelimelta # Tässä toistetaan video / audio -tiedosto, mitä tarkoittaa jatkuvan # streamin tapauksessa? @pc.on("track") def on_track(track): log_info("Track %s received", track.kind) if track.kind == "audio": pc.addTrack(player.audio) recorder.addTrack(track) elif track.kind == "video": create_broadcast(track) pc.addTrack(track) #pc.addTrack(track)#relay.subscribe(broadcast)) @track.on("ended") async def on_ended(): log_info("Track %s ended", track.kind) #await pc.close() #pcs.discard(pc) broadcast_ended() coros = [pc.close() for pc in pcs] await asyncio.gather(*coros) pcs.clear() # handle offer await pc.setRemoteDescription(offer) # Tämä ajetaan kun clientistä on "Listen for..." valittuna # addTrack menee clienttiin ja siihen laitetaan relay broadcastista if params["listen_video"]: log_info("Kuuntelu") for t in pc.getTransceivers(): log_info("Kuuntelu %s", t.kind) # Tarkasta onko "broadcast" olemassa if t.kind == "video" and broadcast: pc.addTrack(relay.subscribe(broadcast)) capabilities = RTCRtpSender.getCapabilities('video') preferences = list( filter(lambda x: x.name == 'H264', capabilities.codecs)) print(preferences) transc = pc.getTransceivers()[0] transc.setCodecPreferences(preferences) # send answer answer = await pc.createAnswer() await pc.setLocalDescription(answer) print(json.dumps({"type": pc.localDescription.type})) return web.Response( content_type="application/json", text=json.dumps({ "sdp": pc.localDescription.sdp, "type": pc.localDescription.type }), )
async def _process_offer( mode: WebRtcMode, pc: RTCPeerConnection, offer: RTCSessionDescription, player_factory: Optional[MediaPlayerFactory], in_recorder_factory: Optional[MediaRecorderFactory], out_recorder_factory: Optional[MediaRecorderFactory], video_transformer: Optional[VideoTransformerBase], video_receiver: Optional[VideoReceiver], async_transform: bool, callback: Callable[[Union[RTCSessionDescription, Exception]], None], ): try: player = None if player_factory: player = player_factory() in_recorder = None if in_recorder_factory: in_recorder = in_recorder_factory() out_recorder = None if out_recorder_factory: out_recorder = out_recorder_factory() @pc.on("iceconnectionstatechange") async def on_iceconnectionstatechange(): logger.info("ICE connection state is %s", pc.iceConnectionState) if pc.iceConnectionState == "failed": await pc.close() if mode == WebRtcMode.SENDRECV: @pc.on("track") def on_track(input_track): logger.info("Track %s received", input_track.kind) output_track = None if input_track.kind == "audio": if player and player.audio: logger.info("Add player to audio track") output_track = player.audio else: # Transforming audio is not supported yet. output_track = input_track # passthrough elif input_track.kind == "video": if player and player.video: logger.info("Add player to video track") output_track = player.video elif video_transformer: VideoTrack = ( AsyncVideoTransformTrack if async_transform else VideoTransformTrack ) logger.info( "Add a input video track %s to " "output track with video_transformer %s", input_track, VideoTrack, ) local_video = VideoTrack( track=input_track, video_transformer=video_transformer ) logger.info("Add the video track with transfomer to %s", pc) output_track = local_video else: output_track = input_track if not output_track: raise Exception( "Neither a player nor a transformer is created. " "Either factory must be set." ) pc.addTrack(output_track) if out_recorder: logger.info("Track %s is added to out_recorder", output_track.kind) out_recorder.addTrack(output_track) if in_recorder: logger.info("Track %s is added to in_recorder", input_track.kind) in_recorder.addTrack(input_track) @input_track.on("ended") async def on_ended(): logger.info("Track %s ended", input_track.kind) if in_recorder: await in_recorder.stop() if out_recorder: await out_recorder.stop() elif mode == WebRtcMode.SENDONLY: @pc.on("track") def on_track(input_track): logger.info("Track %s received", input_track.kind) if input_track.kind == "audio": # Not supported yet pass elif input_track.kind == "video": if video_receiver: logger.info( "Add a track %s to receiver %s", input_track, video_receiver ) video_receiver.addTrack(input_track) if in_recorder: logger.info("Track %s is added to in_recorder", input_track.kind) in_recorder.addTrack(input_track) @input_track.on("ended") async def on_ended(): logger.info("Track %s ended", input_track.kind) if video_receiver: video_receiver.stop() if in_recorder: await in_recorder.stop() await pc.setRemoteDescription(offer) if mode == WebRtcMode.RECVONLY: for t in pc.getTransceivers(): output_track = None if t.kind == "audio": if player and player.audio: output_track = player.audio # pc.addTrack(player.audio) elif t.kind == "video": if player and player.video: # pc.addTrack(player.video) output_track = player.video if output_track: pc.addTrack(output_track) # NOTE: Recording is not supported in this mode # because connecting player to recorder does not work somehow; # it generates unplayable movie files. if video_receiver and video_receiver.hasTrack(): video_receiver.start() if in_recorder: await in_recorder.start() if out_recorder: await out_recorder.start() answer = await pc.createAnswer() await pc.setLocalDescription(answer) callback(pc.localDescription) except Exception as e: logger.debug("Error occurred in process_offer") logger.debug(e) callback(e)
async def offer(request): params = await request.json() print("got offer request: \n" + params["sdp"]) print("offer type: " + params["type"]) offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) pc = RTCPeerConnection() pc_id = "PeerConnection(%s)" % uuid.uuid4() pcs.add(pc) print("Created for %s", request.remote) @pc.on("datachannel") def on_datachannel(channel): global io_channel io_channel = channel @channel.on("message") def on_message(message): if isinstance(message, str): if message.startswith("ch"): new_channels = list( map(lambda x: int(x), message.split(" ", 1)[1].split())) new_channels = { "1": new_channels[0], "2": new_channels[1], "3": new_channels[2], "4": new_channels[3], } global channels channels = new_channels @pc.on("iceconnectionstatechange") async def on_iceconnectionstatechange(): print("ICE connection state is %s", pc.iceConnectionState) if pc.iceConnectionState == "failed": await pc.close() pcs.discard(pc) elif pc.iceConnectionState == "closed": subprocess.Popen( "ps x | awk {'{print $1}'} | awk 'NR > 1' | xargs kill", shell=True) # handle offer await pc.setRemoteDescription(offer) for t in pc.getTransceivers(): if t.kind == "video": pc.addTrack(VideoTrack()) print("added track") # send answer answer = await pc.createAnswer() print("answer: " + str(answer)) await pc.setLocalDescription(answer) print("sdp" + pc.localDescription.sdp) return web.Response( content_type="application/json", text=json.dumps({ "sdp": pc.localDescription.sdp, "type": pc.localDescription.type }), )
async def _process_offer( mode: WebRtcMode, pc: RTCPeerConnection, offer: RTCSessionDescription, player_factory: Optional[MediaPlayerFactory], video_transformer: Optional[VideoTransformerBase], video_receiver: Optional[VideoReceiver], async_transform: bool, callback: Callable[[Union[RTCSessionDescription, Exception]], None], ): try: player = None if player_factory: player = player_factory() @pc.on("iceconnectionstatechange") async def on_iceconnectionstatechange(): logger.info("ICE connection state is %s", pc.iceConnectionState) if pc.iceConnectionState == "failed": await pc.close() if mode == WebRtcMode.SENDRECV: @pc.on("track") def on_track(track): logger.info("Track %s received", track.kind) if track.kind == "audio": if player and player.audio: pc.addTrack(player.audio) elif track.kind == "video": if player and player.video: logger.info("Add player to video track") pc.addTrack(player.video) elif video_transformer: VideoTrack = (AsyncVideoTransformTrack if async_transform else VideoTransformTrack) logger.info( "Add a input video track %s to " "another track with video_transformer %s", track, VideoTrack, ) local_video = VideoTrack( track=track, video_transformer=video_transformer) logger.info( "Add the video track with transfomer to %s", pc) pc.addTrack(local_video) elif mode == WebRtcMode.SENDONLY: @pc.on("track") def on_track(track): logger.info("Track %s received", track.kind) if track.kind == "audio": # Not supported yet pass elif track.kind == "video": if video_receiver: logger.info("Add a track %s to receiver %s", track, video_receiver) video_receiver.addTrack(track) @track.on("ended") async def on_ended(): logger.info("Track %s ended", track.kind) if video_receiver: video_receiver.stop() await pc.setRemoteDescription(offer) if mode == WebRtcMode.RECVONLY: for t in pc.getTransceivers(): if t.kind == "audio": if player and player.audio: pc.addTrack(player.audio) elif t.kind == "video": if player and player.video: pc.addTrack(player.video) if video_receiver and video_receiver.hasTrack(): video_receiver.start() answer = await pc.createAnswer() await pc.setLocalDescription(answer) callback(pc.localDescription) except Exception as e: logger.debug("Error occurred in process_offer") logger.debug(e) callback(e)
async def offer(request): if len(pcs) < MAX_CONNECTION_NUM: params = await request.json() offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) idNum = params["idNum"] # print("idNum(offer) : %d" % idNum) # print("offer func") # print("path : %s" % request.rel_url.path) # print(request.rel_url.path == "/offerFilter") camFilter = (request.rel_url.path == "/offerFilter") indexNum = getIndexFromId(idNum) if indexNum >= 0: if userList[indexNum][0] == request.remote: pc = RTCPeerConnection() pcs.add(pc) userList[indexNum][2] = pc userList[indexNum][4] = camFilter # print(pc) # print(vars(request)) @pc.on("iceconnectionstatechange") async def on_connectionstatechange(): print("Connection state is %s" % pc.connectionState) if pc.connectionState == "failed": deleteAnObj(pc) await pc.close() pcs.discard(pc) owner = False if userList[indexNum][1] == 0: owner = True # open media source audio, video = create_local_tracks(args.play_from, owner = owner, camFilter = camFilter) await pc.setRemoteDescription(offer) for t in pc.getTransceivers(): if t.kind == "audio" and audio: pc.addTrack(audio) elif t.kind == "video" and video: pc.addTrack(video) answer = await pc.createAnswer() await pc.setLocalDescription(answer) return web.Response( content_type="application/json", text=json.dumps( {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type} ), ) else: print("request remote : %s" % request.remote) return web.Response(content_type="text/html", text="busy") else: print("indexNum error : %d" % indexNum) return web.Response(content_type="text/html", text="busy") else: return web.Response(content_type="text/html", text="busy")
async def offer(request): #cap rtc peers at <X> if len(pcs) > pcs_max_size: return params = await request.json() offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) uuid = str(params["uuid"]) pc = next((x for x in pcs if x.uuid == uuid), None) if pc is None: pc = RTCPeerConnection() # configuration=RTCConfiguration( # iceServers=[ # RTCIceServer(urls=["stun:stun.l.google:19302"]), # RTCIceServer(urls=["turn:0.peerjs.com:3478"], username="******", credential="peerjsp"), # RTCIceServer(urls=["turn:turn.bistri.com:80"], username="******", credential="homeo"), # RTCIceServer(urls=["turn:turn.anyfirewall.com:443"], username="******", credential="webrtc") # ])) setattr(pc, 'uuid', uuid) pcs.append(pc) @pc.on("datachannel") def on_datachannel(channel): print("Channel id: {}".format(channel.id)) #print("queue position: {}".format(pcs.index(pc))) #This transport monitor is necessary, otherwise #a closed connection breaks the server on the next connection async def monitor(): while True: if channel.transport.transport.state == "closed": await pc.close() js = getattr(pc, 'js', None) if js is not None: jsdev = next((x for x in jslist if x.dev == js), None) if jsdev is not None: print("Freeing joystick:{}".format(jsdev.idx)) jslist[jslist.index(jsdev)] = jsdev._replace( locked=False) try: pcs.remove(pc) except ValueError: pass break await asyncio.sleep(5) asyncio.ensure_future(monitor()) @channel.on("close") def on_close(): print("Close Channel id: {}".format(channel.id)) js = getattr(pc, 'js', None) if js is not None: jsdev = next((x for x in jslist if x.dev == js), None) if jsdev is not None: print("Freeing joystick:{}".format(jsdev.idx)) jslist[jslist.index(jsdev)] = jsdev._replace( locked=False) setattr(pc, 'js', None) @channel.on("message") async def on_message(message): #we use the 1 a second ping as heartbeat (and query) if isinstance(message, str) and message.startswith("ping"): try: #if at the top of the queue and not assigned if pcs.index(pc) < len(jslist): if not hasattr(pc, 'js'): # if they don't have a js, and one is available jsdev = next( (x for x in jslist if x.locked == False), None) if jsdev is not None: setattr(pc, 'js', jsdev.dev) print("Assigning joystick:{}".format( jsdev.idx)) jslist[jslist.index( jsdev)] = jsdev._replace(locked=True) #trigger to start a direct webrtc video feed (low latency) channel.send("start: Player {}".format( jsdev.idx)) else: # if they do have a joystick, and people are waiting if (len(pcs) > len(jslist)): #set future timestamp if not set if not hasattr(pc, 'ts'): #lets do 60 seconds plus a bit of hidden stopped time setattr(pc, 'ts', int(time()) + 60 + 5) remaining_time = pc.ts - int(time()) if remaining_time < 0: channel.send("stop") await asyncio.sleep(0.1) channel.close() for t in pc.getTransceivers(): await t.stop() await pc.close() js = getattr(pc, 'js', None) if js is not None: jsdev = next((x for x in jslist if x.dev == js), None) if jsdev is not None: print("Freeing joystick:{}". format(jsdev.idx)) jslist[jslist.index( jsdev)] = jsdev._replace( locked=False) try: pcs.remove(pc) except ValueError: pass else: # 5 second hidden game over time channel.send("Time: {}".format( max(0, remaining_time - 5))) else: channel.send("Time: --") else: #send their position in the waitlist channel.send("Waitlist: {} of {}".format( pcs.index(pc) + 1 - len(jslist), len(pcs) - len(jslist))) except ConnectionError: pass elif isinstance(message, str) and message.startswith("controller: "): #if in the last "hidden" 5 seconds of a turn, ignore buttons if hasattr(pc, 'ts') and (pc.ts - int(time()) < 5): return vals = message[12:].split(',') if getattr(pc, 'js', None) is not None: jsupdate_vals(pc.js, vals) elif isinstance(message, str) and message.startswith("key"): #if in the last "hidden" 5 seconds of a turn, ignore buttons if hasattr(pc, 'ts') and (pc.ts - int(time()) < 5): return direction = message[3] key = message[7] dval = 1 if direction == "d" else 0 keys = ['z', 'x', 's', 'u', 'l', 'd', 'r'] kpos = keys.index(key) if getattr(pc, 'js', None) is not None: if (kpos != 2): pc.js.emit(events[kpos], int(dval)) #pico8 hack to use start button to restart pico8 back to main menu elif (kpos == 2 and int(dval) == 1 and pico8proc is not None): pico8proc.stop() pico8proc.is_started = False pico8proc.start() pc.js.flush() elif isinstance(message, str) and message.startswith("gamectl: switch"): #special game control just for pico-8 (for now) gameidx = (gameidx + 1) % len(gamelist) nextgame = gamelist[gameidx] try: os.symlink('pico8/data_{}.pod'.format(nextgame), 'pico8/data_tmp.pod') os.replace('pico8/data_tmp.pod', 'pico8/data.pod') except FileExistsError: pass #sendkeys CTRL+R 'reload' (super ugly CTRL down, then R, then up, up) #even uglier, need to send a couple of times to ensure it gets caught for _ in range(6): self.send_sdl_event(0x400000e0, 0xe0, 1, kmod=kmod) self.send_sdl_event(ord('r'), 0x15, 1, kmod=kmod) self.send_sdl_event(0x400000e0, 0xe0, 0, kmod=kmod) self.send_sdl_event(ord('r'), 0x15, 0, kmod=kmod) @pc.on("iceconnectionstatechange") async def on_iceconnectionstatechange(): print("ICE connection state is %s" % pc.iceConnectionState) if pc.iceConnectionState == "failed": await pc.close() js = getattr(pc, 'js', None) if js is not None: jsdev = next((x for x in jslist if x.dev == js), None) if jsdev is not None: print("Freeing joystick:{}".format(jsdev.idx)) jslist[jslist.index(jsdev)] = jsdev._replace( locked=False) try: pcs.remove(pc) except ValueError: pass # handle offer await pc.setRemoteDescription(offer) for t in pc.getTransceivers(): if t.kind == "video": pc.addTrack(VideoImageTrack()) # send answer answer = await pc.createAnswer() await pc.setLocalDescription(answer) return web.Response( content_type="application/json", text=json.dumps({ "sdp": pc.localDescription.sdp, "type": pc.localDescription.type }), )
async def _process_offer_coro( mode: WebRtcMode, pc: RTCPeerConnection, offer: RTCSessionDescription, relay: MediaRelay, source_video_track: Optional[MediaStreamTrack], source_audio_track: Optional[MediaStreamTrack], in_recorder: Optional[MediaRecorder], out_recorder: Optional[MediaRecorder], video_processor: Optional[VideoProcessorBase], audio_processor: Optional[AudioProcessorBase], video_receiver: Optional[VideoReceiver], audio_receiver: Optional[AudioReceiver], async_processing: bool, sendback_video: bool, sendback_audio: bool, on_track_created: Callable[[TrackType, MediaStreamTrack], None], ): if mode == WebRtcMode.SENDRECV: @pc.on("track") def on_track(input_track): logger.info("Track %s received", input_track.kind) if input_track.kind == "video": on_track_created("input:video", input_track) elif input_track.kind == "audio": on_track_created("input:audio", input_track) output_track = None if input_track.kind == "audio": if source_audio_track: logger.info("Set %s as an input audio track", source_audio_track) output_track = source_audio_track elif audio_processor: AudioTrack = (AsyncAudioProcessTrack if async_processing else AudioProcessTrack) logger.info( "Set %s as an input audio track with audio_processor %s", input_track, AudioTrack, ) output_track = AudioTrack( track=relay.subscribe(input_track), processor=audio_processor, ) else: output_track = input_track # passthrough elif input_track.kind == "video": if source_video_track: logger.info("Set %s as an input video track", source_video_track) output_track = source_video_track elif video_processor: VideoTrack = (AsyncVideoProcessTrack if async_processing else VideoProcessTrack) logger.info( "Set %s as an input video track with video_processor %s", input_track, VideoTrack, ) output_track = VideoTrack( track=relay.subscribe(input_track), processor=video_processor, ) else: output_track = input_track if (output_track.kind == "video" and sendback_video) or (output_track.kind == "audio" and sendback_audio): logger.info( "Add a track %s of kind %s to %s", output_track, output_track.kind, pc, ) pc.addTrack(relay.subscribe(output_track)) else: logger.info("Block a track %s of kind %s", output_track, output_track.kind) if out_recorder: logger.info("Track %s is added to out_recorder", output_track.kind) out_recorder.addTrack(relay.subscribe(output_track)) if in_recorder: logger.info("Track %s is added to in_recorder", input_track.kind) in_recorder.addTrack(relay.subscribe(input_track)) if output_track.kind == "video": on_track_created("output:video", output_track) elif output_track.kind == "audio": on_track_created("output:audio", output_track) @input_track.on("ended") async def on_ended(): logger.info("Track %s ended", input_track.kind) if in_recorder: await in_recorder.stop() if out_recorder: await out_recorder.stop() elif mode == WebRtcMode.SENDONLY: @pc.on("track") def on_track(input_track): logger.info("Track %s received", input_track.kind) if input_track.kind == "video": on_track_created("input:video", input_track) elif input_track.kind == "audio": on_track_created("input:audio", input_track) if input_track.kind == "audio": if audio_receiver: logger.info("Add a track %s to receiver %s", input_track, audio_receiver) audio_receiver.addTrack(input_track) elif input_track.kind == "video": if video_receiver: logger.info("Add a track %s to receiver %s", input_track, video_receiver) video_receiver.addTrack(input_track) if in_recorder: logger.info("Track %s is added to in_recorder", input_track.kind) in_recorder.addTrack(input_track) @input_track.on("ended") async def on_ended(): logger.info("Track %s ended", input_track.kind) if video_receiver: video_receiver.stop() if audio_receiver: audio_receiver.stop() if in_recorder: await in_recorder.stop() await pc.setRemoteDescription(offer) if mode == WebRtcMode.RECVONLY: for t in pc.getTransceivers(): output_track = None if t.kind == "audio": if source_audio_track: if audio_processor: AudioTrack = (AsyncAudioProcessTrack if async_processing else AudioProcessTrack) logger.info( "Set %s as an input audio track with audio_processor %s", source_audio_track, AudioTrack, ) output_track = AudioTrack(track=source_audio_track, processor=audio_processor) else: output_track = source_audio_track # passthrough elif t.kind == "video": if source_video_track: if video_processor: VideoTrack = (AsyncVideoProcessTrack if async_processing else VideoProcessTrack) logger.info( "Set %s as an input video track with video_processor %s", source_video_track, VideoTrack, ) output_track = VideoTrack(track=source_video_track, processor=video_processor) else: output_track = source_video_track # passthrough if output_track: logger.info("Add a track %s to %s", output_track, pc) pc.addTrack(relay.subscribe(output_track)) # NOTE: Recording is not supported in this mode # because connecting player to recorder does not work somehow; # it generates unplayable movie files. if output_track.kind == "video": on_track_created("output:video", output_track) elif output_track.kind == "audio": on_track_created("output:audio", output_track) if video_receiver and video_receiver.hasTrack(): video_receiver.start() if audio_receiver and audio_receiver.hasTrack(): audio_receiver.start() if in_recorder: await in_recorder.start() if out_recorder: await out_recorder.start() answer = await pc.createAnswer() await pc.setLocalDescription(answer) return pc.localDescription
def test_connect_datachannel(self): pc1 = RTCPeerConnection() pc1_data_messages = [] pc1_states = track_states(pc1) pc2 = RTCPeerConnection() pc2_data_channels = [] pc2_data_messages = [] pc2_states = track_states(pc2) @pc2.on('datachannel') def on_datachannel(channel): self.assertEqual(channel.readyState, 'open') pc2_data_channels.append(channel) @channel.on('message') def on_message(message): pc2_data_messages.append(message) if isinstance(message, str): channel.send('string-echo: ' + message) else: channel.send(b'binary-echo: ' + message) # create data channel dc = pc1.createDataChannel('chat', protocol='bob') self.assertEqual(dc.label, 'chat') self.assertEqual(dc.protocol, 'bob') self.assertEqual(dc.readyState, 'connecting') # send messages dc.send('hello') dc.send('') dc.send(b'\x00\x01\x02\x03') dc.send(b'') dc.send(LONG_DATA) with self.assertRaises(ValueError) as cm: dc.send(1234) self.assertEqual(str(cm.exception), "Cannot send unsupported data type: <class 'int'>") @dc.on('message') def on_message(message): pc1_data_messages.append(message) # create offer offer = run(pc1.createOffer()) self.assertEqual(offer.type, 'offer') self.assertTrue('m=application ' in offer.sdp) self.assertFalse('a=candidate:' in offer.sdp) run(pc1.setLocalDescription(offer)) self.assertEqual(pc1.iceConnectionState, 'new') self.assertEqual(pc1.iceGatheringState, 'complete') self.assertTrue('m=application ' in pc1.localDescription.sdp) self.assertTrue('a=candidate:' in pc1.localDescription.sdp) self.assertTrue('a=sctpmap:5000 webrtc-datachannel 65535' in pc1.localDescription.sdp) self.assertTrue('a=fingerprint:sha-256' in pc1.localDescription.sdp) self.assertTrue('a=setup:actpass' in pc1.localDescription.sdp) # handle offer run(pc2.setRemoteDescription(pc1.localDescription)) self.assertEqual(pc2.remoteDescription, pc1.localDescription) self.assertEqual(len(pc2.getReceivers()), 0) self.assertEqual(len(pc2.getSenders()), 0) self.assertEqual(len(pc2.getTransceivers()), 0) # create answer answer = run(pc2.createAnswer()) self.assertEqual(answer.type, 'answer') self.assertTrue('m=application ' in answer.sdp) self.assertFalse('a=candidate:' in answer.sdp) run(pc2.setLocalDescription(answer)) self.assertEqual(pc2.iceConnectionState, 'checking') self.assertEqual(pc2.iceGatheringState, 'complete') self.assertTrue('m=application ' in pc2.localDescription.sdp) self.assertTrue('a=candidate:' in pc2.localDescription.sdp) self.assertTrue('a=sctpmap:5000 webrtc-datachannel 65535' in pc2.localDescription.sdp) self.assertTrue('a=fingerprint:sha-256' in pc2.localDescription.sdp) self.assertTrue('a=setup:active' in pc2.localDescription.sdp) # handle answer run(pc1.setRemoteDescription(pc2.localDescription)) self.assertEqual(pc1.remoteDescription, pc2.localDescription) self.assertEqual(pc1.iceConnectionState, 'checking') # check outcome run(asyncio.sleep(1)) self.assertEqual(pc1.iceConnectionState, 'completed') self.assertEqual(pc2.iceConnectionState, 'completed') self.assertEqual(dc.readyState, 'open') # check pc2 got a datachannel self.assertEqual(len(pc2_data_channels), 1) self.assertEqual(pc2_data_channels[0].label, 'chat') self.assertEqual(pc2_data_channels[0].protocol, 'bob') # check pc2 got messages run(asyncio.sleep(1)) self.assertEqual(pc2_data_messages, [ 'hello', '', b'\x00\x01\x02\x03', b'', LONG_DATA, ]) # check pc1 got replies self.assertEqual(pc1_data_messages, [ 'string-echo: hello', 'string-echo: ', b'binary-echo: \x00\x01\x02\x03', b'binary-echo: ', b'binary-echo: ' + LONG_DATA, ]) # close data channel dc.close() self.assertEqual(dc.readyState, 'closed') # close run(pc1.close()) run(pc2.close()) self.assertEqual(pc1.iceConnectionState, 'closed') self.assertEqual(pc2.iceConnectionState, 'closed') # check state changes self.assertEqual(pc1_states['iceConnectionState'], ['new', 'checking', 'completed', 'closed']) self.assertEqual(pc1_states['iceGatheringState'], ['new', 'gathering', 'complete']) self.assertEqual(pc1_states['signalingState'], ['stable', 'have-local-offer', 'stable', 'closed']) self.assertEqual(pc2_states['iceConnectionState'], ['new', 'checking', 'completed', 'closed']) self.assertEqual(pc2_states['iceGatheringState'], ['new', 'gathering', 'complete']) self.assertEqual(pc2_states['signalingState'], ['stable', 'have-remote-offer', 'stable', 'closed'])
def test_connect_audio_mid_changes(self): pc1 = RTCPeerConnection() pc1_states = track_states(pc1) pc2 = RTCPeerConnection() pc2_states = track_states(pc2) self.assertEqual(pc1.iceConnectionState, 'new') self.assertEqual(pc1.iceGatheringState, 'new') self.assertIsNone(pc1.localDescription) self.assertIsNone(pc1.remoteDescription) self.assertEqual(pc2.iceConnectionState, 'new') self.assertEqual(pc2.iceGatheringState, 'new') self.assertIsNone(pc2.localDescription) self.assertIsNone(pc2.remoteDescription) # add audio tracks immediately pc1.addTrack(AudioStreamTrack()) pc1.getTransceivers()[0].mid = 'sdparta_0' # pretend we're Firefox! self.assertEqual(mids(pc1), ['sdparta_0']) pc2.addTrack(AudioStreamTrack()) self.assertEqual(mids(pc2), ['audio']) # create offer offer = run(pc1.createOffer()) self.assertEqual(offer.type, 'offer') self.assertTrue('m=audio ' in offer.sdp) self.assertFalse('a=candidate:' in offer.sdp) run(pc1.setLocalDescription(offer)) self.assertEqual(pc1.iceConnectionState, 'new') self.assertEqual(pc1.iceGatheringState, 'complete') self.assertTrue('m=audio ' in pc1.localDescription.sdp) self.assertTrue('a=candidate:' in pc1.localDescription.sdp) self.assertTrue('a=sendrecv' in pc1.localDescription.sdp) self.assertTrue('a=fingerprint:sha-256' in pc1.localDescription.sdp) self.assertTrue('a=setup:actpass' in pc1.localDescription.sdp) self.assertTrue('a=mid:sdparta_0' in pc1.localDescription.sdp) # handle offer run(pc2.setRemoteDescription(pc1.localDescription)) self.assertEqual(pc2.remoteDescription, pc1.localDescription) self.assertEqual(len(pc2.getReceivers()), 1) self.assertEqual(len(pc2.getSenders()), 1) self.assertEqual(mids(pc2), ['sdparta_0']) # create answer answer = run(pc2.createAnswer()) self.assertEqual(answer.type, 'answer') self.assertTrue('m=audio ' in answer.sdp) self.assertFalse('a=candidate:' in answer.sdp) run(pc2.setLocalDescription(answer)) self.assertEqual(pc2.iceConnectionState, 'checking') self.assertEqual(pc2.iceGatheringState, 'complete') self.assertTrue('m=audio ' in pc2.localDescription.sdp) self.assertTrue('a=candidate:' in pc2.localDescription.sdp) self.assertTrue('a=sendrecv' in pc1.localDescription.sdp) self.assertTrue('a=fingerprint:sha-256' in pc2.localDescription.sdp) self.assertTrue('a=setup:active' in pc2.localDescription.sdp) self.assertTrue('a=mid:sdparta_0' in pc2.localDescription.sdp) # handle answer run(pc1.setRemoteDescription(pc2.localDescription)) self.assertEqual(pc1.remoteDescription, pc2.localDescription) self.assertEqual(pc1.iceConnectionState, 'checking') # check outcome run(asyncio.sleep(1)) self.assertEqual(pc1.iceConnectionState, 'completed') self.assertEqual(pc2.iceConnectionState, 'completed') # close run(pc1.close()) run(pc2.close()) self.assertEqual(pc1.iceConnectionState, 'closed') self.assertEqual(pc2.iceConnectionState, 'closed') # check state changes self.assertEqual(pc1_states['iceConnectionState'], ['new', 'checking', 'completed', 'closed']) self.assertEqual(pc1_states['iceGatheringState'], ['new', 'gathering', 'complete']) self.assertEqual(pc1_states['signalingState'], ['stable', 'have-local-offer', 'stable', 'closed']) self.assertEqual(pc2_states['iceConnectionState'], ['new', 'checking', 'completed', 'closed']) self.assertEqual(pc2_states['iceGatheringState'], ['new', 'gathering', 'complete']) self.assertEqual(pc2_states['signalingState'], ['stable', 'have-remote-offer', 'stable', 'closed'])
class RTCConnection(SubscriptionProducerConsumer): _log = logging.getLogger("rtcbot.RTCConnection") def __init__( self, defaultChannelOrdered=True, loop=None, rtcConfiguration=RTCConfiguration( [RTCIceServer(urls="stun:stun.l.google.com:19302")]), video_codec_preference=None, audio_codec_preference=None, ): super().__init__( directPutSubscriptionType=asyncio.Queue, defaultSubscriptionType=asyncio.Queue, logger=self._log, ) self._loop = loop if self._loop is None: self._loop = asyncio.get_event_loop() self._dataChannels = {} # These allow us to easily signal when the given events happen self._dataChannelSubscriber = SubscriptionProducer( logger=self._log.getChild("dataChannelSubscriber")) self._rtc = RTCPeerConnection(configuration=rtcConfiguration) self._rtc.on("datachannel", self._onDatachannel) self._rtc.on("iceconnectionstatechange", self._onIceConnectionStateChange) self._rtc.on("track", self._onTrack) self._hasRemoteDescription = False self._defaultChannelOrdered = defaultChannelOrdered self._videoHandler = ConnectionVideoHandler(self._rtc) self._video_codec_preference = video_codec_preference self._audioHandler = ConnectionAudioHandler(self._rtc) self._audio_codec_preference = audio_codec_preference # When the video/audio handlers get closed, close the entire connection self._videoHandler.onClose(self.close) self._audioHandler.onClose(self.close) async def getLocalDescription(self, description=None): """ Gets the description to send on. Creates an initial description if no remote description was passed, and creates a response if a remote was given, """ for t in self._rtc.getTransceivers(): if t.kind == "video" and self._video_codec_preference is not None: t.setCodecPreferences(self._video_codec_preference) elif t.kind == "audio" and self._audio_codec_preference is not None: t.setCodecPreferences(self._audio_codec_preference) if self._hasRemoteDescription or description is not None: # This means that we received an offer - either the remote description # was already set, or we passed in a description. In either case, # instead of initializing a new connection, we prepare a response if not self._hasRemoteDescription: await self.setRemoteDescription(description) self._log.debug("Creating response to connection offer") try: answer = await self._rtc.createAnswer() except AttributeError: self._log.exception( "\n>>> Looks like the offer didn't include the necessary info to set up audio/video. See RTCConnection.video.offerToReceive(). <<<\n\n" ) raise await self._rtc.setLocalDescription(answer) return { "sdp": self._rtc.localDescription.sdp, "type": self._rtc.localDescription.type, } # There was no remote description, which means that we are initializing the # connection. # Before starting init, we create a default data channel for the connection self._log.debug("Setting up default data channel") channel = DataChannel( self._rtc.createDataChannel("default", ordered=self._defaultChannelOrdered)) # Subscribe the default channel directly to our own inputs and outputs. # We have it listen to our own self._get, and write to our self._put_nowait channel.putSubscription(NoClosedSubscription(self._get)) channel.subscribe(self._put_nowait) channel.onReady(lambda: self._setReady(channel.ready)) channel.onClose(self.close) self._dataChannels[channel.name] = channel # Make sure we offer to receive video and audio if if isn't set up yet with # all the receiving transceivers if len(self.video._senders) < self.video._offerToReceive: self._log.debug("Offering to receive video") for i in range(self.video._offerToReceive - len(self.video._senders)): self._rtc.addTransceiver("video", "recvonly") if len(self.audio._senders) < self.audio._offerToReceive: self._log.debug("Offering to receive audio") for i in range(self.audio._offerToReceive - len(self.audio._senders)): self._rtc.addTransceiver("audio", "recvonly") self._log.debug("Creating new connection offer") offer = await self._rtc.createOffer() await self._rtc.setLocalDescription(offer) return { "sdp": self._rtc.localDescription.sdp, "type": self._rtc.localDescription.type, } async def setRemoteDescription(self, description): self._log.debug("Setting remote connection description") await self._rtc.setRemoteDescription( RTCSessionDescription(**description)) self._hasRemoteDescription = True async def _onIceConnectionStateChange(self): self._log.debug("ICE state: %s", self._rtc.iceConnectionState) if self._rtc.iceConnectionState == "failed": self._setError(self._rtc.iceConnectionState) await self.close() def _onDatachannel(self, channel): """ When a data channel comes in, adds it to the data channels, and sets up its messaging and stuff. """ channel = DataChannel(channel) self._log.debug("Got channel: %s", channel.name) if channel.name == "default": # Subscribe the default channel directly to our own inputs and outputs. # We have it listen to our own self._get, and write to our self._put_nowait channel.putSubscription(NoClosedSubscription(self._get)) channel.subscribe(self._put_nowait) channel.onReady(lambda: self._setReady(channel.ready)) channel.onClose(self.close) else: self._dataChannelSubscriber.put_nowait(channel) self._dataChannels[channel.name] = channel def _onTrack(self, track): self._log.debug("Received %s track from connection", track.kind) if track.kind == "audio": self._audioHandler._onTrack(track) elif track.kind == "video": self._videoHandler._onTrack(track) def onDataChannel(self, callback=None): """ Acts as a subscriber... """ return self._dataChannelSubscriber.subscribe(callback) def addDataChannel(self, name, ordered=True): """ Adds a data channel to the connection. Note that the RTCConnection adds a "default" channel automatically, which you can subscribe to directly. """ self._log.debug("Adding data channel to connection") if name in self._dataChannels or name == "default": raise KeyError("Data channel %s already exists", name) dc = DataChannel(self._rtc.createDataChannel(name, ordered=ordered)) self._dataChannels[name] = dc return dc def getDataChannel(self, name): """ Returns the data channel with the given name. Please note that the "default" channel is considered special, and is not returned. """ if name == "default": raise KeyError( "Default channel not available for 'get'. Use the RTCConnection's subscribe and put_nowait methods for access to it." ) return self._dataChannels[name] @property def video(self): """ Convenience function - you can subscribe to it to get video frames once they show up """ return self._videoHandler @property def audio(self): """ Convenience function - you can subscribe to it to get audio once a stream is received """ return self._audioHandler def close(self): """ If the loop is running, returns a future that will close the connection. Otherwise, runs the loop temporarily to complete closing. """ if self.closed: if self._loop.is_running(): async def donothing(): pass return asyncio.ensure_future(donothing()) return None self._log.debug("Closing connection") super().close() # And closes all tracks self.video.close() self.audio.close() for dc in self._dataChannels: self._dataChannels[dc].close() self._dataChannelSubscriber.close() if self._loop.is_running(): self._log.debug("Loop is running - close will return a future!") return asyncio.ensure_future(self._rtc.close()) else: self._loop.run_until_complete(self._rtc.close()) return None def send(self, msg): """ Send is an alias for put_nowait - makes it easier for people new to rtcbot to understand what is going on """ self.put_nowait(msg)
class Camera(PiResource): async def handel(self, actionType, data): if actionType == ActionType.SHOT: await self.___shot(data) if actionType == ActionType.RECORD: await self.___record(data) if actionType == ActionType.STREAM: await self.___stream(data) if actionType == ActionType.TEARDOWN: await self.___teardown(data) pass async def available(self): self.___available = True async def close(self): print("close Camera") await self.___teardown({}) self.___available = False pass ___pc = None ___player = None ___options = {"framerate": "25", "video_size": "640x480"} def __init__(self, param): self.___param = param self.___available = True async def ___shot(self, data): print("SHOT ", "Camera") return None async def ___record(self, data): print("RECORD ", "Camera") return None async def ___stream(self, data): if self.___available and data: offer = RTCSessionDescription(sdp=data["sdp"], type=data["type"]) self.___pc = RTCPeerConnection() @self.___pc.on("iceconnectionstatechange") async def on_iceconnectionstatechange(): print(self.___pc.iceConnectionState) if self.___pc.iceConnectionState == "failed": await self.___teardown({}) await self.___pc.setRemoteDescription(offer) player = MediaPlayer("/dev/video0", format="v4l2", options=self.___options) for t in self.___pc.getTransceivers(): if t.kind == "audio" and player.audio: self.___pc.addTrack(player.audio) elif t.kind == "video" and player.video: self.___pc.addTrack(player.video) answer = await self.___pc.createAnswer() await self.___pc.setLocalDescription(answer) print("STREAM ", "Camera") res = { "sdp": self.___pc.localDescription.sdp, "type": self.___pc.localDescription.type, "action": "STREAM" } self.___param["notify"](res) self.___available = False async def ___teardown(self, data): print("TEARDOWN ", "Camera") if self.___pc: await self.___pc.close() self.___available = True
async def offer(request): params = await request.json() print(params) offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) pc = RTCPeerConnection() client_id = client_manager.create_new_client(pc) # data = [True] #recog_worker_thread = threading.Thread(target=recog_worker,args=(client_manager.get_client(client_id),data,)) #recog_worker_thread.start() @pc.on("iceconnectionstatechange") async def on_iceconnectionstatechange(): print("ICE connection state is %s" % pc.iceConnectionState) if pc.iceConnectionState == "failed": await client_manager.remove_client( client_id) #this already handles pc closing faceregtrack = FacialRecognitionTrack(face_detect, client_manager.get_client(client_id)) @pc.on("datachannel") def on_datachannel(channel): @channel.on("message") def on_message(message): client = client_manager.get_client(client_id) if "$register$" in message: #this is some poorman message handling :) new_subject = message.split("$register$")[1] print("Registering for subject: " + new_subject) client.toggle_register_mode(new_subject) channel.send("$register$") #ack back elif "$recognize$" in message: print("Turned on recognition mode") client.toggle_recognition_mode() channel.send("$recognize$") #ack back @pc.on("close") async def on_close(track): print("Connection with client: " + str(client_id) + " closed") #data[0] = False #recog_worker_thread.join() await client_manager.remove_client(client_id) @pc.on("track") def on_track(track): print("Track %s received" % track.kind) if track.kind == "video": faceregtrack.update(track) await pc.setRemoteDescription(offer) for t in pc.getTransceivers(): if t.kind == "video": pc.addTrack(faceregtrack) answer = await pc.createAnswer() await pc.setLocalDescription(answer) print("Connection with client formed") return web.Response( content_type="application/json", text=json.dumps({ "sdp": pc.localDescription.sdp, "type": pc.localDescription.type }), )