def video():
import cv2
cv2.namedWindow('Output', cv2.WND_PROP_FULLSCREEN)
camera = cv2.VideoCapture(0)
context = zmq.Context()
publisher = context.socket(zmq.PUB)
publisher.bind('tcp://*:{}'.format(CAMERA))
projector = context.socket(zmq.PULL)
projector.bind('tcp://*:{}'.format(PROJECTOR))
eventQ = context.socket(zmq.SUB)
eventQ.connect('tcp://localhost:{}'.format(EVENT))
eventQ.setsockopt(zmq.SUBSCRIBE, b'')
poller = zmq.Poller()
poller.register(eventQ, zmq.POLLIN)
poller.register(projector, zmq.POLLIN)
while True:
events = dict(poller.poll(timeout=0))
if eventQ in events:
pushbutton = eventQ.recv_json()
if 'display2' in pushbutton:
cv2.moveWindow('Output', 2000, 100)
if 'fullscreen' in pushbutton:
cv2.setWindowProperty('Output', cv2.WND_PROP_FULLSCREEN, cv2.cv.CV_WINDOW_FULLSCREEN)
if projector in events:
cv2.imshow('Output', cv2.resize(recv_array(projector), FRAME_SIZE))
else:
cv2.imshow('Output', np.zeros(FRAME_SIZE[::-1]))
_, frame = camera.read()
frame = cv2.resize(frame, FRAME_SIZE)
send_array(publisher, frame)
cv2.waitKey(VIDEO_SAMPLE_TIME)
def live(audio_recognizer, audio_producer, audio2video, scaler, host):
import Oger
me = mp.current_process()
print me.name, 'PID', me.pid
context = zmq.Context()
mic = context.socket(zmq.SUB)
mic.connect('tcp://{}:{}'.format(host, MIC))
mic.setsockopt(zmq.SUBSCRIBE, b'')
speaker = context.socket(zmq.PUSH)
speaker.connect('tcp://{}:{}'.format(host, SPEAKER))
camera = context.socket(zmq.SUB)
camera.connect('tcp://{}:{}'.format(host, CAMERA))
camera.setsockopt(zmq.SUBSCRIBE, b'')
projector = context.socket(zmq.PUSH)
projector.connect('tcp://{}:{}'.format(host, PROJECTOR))
stateQ = context.socket(zmq.SUB)
stateQ.connect('tcp://{}:{}'.format(host, STATE))
stateQ.setsockopt(zmq.SUBSCRIBE, b'')
eventQ = context.socket(zmq.SUB)
eventQ.connect('tcp://{}:{}'.format(host, EVENT))
eventQ.setsockopt(zmq.SUBSCRIBE, b'')
snapshot = context.socket(zmq.REQ)
snapshot.connect('tcp://{}:{}'.format(host, SNAPSHOT))
snapshot.send(b'Send me the state, please')
state = snapshot.recv_json()
sender = context.socket(zmq.PUSH)
sender.connect('tcp://{}:{}'.format(host, EXTERNAL))
sender.send_json('register {}'.format(me.name))
poller = zmq.Poller()
poller.register(mic, zmq.POLLIN)
poller.register(camera, zmq.POLLIN)
poller.register(stateQ, zmq.POLLIN)
poller.register(eventQ, zmq.POLLIN)
previous_prediction = []
# Approximately 10 seconds of audio/video
error = deque(maxlen=3400)
audio = deque(maxlen=3400)
video = deque(maxlen=80)
while True:
events = dict(poller.poll())
if stateQ in events:
state = stateQ.recv_json()
if mic in events:
new_audio = np.atleast_2d(recv_array(mic))
if state['record']:
scaled_signals = scaler.transform(new_audio)
audio.append(np.ndarray.flatten(scaled_signals))
if len(previous_prediction):
error.append(scaled_signals[:,idxs].flatten() - previous_prediction.flatten())
previous_prediction = audio_recognizer(scaled_signals[:,idxs]) # This would not be necessary in a centralized recognizer
if camera in events:
new_video = recv_array(camera)
if state['record']:
video.append(new_video)
if eventQ in events:
pushbutton = eventQ.recv_json()
if 'reset' in pushbutton:
error.clear()
audio.clear()
video.clear()
previous_prediction = []
if 'rmse' in pushbutton:
rmse = np.sqrt((np.array(list(error)).flatten() ** 2).mean())
sender.send_json('{} RMSE {}'.format(me.name, rmse))
if 'respond' in pushbutton and pushbutton['respond'] == me.name:
audio_data = np.array(list(audio))
video_data = np.array(list(video))
print '{} chosen to respond. Audio data: {} Video data: {}'.format(me.name, audio_data.shape, video_data.shape)
if audio_data.size == 0 and video_data.size == 0:
print '*** Audio data and video data arrays are empty. Aborting the response. ***'
continue
row_diff = audio_data.shape[0] - audio_producer.length
if row_diff < 0:
audio_data = np.vstack([ audio_data, np.zeros((-row_diff, audio_data.shape[1])) ])
else:
audio_data = audio_data[:audio_producer.length]
sound = audio_producer(audio_data)
stride = audio_producer.length/audio2video.length
projection = audio2video(audio_data[audio_data.shape[0] - stride*audio2video.length::stride])
# DREAM MODE: You can train a network with zero audio input -> video output, and use this
# to recreate the original training sequence with scary accuracy...
for row in projection:
send_array(projector, row)
for row in scaler.inverse_transform(sound):
send_array(speaker, row)
if 'save' in pushbutton:
filename = '{}.{}'.format(pushbutton['save'], me.name)
pickle.dump((audio_recognizer, audio_producer, audio2video, scaler, host), file(filename, 'w'))
print '{} saved as file {} ({})'.format(me.name, filename, filesize(filename))
def live(audio_recognizer, audio_producer, audio2video, scaler, host):
import Oger
me = mp.current_process()
print me.name, 'PID', me.pid
context = zmq.Context()
mic = context.socket(zmq.SUB)
mic.connect('tcp://{}:{}'.format(host, MIC))
mic.setsockopt(zmq.SUBSCRIBE, b'')
speaker = context.socket(zmq.PUSH)
speaker.connect('tcp://{}:{}'.format(host, SPEAKER))
camera = context.socket(zmq.SUB)
camera.connect('tcp://{}:{}'.format(host, CAMERA))
camera.setsockopt(zmq.SUBSCRIBE, b'')
projector = context.socket(zmq.PUSH)
projector.connect('tcp://{}:{}'.format(host, PROJECTOR))
stateQ = context.socket(zmq.SUB)
stateQ.connect('tcp://{}:{}'.format(host, STATE))
stateQ.setsockopt(zmq.SUBSCRIBE, b'')
eventQ = context.socket(zmq.SUB)
eventQ.connect('tcp://{}:{}'.format(host, EVENT))
eventQ.setsockopt(zmq.SUBSCRIBE, b'')
snapshot = context.socket(zmq.REQ)
snapshot.connect('tcp://{}:{}'.format(host, SNAPSHOT))
snapshot.send(b'Send me the state, please')
state = snapshot.recv_json()
sender = context.socket(zmq.PUSH)
sender.connect('tcp://{}:{}'.format(host, EXTERNAL))
sender.send_json('register {}'.format(me.name))
poller = zmq.Poller()
poller.register(mic, zmq.POLLIN)
poller.register(camera, zmq.POLLIN)
poller.register(stateQ, zmq.POLLIN)
poller.register(eventQ, zmq.POLLIN)
previous_prediction = []
# Approximately 10 seconds of audio/video
error = deque(maxlen=3400)
audio = deque(maxlen=3400)
video = deque(maxlen=80)
while True:
events = dict(poller.poll())
if stateQ in events:
state = stateQ.recv_json()
if mic in events:
new_audio = np.atleast_2d(recv_array(mic))
if state['record']:
scaled_signals = scaler.transform(new_audio)
audio.append(np.ndarray.flatten(scaled_signals))
if len(previous_prediction):
error.append(scaled_signals[:, idxs].flatten() -
previous_prediction.flatten())
previous_prediction = audio_recognizer(
scaled_signals[:, idxs]
) # This would not be necessary in a centralized recognizer
if camera in events:
new_video = recv_array(camera)
if state['record']:
video.append(new_video)
if eventQ in events:
pushbutton = eventQ.recv_json()
if 'reset' in pushbutton:
error.clear()
audio.clear()
video.clear()
previous_prediction = []
if 'rmse' in pushbutton:
rmse = np.sqrt((np.array(list(error)).flatten()**2).mean())
sender.send_json('{} RMSE {}'.format(me.name, rmse))
if 'respond' in pushbutton and pushbutton['respond'] == me.name:
audio_data = np.array(list(audio))
video_data = np.array(list(video))
print '{} chosen to respond. Audio data: {} Video data: {}'.format(
me.name, audio_data.shape, video_data.shape)
if audio_data.size == 0 and video_data.size == 0:
print '*** Audio data and video data arrays are empty. Aborting the response. ***'
continue
row_diff = audio_data.shape[0] - audio_producer.length
if row_diff < 0:
audio_data = np.vstack([
audio_data,
np.zeros((-row_diff, audio_data.shape[1]))
])
else:
audio_data = audio_data[:audio_producer.length]
sound = audio_producer(audio_data)
stride = audio_producer.length / audio2video.length
projection = audio2video(
audio_data[audio_data.shape[0] -
stride * audio2video.length::stride])
# DREAM MODE: You can train a network with zero audio input -> video output, and use this
# to recreate the original training sequence with scary accuracy...
for row in projection:
send_array(projector, row)
for row in scaler.inverse_transform(sound):
send_array(speaker, row)
if 'save' in pushbutton:
filename = '{}.{}'.format(pushbutton['save'], me.name)
pickle.dump((audio_recognizer, audio_producer, audio2video,
scaler, host), file(filename, 'w'))
print '{} saved as file {} ({})'.format(
me.name, filename, filesize(filename))
