class Audio:
RESPEAKER_RATE = 16000
RESPEAKER_CHANNELS = 1
RESPEAKER_WIDTH = 2
# import sounddevice as sd
# print sd.query_devices()
RESPEAKER_INDEX = 2 # refer to input device id
CHUNK = 1024
RECORD_SECONDS = 5
WAVE_OUTPUT_FILENAME = "sc.wav"
def __init__(self):
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
self.mic_tuning = Tuning(dev)
self.mic_tuning.set_vad_threshold(3.5)
self.recording = False
def readDOA(self):
return self.mic_tuning.direction
def readVAD(self):
return self.mic_tuning.is_voice()
@staticmethod
def getSampleSize():
p = pyaudio.PyAudio()
return p.get_sample_size(p.get_format_from_width(
Audio.RESPEAKER_WIDTH))
def record(self):
p = pyaudio.PyAudio()
try:
stream = p.open(rate=Audio.RESPEAKER_RATE,
format=p.get_format_from_width(
Audio.RESPEAKER_WIDTH),
channels=Audio.RESPEAKER_CHANNELS,
input=True)
except IOError:
return
frames = []
for i in range(
0,
int(Audio.RESPEAKER_RATE / Audio.CHUNK *
Audio.RECORD_SECONDS)):
data = stream.read(Audio.CHUNK)
frames.append(data)
stream.stop_stream()
stream.close()
wf = wave.open(Audio.WAVE_OUTPUT_FILENAME, 'wb')
wf.setnchannels(1)
wf.setsampwidth(
p.get_sample_size(p.get_format_from_width(Audio.RESPEAKER_WIDTH)))
wf.setframerate(Audio.RESPEAKER_RATE)
wf.writeframes(b''.join(frames))
wf.close()
def __init__(self):
self.dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
if self.dev:
self.Mic_tuning = Tuning(self.dev)
self.vad_pub = rospy.Publisher("vad", Int16, queue_size=1)
self.doa_pub = rospy.Publisher("Doa", Float32, queue_size=1)
self.publishers()
def main():
# find microphone array device
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
# print("Device: {}".format(dev))
# if the device exists
if dev:
last_doa = 0
mic = Tuning(dev)
file_ts = datetime.today().strftime("%Y%m%d%H%M%S")
with open("log_" + file_ts + ".csv", "w") as csvfile:
csvwriter = csv.writer(csvfile)
csvwriter.writerow(["Timestamp", "VAD", "DOA"])
while True:
ts = time.time()
vad = mic.is_voice()
doa = mic.direction
# print to console
print("[{:.2f}] VAD: {} DOA: {}°".format(ts, vad, doa))
# write to csv
csvwriter.writerow([[ts, vad, doa]])
# wait
time.sleep(0.5)
# if the device does not exist
else:
print("ReSpeaker Mic Array v2.0 device not found")
return
def transfer_audio_msg(self):
"""Function to publish input audio to topic"""
rospy.loginfo("audio input node will start after delay of 5 seconds")
sleep(5)
# Params
self._input = "~input"
_rate_bool = False
#creating the object for reading speech recognized value from respeaker
if self.dev:
self.Mic_tuning = Tuning(self.dev)
# Checking if audio file given or system microphone is needed
if rospy.has_param(self._input):
if rospy.get_param(self._input) != ":default":
_rate_bool = True
stream = open(rospy.get_param(self._input), 'rb')
rate = rospy.Rate(5) # 10hz
else:
# Initializing pyaudio for input from system microhpone
stream = pyaudio.PyAudio().open(format=pyaudio.paInt16,
channels=6,
rate=16000,
input=True,
frames_per_buffer=1024)
stream.start_stream()
else:
rospy.logerr(
"No input means provided. Please use the launch file instead")
while not rospy.is_shutdown():
data = stream.read(1024)
speech = self.Mic_tuning.read('SPEECHDETECTED')
if self.rec_buffers == 0:
a = np.fromstring(data, dtype=np.int16)[0::6]
self.rec_buffers = 1
elif speech == 1:
np.append(a, np.fromstring(data, dtype=np.int16)[0::6])
self.rec_buffers += 1
# else:
# a = np.fromstring(data, dtype=np.int16)[0::6]
# self.rec_buffers = 1
# buf = stream.read(1024)
buf = a.tostring()
if buf:
if self.rec_buffers > 1 and speech == 0:
print("publishing")
# Publish audio to topic
self.pub_.publish(buf)
self.rec_buffers = 0
if _rate_bool:
rate.sleep()
else:
rospy.loginfo("Buffer returned null")
break
def __init__(self):
self.stop = False
logging.basicConfig()
self.logger = logging.getLogger("MMNT")
if DEBUG:
self.logger.setLevel(logging.DEBUG)
else:
self.logger.setLevel(logging.INFO)
self.logger.info("Initializing")
self.masterSampleTime = time.time()
self.slaveSampleTime = time.time()
self.humanSampleTime = time.time()
self.micSampleTime = time.time()
self.logger.debug("Initializing motor control")
self.mc = MotorControl()
self.mc.resetMotors()
self.logger.debug("Initializing microphone")
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
if not dev:
sys.exit("Could not find ReSpeaker Mic Array through USB")
self.mic = Tuning(dev)
self.mic.write("NONSTATNOISEONOFF", 1)
self.mic.write("STATNOISEONOFF", 1)
self.mic.write("ECHOONOFF", 1)
self.logger.debug("Initializing video streams")
self.topCamStream = VideoStream(1)
self.botCamStream = VideoStream(2)
self.logger.debug("Initializing models")
self.ht_model = ht.get_model()
self.tfPose = TfPoseEstimator(get_graph_path(TF_MODEL),
target_size=(VideoStream.DEFAULT_WIDTH,
VideoStream.DEFAULT_HEIGHT))
self.logger.info("Initialization complete")
self.topCamState = State.IDLE
self.botCamState = State.IDLE
self.topCamAngle = 0
self.topAngleUpdated = False
self.botCamAngle = 180
self.botAngleUpdated = False
self.master = Cams.TOP
self.lastMaster = Cams.TOP
self.botCamProc = None
self.topCamProc = None
self.audioMap = Map(15)
self.checkMic()
def respeaker():
global dev
pub = rospy.Publisher('ReSpeaker', Int32, queue_size=1)
rospy.init_node('ReSpeaker_Pub', anonymous=True)
mic_tuning = Tuning(dev)
rate = rospy.Rate(10)
while not rospy.is_shutdown():
try:
if mic_tuning.is_voice() == 1:
direction_str = (360-mic_tuning.direction)*1280/360
pub.publish(direction_str)
except KeyboardInterrupt:
break
rate.sleep()
def __init__(self):
self.dev = usb.core.find(idVendor=VENDORID, idProduct=PRODUCTID)
self.dev_tuning = Tuning(self.dev)
self.pixel_ring = PixelRing(self.dev)
self.pixel_ring.off()
self.led_think()
time.sleep(3)
self.led_listen()
def main():
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
# ser = serial.Serial("/dev/tty.usbmodem14201", 9600, timeout=1)
# ser = serial.Serial("/dev/ttyACM0", 9600, timeout=1) # Ubuntu VM
ser = serial.Serial("/dev/tty.usbmodem14301", 9600, timeout=1) # OSX
time.sleep(3)
prevAngle = 0
if dev:
mic = Tuning(dev)
mic.write("NONSTATNOISEONOFF", 1)
mic.write("STATNOISEONOFF", 1)
while True:
try:
# prev to ref
# 0 to 65:
# 65 to 0
# 350 to 0
# 0 to 270
rotAngle = 0
rotDir = CCW
cwDiff = 0
ccwDiff = 0
refAngle = mic.direction
print "refAngle:", refAngle, "prevAngle:", prevAngle
diff = refAngle - prevAngle
if abs(diff) < THRESHOLD:
time.sleep(2)
continue
if refAngle < prevAngle:
ccwDiff = (360 - prevAngle) + refAngle
cwDiff = prevAngle - refAngle
else:
ccwDiff = refAngle - prevAngle
cwDiff = prevAngle + (360 - refAngle)
if cwDiff < ccwDiff:
rotAngle = cwDiff
rotDir = CW
print "CW rotAngle:", rotAngle
else:
rotAngle = ccwDiff
rotDir = CCW
print "CCW rotAngle:", rotAngle
if rotAngle > 180:
print "~~~~~~~~~~~~ANGLE IS OVER 180 BAD MATH~~~~~~~~~~~~"
cmd = buildMotorCommand(motor1Id, rotDir, rotAngle / stepSize)
print cmd
prevAngle = refAngle
ser.flushInput()
ser.flushOutput()
ser.write(cmd)
time.sleep(2) # minimum time for a 360 from a send command
except KeyboardInterrupt:
print "terminating"
break
class send_params:
def __init__(self):
self.dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
if self.dev:
self.Mic_tuning = Tuning(self.dev)
self.vad_pub = rospy.Publisher("vad", Int16, queue_size=1)
self.doa_pub = rospy.Publisher("Doa", Float32, queue_size=1)
self.publishers()
def publishers(self):
while not rospy.is_shutdown():
doa = self.Mic_tuning.direction
speech = self.Mic_tuning.read('SPEECHDETECTED')
time.sleep(0.05)
self.vad_pub.publish(speech)
self.doa_pub.publish(doa)
from tuning import Tuning
import usb.core
import usb.util
import time
import sys
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
if dev:
Mic_tuning = Tuning(dev)
while True:
try:
#print "\rDIR:", Mic_tuning.direction, ", VAD:", Mic_tuning.is_voice(), "SPEECH:", Mic_tuning.read('SPEECHDETECTED'),
if Mic_tuning.read('SPEECHDETECTED'):
print 's',
elif Mic_tuning.is_voice():
print ".",
sys.stdout.flush()
time.sleep(0.1)
except KeyboardInterrupt:
break
from tuning import Tuning
import usb.core
import usb.util
import time
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
if dev:
Mic_tuning = Tuning(dev)
print(Mic_tuning.direction)
while True:
try:
if Mic_tuning.read('SPEECHDETECTED') == 1:
print(Mic_tuning.direction)
time.sleep(1)
except KeyboardInterrupt:
break
from tuning import Tuning
import usb.core
import usb.util
import time
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
#print dev
if dev:
Mic_tuning = Tuning(dev)
print Mic_tuning.is_voice()
Mic_tuning.set_vad_threshold(15.0)
while True:
try:
# print Mic_tuning.read('GAMMAVAD_SR')
print(Mic_tuning.read('SPEECHDETECTED'),Mic_tuning.is_voice())
# print Mic_tuning.is_voice()
time.sleep(0.05)
except KeyboardInterrupt:
break
head_pub = rospy.Publisher('/qt_robot/head_position/command',
Float64MultiArray,
queue_size=1)
# wait for publisher connections
wtime_begin = rospy.get_time()
while (head_pub.get_num_connections() == 0):
rospy.loginfo("waiting for publisher connections...")
if rospy.get_time() - wtime_begin > 10.0:
rospy.logerr("Timeout while waiting for publisher connection!")
sys.exit()
rospy.sleep(1)
# centering the head
href = Float64MultiArray(data=[0, 0])
head_pub.publish(href)
rospy.sleep(1)
microphone = Tuning(dev)
while not rospy.is_shutdown():
if not microphone.is_voice():
continue
mic = abs(microphone.direction - 180)
angle = mic - MICROPHONE_ANGLE_OFFSET
rospy.loginfo("mic: %d , head: %d" % (mic, angle))
href.data = [angle, 0]
head_pub.publish(href)
rospy.loginfo("shutdowned!")
from tuning import Tuning
import usb.core
import usb.util
import time
if __name__ == '__main__':
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
if dev:
Mic_tuning = Tuning(dev)
while True:
try:
print("Voice activity %d" % Mic_tuning.is_voice())
time.sleep(0.2)
except KeyboardInterrupt:
break
#!/usr/bin/env python
import usb.util
from tuning import Tuning
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
mic_tuning = Tuning(dev)
while True:
if mic_tuning.is_voice() == 1:
direction_str = (360 - mic_tuning.direction) * 1280 / 360
print(direction_str)
break
from tuning import Tuning
import usb.core
import usb.util
import time
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
# print dev
if dev:
Mic_tuning = Tuning(dev)
print
Mic_tuning.is_voice()
while True:
try:
print(Mic_tuning.is_voice())
time.sleep(1)
except KeyboardInterrupt:
break
# for speaker
from subprocess import call
cmd_beg = 'espeak '
cmd_end = ' | aplay ./temp_voice_response.wav 2>/dev/null' # To play back the stored .wav file and to dump the std errors to /dev/null
cmd_out = '--stdout > temp_voice_response.wav ' # To store the voice file
# end of for speaker
# usb core dev
from tuning import Tuning
import usb.core
import usb.util
from usb_pixel_ring_v2 import PixelRing
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
if dev:
mic = Tuning(dev)
pixels = PixelRing(dev)
else:
mic = None
pixels = None
last_time_motor_moved = 0
def main():
model_path = get_model_path()
config = Decoder.default_config()
# config.set_string('-hmm', os.path.join(model_path, 'en-us'))
# config.set_string('-lm', os.path.join(model_path, 'en-us.lm.bin'))
# config.set_string('-dict', os.path.join(model_path, 'cmudict-en-us.dict'))
config.set_string('-hmm', os.path.join(model_path, 'en-us'))
decoder_nbestsize = parser.get('decoder', 'nbestsize')
decoder_options = decoder_options + " " + decoder_nbestsize
except:
pass
try:
decoder_nbestdistinct = parser.get('decoder', 'nbestdistinct')
decoder_options = decoder_options + " " + decoder_nbestdistinct
except:
pass
parser.set('decoder', 'options', decoder_options)
decoder_nbestout = open(decoder_nbestfile, 'w')
if tuning_switch == "on":
Tuning_object = Tuning(parser)
Decoder_object = Decoder_Moses_nbest(parser, Tuning_object)
elif tuning_switch == "off":
Decoder_object = Decoder_Moses_nbest(parser)
if not showweightsflag == "":
decoder_out, decoder_err = Decoder_object.show_weights()
# write weights to stdout
sys.stdout.write(''.join(decoder_out))
sys.stdout.flush()
sys.exit(1)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import socket
import usb.core
import usb.util
from tuning import Tuning
from time import sleep
before_angular = 0
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
while (1):
cliant = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
Mic_tuning = Tuning(dev)
theta = Mic_tuning.direction
s.connect(('localhost', 11451))
if (before_angular - theta >= 40 or before_angular - theta <= -40):
s.sendall(str(theta))
before_angular = theta
def __init__(self):
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
self.mic_tuning = Tuning(dev)
self.mic_tuning.set_vad_threshold(3.5)
self.recording = False
if __name__ == '__main__':
global pepper_is_speaking
client.on_message = on_message #attach function to callback
client.loop_start() #start the loop
rospy.loginfo("Subscribing to topic", "hermes/asr/textCaptured")
client.subscribe("hermes/asr/textCaptured")
ros_listener()
#wait until microphone is correctly restarted from the other script
rospy.loginfo(
"Snips script is sleeping for 20 seconds to wait for respeaker_node to finish launching"
)
time.sleep(20)
if dev:
Mic_tuning = Tuning(dev)
while True:
try:
#rospy.loginfo"VOICEACTIVITY:", Mic_tuning.is_voice()),
speechdetected = Mic_tuning.read(
"VOICEACTIVITY") #SPEECHRECOGNIZED fires too often
#rospy.loginfospeechdetected),
if speechdetected:
if time.time(
) - last_speechdetected_timestamp > LAUNCH_INTERVAL and not pepper_is_speaking:
client.publish('hermes/asr/startListening',
'{"siteId": "default"}')
last_speechdetected_timestamp = time.time(
) #we don't want to trigger more than once per second or so
def find(vid=0x2886, pid=0x0018):
dev = usb.core.find(idVendor=vid, idProduct=pid)
if not dev:
return
return Tuning(dev)
class Moment(object):
def __init__(self):
self.stop = False
logging.basicConfig()
self.logger = logging.getLogger("MMNT")
if DEBUG:
self.logger.setLevel(logging.DEBUG)
else:
self.logger.setLevel(logging.INFO)
self.logger.info("Initializing")
self.masterSampleTime = time.time()
self.slaveSampleTime = time.time()
self.humanSampleTime = time.time()
self.micSampleTime = time.time()
self.logger.debug("Initializing motor control")
self.mc = MotorControl()
self.mc.resetMotors()
self.logger.debug("Initializing microphone")
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
if not dev:
sys.exit("Could not find ReSpeaker Mic Array through USB")
self.mic = Tuning(dev)
self.mic.write("NONSTATNOISEONOFF", 1)
self.mic.write("STATNOISEONOFF", 1)
self.mic.write("ECHOONOFF", 1)
self.logger.debug("Initializing video streams")
self.topCamStream = VideoStream(1)
self.botCamStream = VideoStream(2)
self.logger.debug("Initializing models")
self.ht_model = ht.get_model()
self.tfPose = TfPoseEstimator(get_graph_path(TF_MODEL),
target_size=(VideoStream.DEFAULT_WIDTH,
VideoStream.DEFAULT_HEIGHT))
self.logger.info("Initialization complete")
self.topCamState = State.IDLE
self.botCamState = State.IDLE
self.topCamAngle = 0
self.topAngleUpdated = False
self.botCamAngle = 180
self.botAngleUpdated = False
self.master = Cams.TOP
self.lastMaster = Cams.TOP
self.botCamProc = None
self.topCamProc = None
self.audioMap = Map(15)
self.checkMic()
def stop(self):
self.stop = True
def updateTopAngle(self, angle):
if abs(angle - self.topCamAngle) > ANGLE_THRESHOLD and abs(
angle - self.botCamAngle) > OVERLAP_THRESHOLD:
self.topCamAngle = angle
self.topAngleUpdated = True
def updateBotAngle(self, angle):
if abs(angle - self.botCamAngle) > ANGLE_THRESHOLD and abs(
angle - self.topCamAngle) > OVERLAP_THRESHOLD:
self.botCamAngle = angle
self.botAngleUpdated = True
def updatePositions(self):
# Send Serial Commands
if self.topAngleUpdated and self.botAngleUpdated:
self.logger.debug("Top Angle: {}".format(self.topCamAngle))
self.logger.debug("Bot Angle: {}".format(self.botCamAngle))
self.topAngleUpdated = False
self.botAngleUpdated = False
self.mc.runMotors(self.topCamAngle, self.botCamAngle)
elif self.topAngleUpdated:
self.logger.debug("Top Angle: {}".format(self.topCamAngle))
self.topAngleUpdated = False
self.mc.runTopMotor(self.topCamAngle)
elif self.botAngleUpdated:
self.logger.debug("Bot Angle: {}".format(self.botCamAngle))
self.botAngleUpdated = False
self.mc.runBotMotor(self.botCamAngle)
def isWithinNoiseFov(self, angle):
topDiff = abs(angle - self.topCamAngle)
botDiff = abs(angle - self.botCamAngle)
if topDiff < NOISE_ANGLE_THRESHOLD:
self.topCamState |= State.NOISE
if self.topCamState == State.BOTH:
self.master = Cams.TOP
return True
else:
self.topCamState &= ~State.NOISE
if botDiff < NOISE_ANGLE_THRESHOLD:
self.botCamState |= State.NOISE
if self.botCamState == State.BOTH:
self.master = Cams.BOT
return True
else:
self.botCamState &= ~State.NOISE
return False
def checkMic(self):
speechDetected, micDOA = self.mic.speech_detected(), self.mic.direction
if not speechDetected:
# self.audioMap.update_map_with_no_noise()
self.topCamState &= ~State.NOISE
self.botCamState &= ~State.NOISE
return
self.logger.debug("speech detected from {}".format(micDOA))
self.audioMap.update_map_with_noise(micDOA)
primaryMicDOA, secondaryMicDOA = self.audioMap.get_POI_location()
if DEBUG:
self.audioMap.print_map()
if primaryMicDOA == -1:
self.logger.debug("no good audio source")
return
self.logger.debug("mapped audio from {}".format(primaryMicDOA))
# Check if camera is already looking at the primary noise source
if self.isWithinNoiseFov(primaryMicDOA):
# If camera is already looking, check the secondary noise source
if secondaryMicDOA == -1:
self.logger.debug("no good secondary audio source")
return
elif self.isWithinNoiseFov(secondaryMicDOA):
return
else:
micDOA = secondaryMicDOA
else:
micDOA = primaryMicDOA
topDiff = abs(micDOA - self.topCamAngle)
botDiff = abs(micDOA - self.botCamAngle)
# Camera is NOT looking at the noise source at this point
# If both Cameras are not tracking a human,
# move the closest camera
if self.topCamState < State.HUMAN and self.botCamState < State.HUMAN:
if botDiff < topDiff:
self.botCamState |= State.NOISE
self.updateBotAngle(micDOA)
if self.botCamState == State.IDLE:
self.master = Cams.TOP
else:
self.topCamState |= State.NOISE
self.updateTopAngle(micDOA)
if self.topCamState == State.IDLE:
self.master = Cams.BOT
# One of the cameras are on a human, if the other camera is not on a human, move it
elif self.topCamState < State.HUMAN:
self.topCamState |= State.NOISE
self.updateTopAngle(micDOA)
self.master = Cams.BOT
elif self.botCamState < State.HUMAN:
self.botCamState |= State.NOISE
self.updateBotAngle(micDOA)
self.master = Cams.TOP
# The cameras are on a human
else:
# If both are on a human, move the one that's not master
if self.topCamState == State.HUMAN and self.botCamState == State.HUMAN:
if self.master != Cams.BOT:
self.botCamState |= State.NOISE
self.updateBotAngle(micDOA)
else:
self.topCamState |= State.NOISE
self.updateTopAngle(micDOA)
# One of the cameras are on a HUMAN+NOISE, move the one that's not only on a HUMAN
elif self.topCamState == State.HUMAN:
self.topCamState |= State.NOISE
self.updateTopAngle(micDOA)
self.master = Cams.BOT
elif self.botCamState == State.HUMAN:
self.botCamState |= State.NOISE
self.updateBotAngle(micDOA)
self.master = Cams.TOP
def getBestFace(self, humans):
midX = -1
bestHuman = humans[0]
maxScore = 0
for human in humans:
gotMidX = False
score = 0
currMidX = -1
for part in headParts:
if part in human.body_parts:
score += human.body_parts[part].score
if not gotMidX:
currMidX = human.body_parts[
part].x * VideoStream.DEFAULT_WIDTH
gotMidX = True
if score > maxScore:
maxScore = score
midX = currMidX
bestHuman = human
return bestHuman, midX
def checkHumans(self, frame, camera):
humans = self.tfPose.inference(frame,
resize_to_default=True,
upsample_size=RESIZE_RATIO)
if len(humans):
if camera == Cams.TOP:
self.topCamState |= State.HUMAN
if self.topCamState == State.BOTH:
self.master = Cams.TOP
else:
self.botCamState |= State.HUMAN
if self.botCamState == State.BOTH:
self.master = Cams.BOT
if DISPLAY_VIDEO and DRAW_ON_FRAME:
TfPoseEstimator.draw_humans(frame, humans, imgcopy=False)
human, midX = self.getBestFace(humans)
if (ht.is_hands_above_head(human)):
self.logger.debug("HANDS ABOVE HEAD!!!")
if midX != -1:
centerDiff = abs(midX - VideoStream.DEFAULT_WIDTH / 2)
if centerDiff > FACE_THRESHOLD:
if midX < VideoStream.DEFAULT_WIDTH / 2:
# rotate CCW
if camera == Cams.TOP:
self.updateTopAngle(
(self.topCamAngle +
centerDiff * degreePerPixel) % 360)
else:
self.updateBotAngle(
(self.botCamAngle +
centerDiff * degreePerPixel) % 360)
elif midX > VideoStream.DEFAULT_WIDTH / 2:
# rotate CW
if camera == Cams.TOP:
self.updateTopAngle(
(self.topCamAngle -
centerDiff * degreePerPixel) % 360)
else:
self.updateBotAngle(
(self.botCamAngle -
centerDiff * degreePerPixel) % 360)
else:
if camera == Cams.TOP:
self.topCamState &= ~State.HUMAN
else:
self.botCamState &= ~State.HUMAN
return frame
def playVideo(self, cam):
if cam == Cams.TOP:
if self.botCamProc is not None and self.botCamProc.poll(
) is not None:
self.botCamProc.kill()
self.topCamProc = subprocess.Popen(
"ffmpeg -f v4l2 -i /dev/video3 -f v4l2 /dev/video5",
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
elif cam == Cams.BOT:
if self.topCamProc is not None and self.topCamProc.poll(
) is not None:
self.topCamProc.kill()
self.botCamProc = subprocess.Popen(
"ffmpeg -f v4l2 -i /dev/video4 -f v4l2 /dev/video5",
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
def start(self):
Thread(target=self.run, args=()).start()
def run(self):
self.stop = False
while not self.stop:
try:
topFrame = self.topCamStream.read()
botFrame = self.botCamStream.read()
if time.time() - self.humanSampleTime > HUMAN_SAMPLE_FREQ:
if topFrame is not None:
topFrame = self.checkHumans(topFrame, Cams.TOP)
if botFrame is not None:
botFrame = self.checkHumans(botFrame, Cams.BOT)
self.humanSampleTime = time.time()
if time.time() - self.micSampleTime > MIC_SAMPLE_FREQ:
self.checkMic()
self.micSampleTime = time.time()
self.updatePositions()
# if DISPLAY_VIDEO and topFrame is not None and botFrame is not None:
# if self.master == Cams.TOP:
# if topFrame is not None:
# cv.imshow('Master', topFrame)
# if botFrame is not None:
# cv.imshow('Slave', botFrame)
# else:
# if botFrame is not None:
# cv.imshow('Master', botFrame)
# if topFrame is not None:
# cv.imshow('Slave', topFrame)
# if cv.waitKey(1) == 27:
# pass
if DISPLAY_VIDEO and topFrame is not None and botFrame is not None:
if self.master == Cams.TOP:
top_master = np.concatenate((topFrame, botFrame),
axis=1)
cv.imshow('Master + Slave', top_master)
else:
bot_master = np.concatenate((botFrame, topFrame),
axis=1)
cv.imshow('Master + Slave', bot_master)
if cv.waitKey(1) == 27:
pass
except KeyboardInterrupt:
self.logger.debug("Keyboard interrupt! Terminating.")
break
self.mc.resetMotors()
def __init__(self):
# Initializing publisher with buffer size of 10 messages
self.pub_ = rospy.Publisher("kws_data", String, queue_size=10)
# Initalizing publisher for continuous ASR
self.continuous_pub_ = rospy.Publisher("jsgf_audio",
String,
queue_size=10)
# initialize node
rospy.init_node("kws_control")
# Call custom function on node shutdown
rospy.on_shutdown(self.shutdown)
# for detect is speech
self.count_subs = 0
self.not_speech = 0
self.dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
if self.dev:
self.Mic_tuning = Tuning(self.dev)
self.speech_end_hysteresis = 0
self.utt_started = 0
self.previous_data_buf = []
self.speech = 0
# Params
# File containing language model
_hmm_param = "~hmm"
# Dictionary
_dict_param = "~dict"
# List of keywords to detect
_kws_param = "~kws"
"""Not necessary to provide the next two if _kws_param is provided
Single word which needs to be detected
"""
_keyphrase_param = "~keyphrase"
# Threshold frequency of above mentioned word
_threshold_param = "~threshold"
# Option for continuous
self._option_param = "~option"
self.count_recognized = 0
# Variable to distinguish between kws list and keyphrase.
# Default is keyword list
self._list = True
self.stop_output = False
# Setting param values
if rospy.has_param(_hmm_param):
self.class_hmm = rospy.get_param(_hmm_param)
if rospy.get_param(_hmm_param) == ":default":
if os.path.isdir("/usr/local/share/pocketsphinx/model"):
rospy.loginfo("Loading the default acoustic model")
self.class_hmm = "/usr/local/share/pocketsphinx/model/en-us/en-us"
rospy.loginfo("Done loading the default acoustic model")
else:
rospy.logerr(
"No language model specified. Couldn't find defaut model."
)
return
else:
rospy.loginfo("Couldn't find lm argument")
if rospy.has_param(
_dict_param) and rospy.get_param(_dict_param) != ":default":
self.lexicon = rospy.get_param(_dict_param)
else:
rospy.logerr(
'No dictionary found. Please add an appropriate dictionary argument.'
)
return
if rospy.has_param(
_kws_param) and rospy.get_param(_kws_param) != ":default":
self._list = True
self.kw_list = rospy.get_param(_kws_param)
elif rospy.has_param(_keyphrase_param) and \
rospy.has_param(_threshold_param) and \
rospy.get_param(_keyphrase_param) != ":default" and \
rospy.get_param(_threshold_param) != ":default":
self._list = False
self.keyphrase = rospy.get_param(_keyphrase_param)
self.kws_threshold = rospy.get_param(_threshold_param)
else:
rospy.logerr(
'kws cant run. Please add an appropriate keyword list.')
return
# All params satisfied. Starting recognizer
self.start_recognizer()
def direction():
# Return the direction of arrival of the sound
if dev:
Mic_tuning = Tuning(dev)
return Mic_tuning.direction
from tuning import Tuning
import usb.core
import usb.util
import time
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
if dev:
mic = Tuning(dev)
mic.write("NONSTATNOISEONOFF", 1)
mic.write("NONSTATNOISEONOFF", 1)
while True:
try:
print(mic.direction)
time.sleep(1)
except KeyboardInterrupt:
break
class AudioMessage(object):
"""Class to publish audio to topic"""
def __init__(self):
# Initializing publisher with buffer size of 10 messages
self.pub_ = rospy.Publisher("sphinx_audio", String, queue_size=10)
self.dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
self.rec_buffers = 0
# initialize node
rospy.init_node("audio_control")
# Call custom function on node shutdown
rospy.on_shutdown(self.shutdown)
# All set. Publish to topic
self.transfer_audio_msg()
def transfer_audio_msg(self):
"""Function to publish input audio to topic"""
rospy.loginfo("audio input node will start after delay of 5 seconds")
sleep(5)
# Params
self._input = "~input"
_rate_bool = False
#creating the object for reading speech recognized value from respeaker
if self.dev:
self.Mic_tuning = Tuning(self.dev)
# Checking if audio file given or system microphone is needed
if rospy.has_param(self._input):
if rospy.get_param(self._input) != ":default":
_rate_bool = True
stream = open(rospy.get_param(self._input), 'rb')
rate = rospy.Rate(5) # 10hz
else:
# Initializing pyaudio for input from system microhpone
stream = pyaudio.PyAudio().open(format=pyaudio.paInt16,
channels=6,
rate=16000,
input=True,
frames_per_buffer=1024)
stream.start_stream()
else:
rospy.logerr(
"No input means provided. Please use the launch file instead")
while not rospy.is_shutdown():
data = stream.read(1024)
speech = self.Mic_tuning.read('SPEECHDETECTED')
if self.rec_buffers == 0:
a = np.fromstring(data, dtype=np.int16)[0::6]
self.rec_buffers = 1
elif speech == 1:
np.append(a, np.fromstring(data, dtype=np.int16)[0::6])
self.rec_buffers += 1
# else:
# a = np.fromstring(data, dtype=np.int16)[0::6]
# self.rec_buffers = 1
# buf = stream.read(1024)
buf = a.tostring()
if buf:
if self.rec_buffers > 1 and speech == 0:
print("publishing")
# Publish audio to topic
self.pub_.publish(buf)
self.rec_buffers = 0
if _rate_bool:
rate.sleep()
else:
rospy.loginfo("Buffer returned null")
break
@staticmethod
def shutdown():
"""This function is executed on node shutdown."""
# command executed after Ctrl+C is pressed
rospy.loginfo("Stop ASRControl")
rospy.sleep(1)
from tuning import Tuning
import usb.core
import usb.util
import time
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
#print dev
if dev:
Mic_tuning = Tuning(dev)
print Mic_tuning.is_voice()
while True:
try:
print Mic_tuning.is_voice()
time.sleep(1)
except KeyboardInterrupt:
break
decoder_nbestsize = parser.get("decoder", "nbestsize")
decoder_options = decoder_options + " " + decoder_nbestsize
except:
pass
try:
decoder_nbestdistinct = parser.get("decoder", "nbestdistinct")
decoder_options = decoder_options + " " + decoder_nbestdistinct
except:
pass
parser.set("decoder", "options", decoder_options)
decoder_nbestout = open(decoder_nbestfile, "w")
if tuning_switch == "on":
Tuning_object = Tuning(parser)
Decoder_object = Decoder_Moses_nbest(parser, Tuning_object)
elif tuning_switch == "off":
Decoder_object = Decoder_Moses_nbest(parser)
if not showweightsflag == "":
decoder_out, decoder_err = Decoder_object.show_weights()
# write weights to stdout
sys.stdout.write("".join(decoder_out))
sys.stdout.flush()
sys.exit(1)
elif decoder_type == "Deterministic":
Decoder_object = Decoder_Deterministic(parser)
else:
from tuning import Tuning
import usb.core
import usb.util
import time
if __name__ == '__main__':
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
if dev:
Mic_tuning = Tuning(dev)
while True:
try:
if Mic_tuning.is_voice():
print("I detected voice activity at angle %d" % Mic_tuning.direction)
time.sleep(0.2)
except KeyboardInterrupt:
break
from tuning import Tuning
from pixel_ring import pixel_ring
import usb.core
import usb.util
import time
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
#print dev
if dev:
Mic_tuning = Tuning(dev)
# print Mic_tuning.is_voice(), Mic_tuning.direction
i = Mic_tuning.is_voice()
j = Mic_tuning.direction
while True:
Mic_tuning.set_vad_threshold(1.5)
# pixel_ring.set_brightness(0x00) #turn off LED to conserve power
try:
#if (Mic_tuning.is_voice == True): #Attempting to refine code to only show when Mic_tuning.is_voice is 1
print Mic_tuning.is_voice(
), Mic_tuning.direction #Displays values 0 for noise or 1 for
# human voice and Degrees
time.sleep(1)
except KeyboardInterrupt:
break