def __init__(self, robot_name="", wait_services=False):
self.robot_name = robot_name
self.tf_listener = tf_server.TFClient()
# Body parts
self.parts = dict()
self.parts['base'] = base.Base(self.robot_name, self.tf_listener)
self.parts['torso'] = torso.Torso(self.robot_name, self.tf_listener)
self.parts['leftArm'] = arms.Arm(self.robot_name,
self.tf_listener,
side="left")
self.parts['rightArm'] = arms.Arm(self.robot_name,
self.tf_listener,
side="right")
self.parts['head'] = head.Head(self.robot_name, self.tf_listener)
# Human Robot Interaction
self.parts['lights'] = lights.Lights(self.robot_name, self.tf_listener)
self.parts['speech'] = speech.Speech(
self.robot_name, self.tf_listener,
lambda: self.lights.set_color(1, 0, 0),
lambda: self.lights.set_color(0, 0, 1))
self.parts['hmi'] = api.Api(self.robot_name, self.tf_listener)
self.parts['ears'] = ears.Ears(self.robot_name,
lambda: self.lights.set_color(0, 1, 0),
lambda: self.lights.set_color(0, 0, 1))
self.parts['ebutton'] = ebutton.EButton(self.robot_name,
self.tf_listener)
# Reasoning/world modeling
self.parts['ed'] = world_model_ed.ED(self.robot_name, self.tf_listener)
# Miscellaneous
self.pub_target = rospy.Publisher("/target_location",
geometry_msgs.msg.Pose2D,
queue_size=10)
self.base_link_frame = "/" + self.robot_name + "/base_link"
# Grasp offsets
#TODO: Don't hardcode, load from parameter server to make robot independent.
self.grasp_offset = geometry_msgs.msg.Point(0.5, 0.2, 0.0)
# Create attributes from dict
for k, v in self.parts.iteritems():
setattr(self, k, v)
self.spindle = self.torso # (ToDo: kind of ugly, why do we still have spindle???)
self.arms = OrderedDict(
left=self.leftArm, right=self.rightArm
) # (ToDo: kind of ugly, why do we need this???)
self.ears._hmi = self.hmi # ToDo: when ears is gone, remove this line
# Wait for connections
s = rospy.Time.now()
for k, v in self.parts.iteritems():
v.wait_for_connections(1.0)
e = rospy.Time.now()
rospy.logdebug("Connecting took {} seconds".format((e - s).to_sec()))
def getCommand(self):
s = speech.Speech()
words = s.record().split()
isLED = False
isNews = False
isWeather = False
print words
if words[0] != 'client':
return
for word in words:
if word == 'news':
isNews = True
if word == 'weather':
isWeather = True
if word == 'led':
isLED = True
if isNews:
for word in words:
if word == 'on':
self.sendData('news on')
elif word == 'off':
self.sendData('news off')
if isWeather:
for word in words:
if word == 'on':
self.sendData('weather on')
elif word == 'off':
self.sendData('weather off')
if isLED == True:
for word in words:
if word == 'on':
for word in words:
if word == '1' or word == 'one':
self.sendData('LED 1 on')
elif word == '2' or word == 'two' or word == 'to' or word == 'too':
self.sendData('LED 2 on')
elif word == '3' or word == 'three':
self.sendData('LED 3 on')
elif word == '4' or word == 'four' or word == 'for' or word == 'fore':
self.sendData('LED 4 on')
elif word == '5' or word == 'five':
self.sendData('LED 5 on')
elif word == 'off':
self.sendData('LED off')
else:
return
def main(args):
listen_file = join(args.store, 'listen.wav')
listen = li.Listen()
brain = Brain(listen_file=listen_file)
speech = sp.Speech()
num_asleep = 0
while 1:
listen.record_to_file(listen_file)
response = brain.process()
if response:
speech.speak(response)
if not brain.is_awake():
num_asleep += 1
if num_asleep > 10:
time.sleep(60)
num_asleep = 0
def __init__(self, robot_name="", wait_services=False):
self.robot_name = robot_name
self.tf_listener = tf_server.TFClient()
# Body parts
self.base = base.Base(self.robot_name,
self.tf_listener,
wait_service=wait_services)
self.torso = torso.Torso(self.robot_name, wait_service=wait_services)
self.spindle = self.torso
self.leftArm = arms.Arm(self.robot_name, "left", self.tf_listener)
self.rightArm = arms.Arm(self.robot_name, "right", self.tf_listener)
self.arms = OrderedDict(left=self.leftArm, right=self.rightArm)
self.head = head.Head(self.robot_name)
# Human Robot Interaction
self.speech = speech.Speech(self.robot_name,
wait_service=wait_services)
self.ears = ears.Ears(self.robot_name)
self.ebutton = ebutton.EButton()
self.lights = lights.Lights()
# Perception: can we get rid of this???
self.perception = perception_ed.PerceptionED(
wait_service=wait_services)
# Reasoning/world modeling
self.ed = world_model_ed.ED(wait_service=wait_services)
self.reasoner = reasoner.Reasoner()
# Miscellaneous
self.pub_target = rospy.Publisher("/target_location",
geometry_msgs.msg.Pose2D,
queue_size=10)
self.base_link_frame = "/" + self.robot_name + "/base_link"
#Grasp offsets
#TODO: Don't hardcode, load from parameter server to make robot independent.
self.grasp_offset = geometry_msgs.msg.Point(0.5, 0.2, 0.0)
def __init__(self, robot_name="", wait_services=False):
self.robot_name = robot_name
self.tf_listener = tf.TransformListener()
# Body parts
self.parts = dict()
self.parts['base'] = base.Base(self.robot_name, self.tf_listener)
self.parts['torso'] = torso.Torso(self.robot_name, self.tf_listener)
self.parts['leftArm'] = arms.Arm(self.robot_name,
self.tf_listener,
side="left")
self.parts['rightArm'] = arms.Arm(self.robot_name,
self.tf_listener,
side="right")
self.parts['head'] = head.Head(self.robot_name, self.tf_listener)
self.parts['perception'] = perception.Perception(
self.robot_name, self.tf_listener)
self.parts['ssl'] = ssl.SSL(self.robot_name, self.tf_listener)
# Human Robot Interaction
self.parts['lights'] = lights.Lights(self.robot_name, self.tf_listener)
self.parts['speech'] = speech.Speech(
self.robot_name, self.tf_listener,
lambda: self.lights.set_color_colorRGBA(lights.SPEAKING),
lambda: self.lights.set_color_colorRGBA(lights.RESET))
self.parts['hmi'] = api.Api(
self.robot_name, self.tf_listener,
lambda: self.lights.set_color_colorRGBA(lights.LISTENING),
lambda: self.lights.set_color_colorRGBA(lights.RESET))
self.parts['ears'] = ears.Ears(
self.robot_name, self.tf_listener,
lambda: self.lights.set_color_colorRGBA(lights.LISTENING),
lambda: self.lights.set_color_colorRGBA(lights.RESET))
self.parts['ebutton'] = ebutton.EButton(self.robot_name,
self.tf_listener)
# Reasoning/world modeling
self.parts['ed'] = world_model_ed.ED(self.robot_name, self.tf_listener)
# Miscellaneous
self.pub_target = rospy.Publisher("/target_location",
geometry_msgs.msg.Pose2D,
queue_size=10)
self.base_link_frame = "/" + self.robot_name + "/base_link"
# Check hardware status
self._hardware_status_sub = rospy.Subscriber(
"/" + self.robot_name + "/hardware_status", DiagnosticArray,
self.handle_hardware_status)
# Grasp offsets
#TODO: Don't hardcode, load from parameter server to make robot independent.
self.grasp_offset = geometry_msgs.msg.Point(0.5, 0.2, 0.0)
# Create attributes from dict
for partname, bodypart in self.parts.iteritems():
setattr(self, partname, bodypart)
self.arms = OrderedDict(
left=self.leftArm, right=self.rightArm
) # (ToDo: kind of ugly, why do we need this???)
self.ears._hmi = self.hmi # ToDo: when ears is gone, remove this line
# Wait for connections
s = rospy.Time.now()
for partname, bodypart in self.parts.iteritems():
bodypart.wait_for_connections(1.0)
e = rospy.Time.now()
rospy.logdebug("Connecting took {} seconds".format((e - s).to_sec()))
if not self.operational:
not_operational_parts = [
name for name, part in self.parts.iteritems()
if not part.operational
]
rospy.logwarn("Not all hardware operational: {parts}".format(
parts=not_operational_parts))
import speech as sp
language="english"
conf = sp.Speech("speech_conf.json", language, writedebug=False)
KEYWORDS = "KEYWORDS"
AFFIRMATIVE = "affirmative"
NEGATIVE = "negative"
OPTIONS = "options"
STOP="stop"
REPEAT="repeat"
STOP_INTERACTION = "STOP-INTERACTION"
reader = open('speechLog.log','r')
writer = open('speechLogproc.log', 'w')
lines=[]
for line in reader.readlines():
lines.append(line)
for index, line in enumerate(lines):
if "[ ] USER >" in line:
writer.write(line.replace("[ ]","[FROZEN]"))
else:
writer.write(line)
if sp.TAGS[2] in line or sp.TAGS[3] in line or sp.TAGS[4] in line or sp.TAGS[11] in line:
if sp.TAGS[2] in line:
usertag="AACo"
elif sp.TAGS[3] in line:
usertag="AAEv"
elif sp.TAGS[11] in line:
usertag="AALo"
def handle_collision(self, other):
if not self.collideable:
return
self.gravity = 600
if self.collides_with(other) and other.type == "Solid":
# print self.x, self.y, " >>> ", other.x / 32, other.y / 32
dx = int(other.width + self.width)/2 - 2
dy = int(other.height + self.height)/2 - 2
if (other.x - dx < self.x < other.x + dx):
if (self.y > other.y):
self.gravity = 0;
self.velY = max(self.velY, 0)
self.y = int(other.y + other.height / 2 + self.height / 2)
self.jump = True
#print "top"
elif (self.y < other.y):
self.velY = min(self.velY, 0)
self.y = int(other.y - other.height / 2 - self.height / 2)
#print "bottom"
elif (other.y - dy < self.y < other.y + dy):
if (other.x > self.x):
self.velX = min(0, self.velX)
elif (other.x < self.x):
self.velX = max(0, self.velX)
#print "side"
elif self.collides_with(other) and other.type == "JumpThru":
dx = int(other.width + self.width)/2 - 2
dy = int(other.height + self.height)/2 - 2
if (other.x - dx < self.x < other.x + dx):
if (self.y > other.y + dy / 2 and self.velY < 0):
self.velY = max(self.velY, 0)
self.gravity = 0
self.y = int(other.y + other.height / 2 + self.height / 2)
self.jump = True
elif (self.y >= other.y + dy):
self.velY = max(self.velY, 0)
self.gravity = 0
self.y = int(other.y + other.height / 2 + self.height / 2)
self.jump = True
elif self.collides_with(other) and other.type == "Subghost" and other not in self.subghosts and self.type == "Player":
#other.cleanup = True
print "hey"
self.subghosts.append(other)
other.collected = True
#hazard - lose ghost if collide.
elif self.collides_with(other) and other.type == "Hazard":
if len(self.subghosts) > 0:
self.speechtimer = 2.0
self.bubble = speech.Speech("Could you come on, man?", self.x, self.y)
sg = self.subghosts.pop()
sg.collected = False
self.animation = 0
elif self.collides_with(other) and other.type == "Spirit" and self.type == "Player":
if len(self.subghosts) >= other.rank:
for i in range(len(self.subghosts)):
self.subghosts[i].set_circle(5.0 + 0.3 * (i + 1), other.x, other.y, other.width * 2)
else:
self.animation = 0
self.speechtimer = 2.0
self.bubble = speech.Speech("Could you cool it?", self.x, self.y)
if len(self.subghosts) > 0:
sg = self.subghosts.pop()
sg.collected = False
class Player(gameobject.GameObject):
type = "Player"
gravity = 600
speed = 200
jumpstrength = 300
right = False
left = False
frames = []
frame = 0
animation = 2
velX = 0
velY = 0
jump = False
subghosts = []
speechtimer = 0.0
bubble = speech.Speech("", 0, 0)
c = (-0.7, -0.384)
def draw(self):
#tx = self.sprite.image.get_texture()
self.sprite.x = int(self.x)
self.sprite.y = int(self.y)
#self.shader.bind()
#self.shader.uniformf('C', *self.c)
self.sprite.draw()
#tx.blit(self.x, self.y)
#self.shader.unbind()
if (self.speechtimer > 0):
self.bubble.draw()
# if len(self.subghosts) > 0:
# for i in range(len(self.subghosts)):
# self.subghosts[i].draw()
def do_jump(self):
if self.jump:
self.velY = self.jumpstrength
for i in range(len(self.subghosts)):
self.subghosts[i].jumptimer = 0.3 * (i + 1)
def update(self, dt):
for col in self.collided:
col -= 1
self.jump = False
if self.right:
self.animation = 2
self.velX = self.speed
elif self.left:
self.animation = 1
self.velX = -self.speed
else:
if self.velX > 10:
self.velX *= 7/8
elif self.velX < -10:
self.velX *= 7/8
else:
self.velX = 0
if abs(self.velX) > 10:
self.frame += 1
self.sprite = pyglet.sprite.Sprite(self.frames[self.animation * 2 + int(self.frame / 10) % 2], x=self.x, y=self.y)
else:
self.frame = 0
self.sprite = pyglet.sprite.Sprite(self.frames[self.animation * 2 + int(self.frame / 10) % 2], x=self.x, y=self.y)
if len(self.subghosts) > 0 and self.type == "Player":
if self.right:
self.subghosts[0].destX = self.x - 32
elif self.left:
self.subghosts[0].destX = self.x + 32
for i in range(1, len(self.subghosts)):
if self.right:
self.subghosts[i].destX = self.subghosts[i - 1].x - 32
elif self.left:
self.subghosts[i].destX = self.subghosts[i - 1].x + 32
if self.speechtimer > 0:
self.speechtimer -= dt
self.bubble.x = self.x
self.bubble.y = self.y
self.bubble.update(dt)
self.bubble.opacity = min(255, int(255 * (self.speechtimer / 1.0)))
def handle_collision(self, other):
if not self.collideable:
return
self.gravity = 600
if self.collides_with(other) and other.type == "Solid":
# print self.x, self.y, " >>> ", other.x / 32, other.y / 32
dx = int(other.width + self.width)/2 - 2
dy = int(other.height + self.height)/2 - 2
if (other.x - dx < self.x < other.x + dx):
if (self.y > other.y):
self.gravity = 0;
self.velY = max(self.velY, 0)
self.y = int(other.y + other.height / 2 + self.height / 2)
self.jump = True
#print "top"
elif (self.y < other.y):
self.velY = min(self.velY, 0)
self.y = int(other.y - other.height / 2 - self.height / 2)
#print "bottom"
elif (other.y - dy < self.y < other.y + dy):
if (other.x > self.x):
self.velX = min(0, self.velX)
elif (other.x < self.x):
self.velX = max(0, self.velX)
#print "side"
elif self.collides_with(other) and other.type == "JumpThru":
dx = int(other.width + self.width)/2 - 2
dy = int(other.height + self.height)/2 - 2
if (other.x - dx < self.x < other.x + dx):
if (self.y > other.y + dy / 2 and self.velY < 0):
self.velY = max(self.velY, 0)
self.gravity = 0
self.y = int(other.y + other.height / 2 + self.height / 2)
self.jump = True
elif (self.y >= other.y + dy):
self.velY = max(self.velY, 0)
self.gravity = 0
self.y = int(other.y + other.height / 2 + self.height / 2)
self.jump = True
elif self.collides_with(other) and other.type == "Subghost" and other not in self.subghosts and self.type == "Player":
#other.cleanup = True
print "hey"
self.subghosts.append(other)
other.collected = True
#hazard - lose ghost if collide.
elif self.collides_with(other) and other.type == "Hazard":
if len(self.subghosts) > 0:
self.speechtimer = 2.0
self.bubble = speech.Speech("Could you come on, man?", self.x, self.y)
sg = self.subghosts.pop()
sg.collected = False
self.animation = 0
elif self.collides_with(other) and other.type == "Spirit" and self.type == "Player":
if len(self.subghosts) >= other.rank:
for i in range(len(self.subghosts)):
self.subghosts[i].set_circle(5.0 + 0.3 * (i + 1), other.x, other.y, other.width * 2)
else:
self.animation = 0
self.speechtimer = 2.0
self.bubble = speech.Speech("Could you cool it?", self.x, self.y)
if len(self.subghosts) > 0:
sg = self.subghosts.pop()
sg.collected = False
# print f
#speeches = [s for s in speeches if s.alignment.alignments is not None]
#for i in tqdm(range(len(speeches))):
# print "Processing speech %d of %d" % (i, len(speeches))
# speech = speeches[i]
# if os.path.exists(speech.alignment_file):
# speech.load_librosa()
# output_path = speech.audio_file.replace('/audio/','/phrase_audio_features/').replace('.mp3','.pkl')
# if not os.path.exists(output_path):
# print output_path
# features = get_audio_features_by_phrase(speech)
# with open(output_path,'wb') as f:
# pickle.dump(features, f)
paths = open('/home/jrgillick/Applause/missing_phrase_audio_paths.txt'
).read().split('\n')[0:-1][start_index:end_index]
for p in paths:
suffix = p.split('/')[-1].split('.')[0]
print suffix
s = speech.Speech(suffix)
if os.path.exists(s.alignment_file):
s.load_librosa()
output_path = s.audio_file.replace(
'/audio/', '/phrase_audio_features/').replace('.mp3', '.pkl')
if not os.path.exists(output_path):
print output_path
features = get_audio_features_by_phrase(s)
with open(output_path, 'wb') as f:
pickle.dump(features, f)
"""
Primary run loop and logic
"""
import alwaysai_helper as aai
import motor as m
import speech
import os
# Bounding box area threshold to distinguish between
# a close and far person
CLOSE_THRESHOLD = 50000
SEEN_FAR_IDS = {}
SEEN_NEAR_IDS = {}
AUDIO_PLAYER = speech.Speech(os.path.join(os.getcwd(), 'bin', 'audio'),
"vomit_candy.wav", "come_closer.wav")
def main():
global SEEN_FAR_IDS
global SEEN_NEAR_IDS
global AUDIO_PLAYER
# Load configuration set in alwaysai.app.json so we don't have to
# edit code to redeploy with every change
config = aai.loadJSON('alwaysai.app.json')['tracker_1']
# Components contains references to aai element such as the
# object tracker, centroid tracker, streamer, fps, etc.
components = aai.get_components(config)
# object_detector = aai.object_detector_from(config)
import sys, os, librosa, pickle
sys.path.insert(0, '../core')
import alignment, speech, applause_list, file_loader, text_features,re
import librosa, numpy as np
from tqdm import tqdm
all_files = file_loader.get_non_duplicate_file_endings()
if __name__ == '__main__':
start_index = int(sys.argv[1])
end_index = int(sys.argv[2])
print "Loading speeches %d to %d" % (start_index, end_index)
speeches = [speech.Speech(f) for f in tqdm(all_files[start_index:end_index])]
print
for i in tqdm(range(len(speeches))):
print "Processing speech %d of %d" % (i, len(speeches))
speech = speeches[i]
crowd_audio_path = speech.get_crowd_audio_path()
crowd_rmse_path = '/'.join(crowd_audio_path.split('/')[0:-1]).replace('audio_separated','crowd_rmse') + '.pkl'
if not os.path.exists(crowd_rmse_path):
print crowd_rmse_path
crowd_y,sr = librosa.load(crowd_audio_path)
with open(crowd_rmse_path,'wb') as f:
pickle.dump(librosa.feature.rmse(crowd_y), f)
def __init__(self):
super(PiServer, self).__init__(address='192.168.0.16')
self.timeout = 2.0
self.voice = speech.Speech()
def main():
# Setup all modules
vision = v.Vision()
speech = s.Speech()
motor = m.Motor()
motion_detector = md.Motion_Detector()