async def run(robot: cozmo.robot.Robot):
'''The core of the braitenberg machine program'''
# Move lift down and tilt the head up
await robot.set_head_angle(degrees(0)).wait_for_completed()
await robot.set_lift_height(0.0).wait_for_completed()
await robot.say_text('Game is on').wait_for_completed()
print("Press CTRL-C to quit")
camera = robot.camera
fixed_gain = camera.gain
fixed_exposure_ms = 20
camera.set_manual_exposure(fixed_exposure_ms, fixed_gain)
await robot.say_text('Start Learning').wait_for_completed()
img_clf = ImageClassifier()
# load images
(train_raw, train_labels) = img_clf.load_data_from_folder('./train/')
# convert images into features
train_data = img_clf.extract_image_features(train_raw)
# train model and test on training data
img_clf.train_classifier(train_data, train_labels)
await robot.say_text("Let's do this").wait_for_completed()
cumulation_predicted_labels = Counter()
nothing_count = 0
while True:
# get camera image
event = await robot.world.wait_for(cozmo.camera.EvtNewRawCameraImage,
timeout=30)
# convert images into features
test_data = img_clf.extract_image_features([np.asarray(event.image)])
[predicted_label] = img_clf.predict_labels(test_data)
if predicted_label != 'none':
nothing_count = 0
cumulation_predicted_labels[predicted_label] += 1
else:
nothing_count += 1
if nothing_count > 50:
nothing_count = 0
cumulation_predicted_labels.clear()
print(str(cumulation_predicted_labels))
if sum(cumulation_predicted_labels.values()) >= 10:
if cumulation_predicted_labels.most_common()[0][1] > 5:
label = cumulation_predicted_labels.most_common()[0][0]
await robot.say_text(label).wait_for_completed()
await robot.play_anim_trigger(
cozmo.anim.Triggers.AcknowledgeObject
).wait_for_completed()
nothing_count = 0
cumulation_predicted_labels.clear()
else:
nothing_count = 0
cumulation_predicted_labels.clear()
time.sleep(.1)
def run(self, sdk_conn):
robot = sdk_conn.wait_for_robot()
robot.camera.image_stream_enabled = True
robot.camera.color_image_enabled = False
robot.camera.enable_auto_exposure()
robot.set_head_angle(cozmo.util.degrees(0)).wait_for_completed()
if self.observed_symbol is None:
while True:
time.sleep(0.5)
# Keep observing image
latest_image = robot.world.latest_image
new_image = latest_image.raw_image
classifier = load(
'clf.joblib') # needs file that is too big for github
img_features = ImageClassifier.extract_image_features(
classifier, [new_image])
self.label = classifier.predict(img_features)[0]
robot.say_text(self.label).wait_for_completed()
timestamp = datetime.datetime.now().strftime("%dT%H%M%S%f")
# new_image.save("./imgs/" + str(self.label) + "_" + timestamp + ".bmp")
next = self.next_state()
if not next is None:
return next
return None
def idle(robot: cozmo.robot.Robot):
# Look out for the secret symbols by monitoring the stream of images from the camera.
# Classify each symbol (image) using the model you developed in Lab1. If one of the symbols is
# recognized (i.e. not “none”), use the built-in text-to-speech functionality to have the robot say the
# name of the recognized symbol, then switch to the appropriate state
robot.set_head_angle(degrees(0)).wait_for_completed()
IC.classifier = joblib.load('trained_model.pkl')
images = []
while len(images) < 3:
if len(images) == 2:
robot.turn_in_place(
angle=cozmo.util.Angle(degrees=5),
speed=cozmo.util.Angle(degrees=5)).wait_for_completed()
else:
robot.turn_in_place(
angle=cozmo.util.Angle(degrees=5),
speed=cozmo.util.Angle(degrees=5)).wait_for_completed()
images.append(numpy.asarray(robot.world.latest_image.raw_image))
labels = IC.predict_labels(
IC, IC.extract_image_features(IC.classifier, images))
label = max(set(labels), key=list(labels).count)
FSM(robot, label)
async def classifyImage(robot: cozmo.robot.Robot):
img_clf = ImageClassifier()
# load images
(train_raw, train_labels) = img_clf.load_data_from_folder('./train/')
# convert images into features
train_data = img_clf.extract_image_features(train_raw)
# train model and test on training data
img_clf.train_classifier(train_data, train_labels)
robot.world.image_annotator.add_annotator('image', ImageAnnotator)
await robot.set_head_angle(degrees(0)).wait_for_completed()
print("Press CTRL-C to quit")
try:
while True:
robot.move_lift(-1)
#get camera image
event = await robot.world.wait_for(
cozmo.camera.EvtNewRawCameraImage, timeout=30)
#convert camera image to opencv format
opencv_image = cv2.cvtColor(np.asarray(event.image),
cv2.COLOR_RGB2GRAY)
ImageAnnotator.image = opencv_image
opencv_images = []
opencv_images.append(opencv_image)
image_data = img_clf.extract_image_features(opencv_images)
image_name = img_clf.predict_labels(image_data)[0]
print(image_name)
await robot.say_text(image_name).wait_for_completed()
robot.move_lift(10)
time.sleep(3)
except KeyboardInterrupt:
print("")
print("Exit requested by user")
except cozmo.RobotBusy as e:
print(e)
def getTrainedModel():
img_clf = ImageClassifier()
# load images
(train_raw, train_labels) = img_clf.load_data_from_folder('./train/')
(test_raw, test_labels) = img_clf.load_data_from_folder('./test/')
# convert images into features
train_data = img_clf.extract_image_features(train_raw)
# train model and test on training data
img_clf.train_classifier(train_data, train_labels)
return img_clf
async def cozmoCardClassifier(robot: cozmo.robot.Robot):
''' Cozmo Card classifier implementation - gathers 10 images then runs a prediction on them to see if it saw a familiar card '''
robot.move_lift(-3)
robot.set_head_angle(cozmo.util.degrees(0)).wait_for_completed()
model_file = Path(ModelFileName)
if model_file.is_file():
with open(ModelFileName, 'rb') as input:
imgClassifier = pickle.load(input)
else:
with open(ModelFileName, 'wb') as output:
imgClassifier = ImageClassifier()
imgClassifier.train_model()
pickle.dump(imgClassifier, output, pickle.HIGHEST_PROTOCOL)
while True:
images = []
for iterN in range(10):
#get camera image
event = await robot.world.wait_for(cozmo.camera.EvtNewRawCameraImage, timeout=30)
#convert camera image to opencv format
opencv_image = cv2.cvtColor(np.asarray(event.image), cv2.COLOR_RGB2GRAY)
images.append(opencv_image)
predictions = imgClassifier.predict_labels(imgClassifier.extract_image_features(images))
print("predictions")
print(predictions)
(mostLikelyCardType, mostLikelyCardFrequency) = max([(key,len(list(group))) for key, group in groupby(predictions)], key=lambda x: x[1])
print("likely going to be %s" % mostLikelyCardType)
if mostLikelyCardType in ['drone', 'hands', 'inspection'] and mostLikelyCardFrequency > 5:
textToSay = "I think I see '%s'" % mostLikelyCardType
await robot.say_text(textToSay, voice_pitch=0.5, duration_scalar=0.6).wait_for_completed()
if mostLikelyCardType == 'drone':
await robot.play_anim(name="anim_reacttoface_unidentified_02").wait_for_completed()
elif mostLikelyCardType == 'hands':
await robot.play_anim(name="anim_speedtap_wingame_intensity02_01").wait_for_completed()
elif mostLikelyCardType == 'inspection':
await robot.play_anim(name="anim_poked_giggle").wait_for_completed()
import sys
import cozmo
import datetime
import time
import numpy as np
from imgclassification import ImageClassifier
img_clf = ImageClassifier()
def train():
# load images
(train_raw, train_labels) = img_clf.load_data_from_folder('./train/')
# convert images into features
train_data = img_clf.extract_image_features(train_raw)
# train model and test on training data
img_clf.train_classifier(train_data, train_labels)
def createImageClassifier():
train()
return img_clf
async def run(robot: cozmo.robot.Robot):
'''The run method runs once the Cozmo SDK is connected.'''
img_clf = ImageClassifier()
# load images
(train_raw, train_labels) = img_clf.load_data_from_folder('./train/')
# convert images into features
train_data = img_clf.extract_image_features(train_raw)
# train model and test on training data
img_clf.train_classifier(train_data, train_labels)
try:
last_10 = Queue(maxsize=10)
counts = {}
while True:
# get camera image
event = await robot.world.wait_for(
cozmo.camera.EvtNewRawCameraImage)
image = color.rgb2gray(np.asarray(event.image))
# predict image
predicted = img_clf.predict_labels(np.expand_dims(image,
axis=0))[0]
# update last 10 seen
if not last_10.empty():
removed = last_10.get()
counts[removed] = counts[removed] - 1
# add last seen
last_10.put(predicted)
if predicted in counts:
counts[predicted] = counts[predicted] + 1
# predicted was guessed 8 out of last 10 times
if counts[predicted] >= 8:
if predicted == 'plane':
await robot.say_text(predicted).wait_for_completed()
await robot.play_anim_trigger(
cozmo.anim.Triggers.CodeLabDog
).wait_for_completed()
elif predicted == 'hands':
await robot.say_text(predicted).wait_for_completed()
await robot.play_anim_trigger(
cozmo.anim.Triggers.CodeLabFireTruck
).wait_for_completed()
elif predicted == 'place':
await robot.say_text(predicted).wait_for_completed()
await robot.play_anim_trigger(
cozmo.anim.Triggers.CodeLabFrustrated
).wait_for_completed()
# reset last 10
last_10 = Queue(maxsize=10)
counts = {}
time.sleep(5)
else:
counts[predicted] = 1
except KeyboardInterrupt:
print("")
print("Exit requested by user")
except cozmo.RobotBusy as e:
print(e)
def run(sdk_conn):
img_clf = ImageClassifier()
# load images
(train_raw, train_labels) = img_clf.load_data_from_folder('./train/')
# convert images into features
train_data = img_clf.extract_image_features(train_raw)
# train model and test on training data
img_clf.train_classifier(train_data, train_labels)
robot = sdk_conn.wait_for_robot()
robot.camera.image_stream_enabled = True
robot.camera.color_image_enabled = False
robot.camera.enable_auto_exposure()
robot.set_head_angle(cozmo.util.degrees(0)).wait_for_completed()
robot.say_text('Ready').wait_for_completed()
IDLE = 'idle'
DRONE = 'drone'
ORDER = 'order'
INSPECTION = 'inspection'
NONE = 'none'
current_state = IDLE
while True:
if current_state == IDLE:
images = []
while len(images) < 4:
if len(images) % 2 == 0:
robot.turn_in_place(angle=cozmo.util.Angle(degrees=10),
speed=cozmo.util.Angle(
degrees=10)).wait_for_completed()
else:
robot.turn_in_place(angle=cozmo.util.Angle(degrees=-10),
speed=cozmo.util.Angle(
degrees=10)).wait_for_completed()
latest_image = robot.world.latest_image
new_image = latest_image.raw_image
new_image = numpy.asarray(new_image)
images.append(new_image)
features = img_clf.extract_image_features(images)
predicted_labels = img_clf.predict_labels(features)
predicted_dict = dict()
for x in predicted_labels:
if x in predicted_dict:
predicted_dict[x] += 1
else:
predicted_dict[x] = 1
max_count = 0
max_prediction = None
for x in predicted_dict:
if predicted_dict[x] > max_count:
max_prediction = x
max_count = predicted_dict[x]
if max_prediction == DRONE:
robot.say_text(max_prediction).wait_for_completed()
current_state = DRONE
elif max_prediction == ORDER:
robot.say_text(max_prediction).wait_for_completed()
current_state = ORDER
elif max_prediction == INSPECTION:
robot.say_text(max_prediction).wait_for_completed()
current_state = INSPECTION
elif current_state == DRONE:
robot.turn_in_place(
angle=cozmo.util.Angle(degrees=90),
speed=cozmo.util.Angle(degrees=90)).wait_for_completed()
cube = robot.world.wait_for_observed_light_cube()
robot.pickup_object(cube, num_retries=5).wait_for_completed()
robot.drive_straight(
cozmo.util.Distance(distance_mm=100),
cozmo.util.Speed(speed_mmps=50)).wait_for_completed()
robot.place_object_on_ground_here(cube).wait_for_completed()
robot.drive_straight(
cozmo.util.Distance(distance_mm=-100),
cozmo.util.Speed(speed_mmps=50)).wait_for_completed()
current_state = IDLE
elif current_state == ORDER:
robot.drive_wheels(l_wheel_speed=20, r_wheel_speed=40, duration=30)
current_state = IDLE
elif current_state == INSPECTION:
robot.turn_in_place(
angle=cozmo.util.Angle(degrees=90),
speed=cozmo.util.Angle(degrees=90)).wait_for_completed()
for i in range(4):
lift = robot.set_lift_height(height=1,
max_speed=0.5,
in_parallel=True)
straight = robot.drive_straight(
distance=cozmo.util.Distance(distance_mm=200),
speed=cozmo.util.Speed(speed_mmps=50),
in_parallel=True)
lift.wait_for_completed()
lower = robot.set_lift_height(height=0,
max_speed=0.5,
in_parallel=True)
straight.wait_for_completed()
lower.wait_for_completed()
robot.turn_in_place(
angle=cozmo.util.Angle(degrees=90),
speed=cozmo.util.Angle(degrees=90)).wait_for_completed()
robot.set_lift_height(height=0,
max_speed=0.5,
accel=0,
in_parallel=True).wait_for_completed()
current_state = IDLE
async def run(robot: cozmo.robot.Robot):
'''The run method runs once the Cozmo SDK is connected.'''
# Get image classifier
img_clf = ImageClassifier()
# Model was previously trained and saved in imgRecognition.sav
# print("Started training algorithm")
# classifierModel = img_clf.build_and_return_classifier()
# print("Finished training algorithm")
# print("Saving model")
# pickle.dump(classifierModel, open("imgRecognition.sav", 'wb'))
# print("Finished saving model")
# Reading model previously saved
savedClassifier = pickle.load(open("imgRecognition.sav", 'rb'))
print("Putting Cozmo in a idle position")
await robot.set_head_angle(degrees(0)).wait_for_completed()
robot.move_lift(-3)
print("Finished setting Cozmo")
try:
while True:
symbolCount = {
"drone" : 0,
"hands" : 0,
"inspection" : 0,
"order" : 0,
"place" : 0,
"plane" : 0,
"truck" : 0,
"none" : 0,
}
for i in range(10):
print("Getting cozmo's image")
#get camera image
event = await robot.world.wait_for(cozmo.camera.EvtNewRawCameraImage, timeout=30)
print("Image received")
#convert camera image to opencv format
opencv_image = cv2.cvtColor(np.asarray(event.image), cv2.COLOR_RGB2GRAY)
print("Converted to opencv format")
#Find the symbol
imageInstance = img_clf.extract_image_features([opencv_image])
print("Generated instance by extracting features")
symbolsList = savedClassifier.predict(imageInstance)
print("Predicted label:", symbolsList[0])
symbol = symbolsList[0]
symbolCount[symbol] = symbolCount[symbol] + 1
print("symbolCount:", symbolCount)
finalSymbol = max(symbolCount.items(), key=operator.itemgetter(1))[0]
# Make cozmo tell what he's seeing
await robot.say_text(finalSymbol).wait_for_completed()
if finalSymbol == "drone":
await robot.play_anim(name="anim_bored_01").wait_for_completed()
elif finalSymbol == "hands":
await robot.play_anim(name="anim_poked_giggle").wait_for_completed()
elif finalSymbol == "inspection":
await robot.play_anim(name="anim_pounce_success_02").wait_for_completed()
elif finalSymbol == "order":
await robot.play_anim(name="anim_bored_event_02").wait_for_completed()
elif finalSymbol == "place":
await robot.play_anim(name="anim_bored_event_03").wait_for_completed()
elif finalSymbol == "plane":
await robot.play_anim(name="anim_petdetection_cat_01").wait_for_completed()
elif finalSymbol == "truck":
await robot.play_anim(name="anim_petdetection_dog_03").wait_for_completed()
elif finalSymbol == "none":
await robot.play_anim(name="anim_reacttoface_unidentified_02").wait_for_completed()
print("Putting Cozmo in a idle position")
await robot.set_head_angle(degrees(0)).wait_for_completed()
robot.move_lift(-3)
print("Finished setting Cozmo")
time.sleep(5)
except KeyboardInterrupt:
print("")
print("Exit requested by user")
except cozmo.RobotBusy as e:
print("COZMO IS BUSY")
print(e)
import cv2
import numpy as np
sys.path.insert(0, '../lab11')
from imgclassification import ImageClassifier
import time
import cozmo
from cozmo.util import degrees
from collections import Counter
try:
from PIL import ImageDraw, ImageFont
except ImportError:
sys.exit('run `pip3 install --user Pillow numpy` to run this example')
img_clf = ImageClassifier()
#Define a decorator as a subclass of Annotator; displays battery voltage
class BatteryAnnotator(cozmo.annotate.Annotator):
def apply(self, image, scale):
d = ImageDraw.Draw(image)
bounds = (0, 0, image.width, image.height)
batt = self.world.robot.battery_voltage
text = cozmo.annotate.ImageText('BATT %.1fv' % batt, color='green')
text.render(d, bounds)
async def run(robot: cozmo.robot.Robot):
robot.world.image_annotator.add_annotator('battery', BatteryAnnotator)