def getNLP_LfD_LandmarkDisplacement(nlp_grounder, user_id, exp_id, n_demos):
relation_offset = 11 #hard coded offset to get back to relations in range 0:8
relation, object_info = nlp_grounder.predict_goal(user_id, exp_id, n_demos)
print(len(relation), len(object_info))
print(relation)
#get lfd to use along with language or in place of language if ambiguous
lfd_shape_color, lfd_displacement = lfd.getMostLikelyLandmarkDisplacement(
user_id, exp_id, n_demos)
#check if we have one relation and one object
if len(relation) == 1 and len(object_info) == 1:
#print "grounding both"
#first check if nlp grounded landmark is actually a possible landmark
nlp_shape_color = (object_info[0][3], object_info[0][4])
yaml_file = lfd.getYamlFile(user_id, exp_id, 0)
landmark_coord = lfd.getFeatureCoordinates(nlp_shape_color, yaml_file)
if landmark_coord is None:
#print "NLP returned invalid landmark"
return lfd_shape_color, lfd_displacement
else: #use valid landmark and displacement to guess placement location
#get shape,color
nlp_shape_color = (object_info[0][3], object_info[0][4])
#get displacement from learned gmm
gmm_filename = "../data/gmm_params.yaml"
model_data = lfd.getYamlData(gmm_filename)
nlp_displacement = model_data['mu'][relation[0] - relation_offset]
return nlp_shape_color, nlp_displacement
#check if we have only one object and can ground on that
elif len(object_info) == 1:
#print "grounding on object only"
nlp_shape_color = (object_info[0][3], object_info[0][4])
nlp_displacement = lfd.getDisplacementFromLandmark(
user_id, exp_id, n_demos, nlp_shape_color)
return nlp_shape_color, nlp_displacement
#if we have one relationship and multiple items, figure out which item by using relationship
elif len(relation) == 1 and len(object_info) > 0:
#print "grounding on relation only"
#get predicted displacement based on relationship
gmm_filename = "../data/gmm_params.yaml"
model_data = lfd.getYamlData(gmm_filename)
nlp_displacement = model_data['mu'][relation[0] - relation_offset]
#get placement data and object locations
#TODO which demo should I pick from, what should n_demos be??
yaml_file = lfd.getYamlFile(user_id, exp_id, n_demos)
placement_coord = lfd.getPlacementCoordinates(yaml_file, user_id,
exp_id)
min_dist = 10000
for obj in object_info:
shape_color = (obj[3], obj[4])
landmark_coord = lfd.getFeatureCoordinates(shape_color, yaml_file)
predicted_placement = np.array(landmark_coord) + np.array(
nlp_displacement)
placement_error = np.linalg.norm(predicted_placement -
placement_coord)
if placement_error < min_dist:
min_dist = placement_error
best_landmark = shape_color
return best_landmark, nlp_displacement
#TODO case where only relationship is grounded?
#elif len(relation) == 1:
#fall back on pure lfd as a last resort
else:
#print "ambiguous grounding"
return lfd_shape_color, lfd_displacement
def main():
warnings.filterwarnings("ignore")
#set up train and test
#train_data = [(u,e) for u in range(30) for e in range(1,4)]
#test_data = [(u,e) for u in range(30) for e in range(4,5)]
#train_data = [(u,e) for u in range(25) for e in range(5)]
#test_data = [(u,e) for u in range(25,30) for e in range(5)]
data = []
sub_user = []
for u in range(30):
for e in range(5):
if(u%2 == 1 or e != 0):
sub_user.append((u,e))
if(u%2 == 1):
data.append(sub_user)
sub_user = []
total_demos = 10 #total number of demos possible
max_demos = 4 #maximum number of demos given to robot to learn from
thresh = 125
######################
#training code here
######################
num_folds = 10#len(data)
#kf = KFold(n=len(data),n_folds=num_folds)
kf = KFold(n_splits=num_folds)
#for all folds
all_lfd_errors = []
all_nlp_errors = []
count = 0
for train,test in kf.split(data):#kf:
count = count + 1
print("Cross validation round " + str(count))
train_data =[]
test_data = []
for i in train:
train_data = train_data + data[i]
for i in test:
test_data = test_data + data[i]
#TODO learn the gmm componenets and label the data
print( "--learning gmm components--")
gmm.labelTrainingDataDisplacements(train_data, total_demos)
#learn groundings with training data
print( "--grounding language--")
nlp_grounder = nlp()
nlp_grounder.train(train_data)
######################
#testing code
######################
print( "--testing generalization error--")
#matrix of zeros where each row is new test case and cols are ave errors for learning from 1:max_demos demonstrations
lfd_errors = np.zeros((len(test_data), max_demos))
nlp_errors = np.zeros((len(test_data), max_demos))
for i in range(len(test_data)):
user_id, exp_id = test_data[i]
for n_demos in range(1,max_demos+1):
#get best guess of landmark and displacement using pure LfD
lfd_shape_color, lfd_displacement = lfd.getMostLikelyLandmarkDisplacement(user_id, exp_id, n_demos)
#guess landmark and displacement using pure NLP
nlp_shape_color, nlp_displacement = getNLP_LfD_LandmarkDisplacementDoubleCheck(nlp_grounder, user_id, exp_id, n_demos, thresh)
#print nlp_shape_color, nlp_displacement
#compute accuracy over a test demo specified by demo_id
for demo_id in range(max_demos, total_demos):
lfd_errors[i, n_demos-1] = averageDistanceError(lfd_shape_color, lfd_displacement, user_id, exp_id, max_demos, total_demos)
nlp_errors[i, n_demos-1] = averageDistanceError(nlp_shape_color, nlp_displacement, user_id, exp_id, max_demos, total_demos)
#TODO add random
#TODO add average baseline
all_lfd_errors.append(lfd_errors)
all_nlp_errors.append(nlp_errors)
###################
#plot errors
###################
lfd_errors = np.zeros((len(test_data), max_demos))
nlp_errors = np.zeros((len(test_data), max_demos))
for validation_error in all_lfd_errors:
lfd_errors = np.add(lfd_errors, validation_error)
lfd_errors = np.divide(lfd_errors,num_folds)
for validation_error in all_nlp_errors:
nlp_errors = np.add(nlp_errors, validation_error)
nlp_errors = np.divide(nlp_errors,num_folds)
print( "lfd")
print( lfd_errors)
print( "nlp")
print( nlp_errors)
yerr = 10
plt.figure()
plt.errorbar(range(1,max_demos+1),np.mean(lfd_errors,0), yerr=np.std(lfd_errors,0), fmt='bo-', label='lfd')
plt.errorbar(range(1,max_demos+1),np.mean(nlp_errors,0), yerr=np.std(nlp_errors,0), fmt='go--', label='nlp+lfd')
plt.xticks(range(1,max_demos+1))
plt.xlabel('number of demonstrations')
plt.ylabel('generalization L2 error')
plt.legend(loc='best')
plt.show()
def getNLP_LfD_LandmarkDisplacementDoubleCheck(nlp_grounder, user_id, exp_id,
n_demos, thresh):
relation_offset = 11 #hard coded offset to get back to relations in range 0:8
relation, object_info = nlp_grounder.predict_goal(user_id, exp_id, n_demos)
#print len(relation), len(object_info)
#print relation
#get lfd to use along with language or in place of language if ambiguous
lfd_shape_color, lfd_displacement = lfd.getMostLikelyLandmarkDisplacement(
user_id, exp_id, n_demos)
#check if we have one relation and one object
grounded_prediction = False #flag to check if we think we've found a grounding
if len(relation) == 1 and len(object_info) == 1:
#print "grounding both"
#first check if nlp grounded landmark is actually a possible landmark
nlp_shape_color = (object_info[0][3], object_info[0][4])
yaml_file = lfd.getYamlFile(user_id, exp_id, 0)
landmark_coord = lfd.getFeatureCoordinates(nlp_shape_color, yaml_file)
if landmark_coord is not None:
#get shape,color
nlp_shape_color = (object_info[0][3], object_info[0][4])
#get displacement from learned gmm
gmm_filename = "../data/gmm_params.yaml"
model_data = lfd.getYamlData(gmm_filename)
nlp_displacement = model_data['mu'][relation[0] - relation_offset]
grounded_prediction = True
#check if we have only one object and can ground on that
elif len(object_info) == 1:
#print "grounding on object only"
nlp_shape_color = (object_info[0][3], object_info[0][4])
nlp_displacement = lfd.getDisplacementFromLandmark(
user_id, exp_id, n_demos, nlp_shape_color)
grounded_prediction = True
#if we have one relationship and multiple items, figure out which item by using relationship
elif len(relation) == 1 and len(object_info) > 0:
#print "grounding on relation only"
#get predicted displacement based on relationship
gmm_filename = "../data/gmm_params.yaml"
model_data = lfd.getYamlData(gmm_filename)
nlp_displacement = model_data['mu'][relation[0] - relation_offset]
#get placement data and object locations
#TODO which demo should I pick from, what should n_demos be??
yaml_file = lfd.getYamlFile(user_id, exp_id, n_demos)
placement_coord = lfd.getPlacementCoordinates(yaml_file, user_id,
exp_id)
min_dist = 10000
for obj in object_info:
shape_color = (obj[3], obj[4])
landmark_coord = lfd.getFeatureCoordinates(shape_color, yaml_file)
predicted_placement = np.array(landmark_coord) + np.array(
nlp_displacement)
placement_error = np.linalg.norm(predicted_placement -
placement_coord)
if placement_error < min_dist:
min_dist = placement_error
best_landmark = shape_color
nlp_shape_color = best_landmark
grounded_prediction = True
#TODO case where only relationship is grounded?
#elif len(relation) == 1:
#see if I have a grounded prediction or if I should go with pure lfd
if grounded_prediction:
#check if grounding prediction seems to match what's been demonstrated
ave_error = averageDistanceError(nlp_shape_color, nlp_displacement,
user_id, exp_id, 0, n_demos + 1)
#print "ave error", ave_error
if ave_error < thresh:
#print "confident in grounding"
return nlp_shape_color, nlp_displacement, relation, object_info
#fall back on pure lfd as a last resort
#print "ambiguous grounding"
return lfd_shape_color, lfd_displacement, relation, object_info
def experiment(train_data, test_data):
total_demos = 10 #total number of demos possible
max_demos = 4 #maximum number of demos given to robot to learn from (the rest are used to test generalizabiltiy)
thresh = 150
#TODO learn the gmm componenets and label the data
print("--learning gmm components--")
gmm_model = gmm.labelTrainingDataDisplacements(train_data, total_demos)
gmm.labelTestingDataDisplacements(test_data, total_demos, gmm_model)
#learn groundings with training data
print("--grounding language--")
nlp_grounder = nlp()
nlp_grounder.train(train_data)
######################
#testing code
######################
print("--testing generalization error--")
#matrix of zeros where each row is new test case and cols are ave errors for learning from 1:max_demos demonstrations
lfd_errors = np.zeros((len(test_data), max_demos))
nlp_errors = np.zeros((len(test_data), max_demos))
random_errors = np.zeros((len(test_data), max_demos))
aveplace_errors = np.zeros((len(test_data), max_demos))
lfd_correct = np.zeros((3, max_demos))
nlp_correct = np.zeros((3, max_demos))
nlp_lfd_correct = np.zeros((3, max_demos))
"""lfd_spatial_correct = np.zeros(max_demos)
lfd_object_correct = np.zeros(max_demos)
lfd_both_correct = np.zeros(max_demos)
nlp_spatial_correct = np.zeros(max_demos)
nlp_object_correct = np.zeros(max_demos)
nlp_both_correct = np.zeros(max_demos)
nlp_lfd_spatial_correct = np.zeros(max_demos)
nlp_lfd_object_correct = np.zeros(max_demos)
nlp_lfd_both_correct = np.zeros(max_demos)"""
for i in range(len(test_data)):
user_id, exp_id = test_data[i]
for n_demos in range(1, max_demos + 1):
#get best guess of landmark and displacement using pure LfD
lfd_shape_color, lfd_displacement = lfd.getMostLikelyLandmarkDisplacement(
user_id, exp_id, n_demos)
#guess landmark and displacement using NLP+LfD
nlp_lfd_shape_color, nlp_displacement, nlp_relation, nlp_shape_color = nlp_lfd.getNLP_LfD_LandmarkDisplacementDoubleCheck(
nlp_grounder, user_id, exp_id, n_demos, thresh)
world = lfd.getYamlFile(user_id, exp_id, 0)
lfd_relation = gmm_model.predict(lfd_displacement)[0]
nlp_lfd_relation = gmm_model.predict(nlp_displacement)[0]
with open(world, 'r') as f:
world_file = yaml.load(f)
landmark = world_file["objects"][0]
with open(
"../data/user_" + str(user_id) + "/experiment_" +
str(exp_id) + "/displacements0.yaml", 'r') as g:
disp_file = yaml.load(g)
disp = disp_file["objects"][0]
if len(nlp_shape_color
) == 1 and nlp_shape_color[0][3] == landmark[
3] and nlp_shape_color[0][4] == landmark[4]:
nlp_correct[0][n_demos -
1] = nlp_correct[0][n_demos - 1] + 1
if len(nlp_relation
) == 1 and nlp_relation[0] - 11 == disp[0]:
nlp_correct[2][n_demos -
1] = nlp_correct[2][n_demos - 1] + 1
if len(nlp_relation) == 1 and nlp_relation[0] - 11 == disp[0]:
nlp_correct[1][n_demos -
1] = nlp_correct[1][n_demos - 1] + 1
if lfd_shape_color[0] == landmark[3] and lfd_shape_color[
1] == landmark[4]:
lfd_correct[0][n_demos -
1] = lfd_correct[0][n_demos - 1] + 1
if lfd_relation == disp[0]:
lfd_correct[2][n_demos -
1] = lfd_correct[2][n_demos - 1] + 1
if lfd_relation == disp[0]:
lfd_correct[1][n_demos -
1] = lfd_correct[1][n_demos - 1] + 1
if nlp_lfd_shape_color[0] == landmark[
3] and nlp_lfd_shape_color[1] == landmark[4]:
nlp_lfd_correct[0][n_demos -
1] = nlp_lfd_correct[0][n_demos - 1] + 1
if nlp_lfd_relation == disp[0]:
nlp_lfd_correct[2][n_demos -
1] = nlp_lfd_correct[2][n_demos -
1] + 1
if nlp_lfd_relation == disp[0]:
nlp_lfd_correct[1][n_demos -
1] = nlp_lfd_correct[1][n_demos - 1] + 1
#guess placment randomly
rand_placement = lfd.getRandomPointOnTable(user_id, exp_id,
n_demos)
#guess placement as average of demonstrated placements
ave_placement = lfd.getMeanPlacementCoordinates(
user_id, exp_id, n_demos)
#compute accuracy over a test demo specified by demo_id
for demo_id in range(max_demos, total_demos):
#pure lfd error
lfd_errors[i, n_demos - 1] = nlp_lfd.averageDistanceError(
lfd_shape_color, lfd_displacement, user_id, exp_id,
max_demos, total_demos)
#nlp+lfd error
nlp_errors[i, n_demos - 1] = nlp_lfd.averageDistanceError(
nlp_lfd_shape_color, nlp_displacement, user_id, exp_id,
max_demos, total_demos)
#random baseline error
random_errors[i, n_demos -
1] = nlp_lfd.averageDistanceToPointError(
rand_placement, user_id, exp_id, max_demos,
total_demos)
#average placement pos baseline error
aveplace_errors[i, n_demos -
1] = nlp_lfd.averageDistanceToPointError(
ave_placement, user_id, exp_id, max_demos,
total_demos)
return (lfd_correct, nlp_correct, nlp_lfd_correct)