def get_freq_grasp(side, train_limit, test_limit):
"""
get frequent grasp types and their occurrence frequence
"""
train_file = "data/feature_grasp_train.csv"
test_file = "data/feature_grasp_test.csv"
f_feat = open(train_file, "r")
first_line = f_feat.readline()
tags = first_line.split(',')
idx_g = -1
idx_s = -1
for i in range(len(tags)):
if tags[i] == "grasp":
idx_g = i
if tags[i] == "side":
idx_s = i
assert (idx_g != -1 and idx_s != -1)
grasp_count_train = {}
features = f_feat.readlines()
for item in features:
item_split = item.split(',')
if side != item_split[idx_s]:
continue
grasp_type = transfer_grasp_label(item_split[idx_g])
if grasp_count_train.has_key(grasp_type):
grasp_count_train[grasp_type] = grasp_count_train[grasp_type] + 1
else:
grasp_count_train[grasp_type] = 1
for item in grasp_count_train.keys():
if grasp_count_train[item] < train_limit:
del grasp_count_train[item]
f_feat.close()
f_feat = open(test_file, "r")
first_line = f_feat.readline()
tags = first_line.split(',')
idx_g = -1
idx_s = -1
for i in range(len(tags)):
if tags[i] == "grasp":
idx_g = i
if tags[i] == "side":
idx_s = i
assert (idx_g != -1 and idx_s != -1)
grasp_count_test = {}
features = f_feat.readlines()
for item in features:
item_split = item.split(',')
if side != item_split[idx_s]:
continue
grasp_type = transfer_grasp_label(item_split[idx_g])
if grasp_count_test.has_key(grasp_type):
grasp_count_test[grasp_type] = grasp_count_test[grasp_type] + 1
else:
grasp_count_test[grasp_type] = 1
for item in grasp_count_test.keys():
if grasp_count_test[item] < test_limit:
del grasp_count_test[item]
f_feat.close()
return [grasp_count_train, grasp_count_test]
def write_svmdata_grasp(srcfile, datafile, grasp_type, side, isTest):
"""
write formed data required by libsvm\n
"""
f_feat = open(srcfile, "r")
first_line = f_feat.readline()
tags = first_line.split(',')
idx_g = -1
idx_f = -1
idx_s = -1
for i in range(len(tags)):
if tags[i] == "grasp":
idx_g = i
if tags[i].find("feature") == 0:
idx_f = i
if tags[i] == "side":
idx_s = i
assert (idx_g != -1 and idx_f != -1 and idx_s != -1)
items = f_feat.readlines()
f_feat.close()
labels = []
features = []
for item in items:
item_split = item.split(',')
#only process right hand; modify this if double hands are to be process
if side != item_split[idx_s]:
continue
label = 0
if grasp_type == transfer_grasp_label(item_split[idx_g]):
label = 1
labels.append(label)
features.append(item_split[idx_f:len(item_split) - 1])
#project to PC axis
data = np.array(features, dtype=np.float32)
# print "data before pca: ", data.shape
# data = feature_pca(data)
# print "data after pca: ", data.shape
#make sure the first data is positive sample
if labels[0] < 1 and isTest == 0:
idx_pos = -1
for i in range(len(labels)):
if labels[i] == 1:
idx_pos = i
assert (idx_pos != -1)
data_temp = np.array(data[idx_pos, :], data.dtype)
for j in range(data.shape[1]):
data[idx_pos, j] = data[0, j]
data[0, j] = data_temp[j]
labels[idx_pos] = labels[0]
labels[0] = 1
#write formed data required by libsvm
f_svmdata = open(datafile, "w")
for i in range(len(labels)):
f_svmdata.write(str(labels[i]) + " ")
for j in range(data.shape[1]):
f_svmdata.write(str(j + 1) + ":" + str(data[i, j]) + " ")
f_svmdata.write("\n")
f_svmdata.close()
def extract_cnn_grasp_attribute(seqs):
"""
extract deep convolutional network features for hand grasps\n
seqs: list of sequence names\n
dstfile: file to which features are written
"""
g_a_left = {}
g_a_right = {}
for seq in seqs:
#collect all yaml file names
print('Prepare cnn feature from ' + seq + '...')
command = "ls " + seq + "/*.yml > filename.txt"
os.system(command)
f_files = open("filename.txt", "r")
files = f_files.readlines()
f_files.close()
for j in range(len(files)):
#read from each yaml file
ymlfile = files[j][:len(files[j])-1]
print(ymlfile)
f_yml = open(ymlfile, "r")
yml_dict = yaml.load(f_yml.read())
f_yml.close()
seqname = str(yml_dict["seqname"])
filename = str(yml_dict["filename"])
is_hLvisible = yml_dict["lefthand"]["visible"]
hL_grasp = yml_dict["lefthand"]["grasp"]
item_split = hL_grasp.split()
hL_grasp = "-".join(item_split)
hL_grasp = transfer_grasp_label(hL_grasp)
hL_xmin = yml_dict["lefthand"]["bndbox"]["xmin"]
hL_ymin = yml_dict["lefthand"]["bndbox"]["ymin"]
hL_xmax = yml_dict["lefthand"]["bndbox"]["xmax"]
hL_ymax = yml_dict["lefthand"]["bndbox"]["ymax"]
is_hRvisible = yml_dict["righthand"]["visible"]
hR_grasp = yml_dict["righthand"]["grasp"]
item_split = hR_grasp.split()
hR_grasp = "-".join(item_split)
hR_grasp = transfer_grasp_label(hR_grasp)
hR_xmin = yml_dict["righthand"]["bndbox"]["xmin"]
hR_ymin = yml_dict["righthand"]["bndbox"]["ymin"]
hR_xmax = yml_dict["righthand"]["bndbox"]["xmax"]
hR_ymax = yml_dict["righthand"]["bndbox"]["ymax"]
is_oLvisible = yml_dict["leftobject"]["visible"]
oL_xmin = yml_dict["leftobject"]["bndbox"]["xmin"]
oL_ymin = yml_dict["leftobject"]["bndbox"]["ymin"]
oL_xmax = yml_dict["leftobject"]["bndbox"]["xmax"]
oL_ymax = yml_dict["leftobject"]["bndbox"]["ymax"]
is_oLprismatic = yml_dict["leftobject"]["attribute"]["prismatic"]
is_oLsphere = yml_dict["leftobject"]["attribute"]["sphere"]
is_oLflat = yml_dict["leftobject"]["attribute"]["flat"]
is_oLrigid = yml_dict["leftobject"]["attribute"]["rigid"]
is_oRvisible = yml_dict["rightobject"]["visible"]
oR_xmin = yml_dict["rightobject"]["bndbox"]["xmin"]
oR_ymin = yml_dict["rightobject"]["bndbox"]["ymin"]
oR_xmax = yml_dict["rightobject"]["bndbox"]["xmax"]
oR_ymax = yml_dict["rightobject"]["bndbox"]["ymax"]
is_oRprismatic = yml_dict["rightobject"]["attribute"]["prismatic"]
is_oRsphere = yml_dict["rightobject"]["attribute"]["sphere"]
is_oRflat = yml_dict["rightobject"]["attribute"]["flat"]
is_oRrigid = yml_dict["rightobject"]["attribute"]["rigid"]
img = caffe.io.load_image(img_dir+"/"+seqname+"/"+filename+".jpg")
if is_hLvisible == 1 and is_oLvisible == 1:
value_oneframe = []
value_oneframe.append(str(seqname)+","+str(filename)+",left")
value_oneframe.append(str(is_oLprismatic)+","+str(is_oLsphere)+","+str(is_oLflat)+","+str(is_oLrigid))
imgroi = img[hL_ymin:hL_ymax+1, hL_xmin:hL_xmax+1]
net.predict([imgroi])
feat = net.blobs[LAYER].data[INDEX].flatten().tolist()
feat_str = ""
for value in feat:
feat_str = feat_str + str(value) + ','
value_oneframe.append(feat_str)
imgroi = img[oL_ymin:oL_ymax+1, oL_xmin:oL_xmax+1]
net.predict([imgroi])
feat = net.blobs[LAYER].data[INDEX].flatten().tolist()
feat_str = ""
for value in feat:
feat_str = feat_str + str(value) + ','
value_oneframe.append(feat_str)
if g_a_left.has_key(hL_grasp):
g_a_left[hL_grasp].append(value_oneframe)
else:
g_a_left[hL_grasp] = []
g_a_left[hL_grasp].append(value_oneframe)
if is_hRvisible == 1 and is_oRvisible:
value_oneframe = []
value_oneframe.append(str(seqname)+","+str(filename)+",right")
value_oneframe.append(str(is_oRprismatic)+","+str(is_oRsphere)+","+str(is_oRflat)+","+str(is_oRrigid))
imgroi = img[hR_ymin:hR_ymax+1, hR_xmin:hR_xmax+1]
net.predict([imgroi])
feat = net.blobs[LAYER].data[INDEX].flatten().tolist()
feat_str = ""
for value in feat:
feat_str = feat_str + str(value) + ','
value_oneframe.append(feat_str)
imgroi = img[oR_ymin:oR_ymax+1, oR_xmin:oR_xmax+1]
net.predict([imgroi])
feat = net.blobs[LAYER].data[INDEX].flatten().tolist()
feat_str = ""
for value in feat:
feat_str = feat_str + str(value) + ','
value_oneframe.append(feat_str)
if g_a_right.has_key(hR_grasp):
g_a_right[hR_grasp].append(value_oneframe)
else:
g_a_right[hR_grasp] = []
g_a_right[hR_grasp].append(value_oneframe)
# break
# break
return g_a_left, g_a_right
def edge_potential(seqs, attributes, grasps, side):
"""
get edge potential of object attribute and grasp type from training data
"""
print "train for edge potential...\n"
pA = np.zeros(2**len(attributes), np.float32)
pGbyA = np.zeros((pA.size, len(grasps)), np.float32)
for seq in seqs:
#collect all yaml file names
command = "ls " + seq + "/*.yml > filename.txt"
os.system(command)
f_files = open("filename.txt", "r")
files = f_files.readlines()
f_files.close()
for j in range(len(files)):
#read from each yaml file
ymlfile = files[j][:len(files[j]) - 1]
f_yml = open(ymlfile, "r")
yml_dict = yaml.load(f_yml.read())
f_yml.close()
seqname = str(yml_dict["seqname"])
filename = str(yml_dict["filename"])
is_oLvisible = yml_dict["leftobject"]["visible"]
is_oLprismatic = yml_dict["leftobject"]["attribute"]["prismatic"]
is_oLsphere = yml_dict["leftobject"]["attribute"]["sphere"]
is_oLflat = yml_dict["leftobject"]["attribute"]["flat"]
is_oLrigid = yml_dict["leftobject"]["attribute"]["rigid"]
is_oRvisible = yml_dict["rightobject"]["visible"]
is_oRprismatic = yml_dict["rightobject"]["attribute"]["prismatic"]
is_oRsphere = yml_dict["rightobject"]["attribute"]["sphere"]
is_oRflat = yml_dict["rightobject"]["attribute"]["flat"]
is_oRrigid = yml_dict["rightobject"]["attribute"]["rigid"]
is_hLvisible = yml_dict["lefthand"]["visible"]
hL_grasp = yml_dict["lefthand"]["grasp"]
item_split = hL_grasp.split()
hL_grasp = "-".join(item_split)
hL_grasp = transfer_grasp_label(hL_grasp)
is_hRvisible = yml_dict["righthand"]["visible"]
hR_grasp = yml_dict["righthand"]["grasp"]
item_split = hR_grasp.split()
hR_grasp = "-".join(item_split)
hR_grasp = transfer_grasp_label(hR_grasp)
if is_oLvisible == 1 and is_hLvisible == 1 and side == "left":
idx_attribute = 0
if is_oLflat == 1:
idx_attribute = idx_attribute | FLAT_ID
if is_oLprismatic == 1:
idx_attribute = idx_attribute | PRISMATIC_ID
if is_oLrigid == 1:
idx_attribute = idx_attribute | RIGID_ID
if is_oLsphere == 1:
idx_attribute = idx_attribute | SPHERE_ID
pA[idx_attribute] += 1
idx_grasp = -1
for i in range(len(grasps)):
if hL_grasp == grasps[i]:
idx_grasp = i
if idx_grasp != -1:
pGbyA[idx_attribute, idx_grasp] += 1
if is_oRvisible == 1 and is_hRvisible == 1 and side == "right":
idx_attribute = 0
if is_oRflat == 1:
idx_attribute = idx_attribute | FLAT_ID
if is_oRprismatic == 1:
idx_attribute = idx_attribute | PRISMATIC_ID
if is_oRrigid == 1:
idx_attribute = idx_attribute | RIGID_ID
if is_oRsphere == 1:
idx_attribute = idx_attribute | SPHERE_ID
pA[idx_attribute] += 1
idx_grasp = -1
for i in range(len(grasps)):
if hR_grasp == grasps[i]:
idx_grasp = i
if idx_grasp != -1:
pGbyA[idx_attribute, idx_grasp] += 1
pA.__idiv__(np.sum(pA)) #divide by sum to get proportion
pGA = np.zeros(pGbyA.shape, pGbyA.dtype)
L = 1.0 #universally added number
pGbyA.__iadd__(L)
for i in range(pGbyA.shape[0]):
if np.sum(pGbyA[i, :]) == 0:
continue
pGbyA[i, :].__idiv__(np.sum(
pGbyA[i, :])) #divide by line sum to get proportion
for j in range(pGbyA.shape[1]):
pGA[i,
j] = pGbyA[i,
j] #* pA[i] uncomment when consider different prior
return pGA