def main():
parser = argparse.ArgumentParser()
parser.add_argument('raw_path')
parser.add_argument('model_path')
args = parser.parse_args()
raw_path = args.raw_path
model_path = args.model_path
mask_path = raw_to_mask_path(raw_path)
raw = imread(raw_path)
mask = imread(mask_path)
y_min, x_min, y_max, x_max = mask_to_roi(mask)
im = raw[y_min:y_max, x_min:x_max]
model = VGG_mini_ABN()
serializers.load_hdf5(model_path, model)
im = resize(im, (128, 128), preserve_range=True)
x_data = np.array([im_to_blob(im)], dtype=np.float32)
x = Variable(x_data, volatile=True)
model.train = False
y = model(x)
y_data = y.data
print(OBJECT_CLASSES[np.argmax(y_data[0])])
def setUp(self):
self.model = CAEPool()
self.optimizer = O.Adam()
self.optimizer.setup(self.model)
img = doll()
x_data = np.array([im_to_blob(im_preprocess(img))])
self.x = Variable(x_data)
def setUp(self):
self.model = VGG_mini_ABN()
self.optimizer = O.Adam()
self.optimizer.setup(self.model)
img = doll()
img = resize(img, (128, 128), preserve_range=True)
x_data = np.array([im_to_blob(img)])
y_data = np.array([0], dtype=np.int32)
self.x = Variable(x_data)
self.y = Variable(y_data)
def dataset_to_xt_data(dataset, crop_roi):
"""Convert dataset to x_data and t_data"""
x_data = []
for raw_path in dataset.filenames:
import cv2
raw = cv2.imread(raw_path)
raw = resize(raw, (267, 178), preserve_range=True)
x_data.append(im_to_blob(raw))
x_data = np.array(x_data, dtype=np.float32)
t_data = dataset.target.astype(np.int32)
return x_data, t_data
def setUp(self):
cae_ones_h5 = os.path.join(here, 'data/cae_ones.h5')
vgg_h5 = os.path.join(here, 'data/vgg.h5')
self.model = CAEOnesRoiVGG(
initial_roi=[100, 100, 300, 300],
cae_ones_h5=cae_ones_h5, vgg_h5=vgg_h5)
self.optimizer = O.Adam()
self.optimizer.setup(self.model)
img = doll()
x_data = np.array([im_to_blob(im_preprocess(img))])
self.x = Variable(x_data)
t_data = np.array([0], dtype=np.int32)
self.t = Variable(t_data)
def dataset_to_xt_data(dataset, crop_roi):
"""Convert dataset to x_data and t_data"""
x_data = []
for raw_path in dataset.filenames:
raw = im_preprocess(imread(raw_path))
if crop_roi:
mask_path = raw_to_mask_path(raw_path)
roi = mask_to_roi(im_preprocess(imread(mask_path)))
raw = raw[roi[0]:roi[2], roi[1]:roi[3]]
raw = resize(raw, (128, 128), preserve_range=True)
x_data.append(im_to_blob(raw))
x_data = np.array(x_data, dtype=np.float32)
t_data = dataset.target.astype(np.int32)
return x_data, t_data
def x0_to_x1(self, x0, roi_scale):
on_gpu = isinstance(x0.data, cupy.ndarray)
roi_scale_data = cuda.to_cpu(roi_scale.data) \
if on_gpu else roi_scale.data
rois_data = (self.initial_roi * roi_scale_data).astype(int)
x0_data = cuda.to_cpu(x0.data) if on_gpu else x0.data
cropped = []
for i in xrange(len(x0_data)):
roi = rois_data[i]
im = blob_to_im(x0_data[i])
im = im[roi[0]:roi[2], roi[1]:roi[3]]
if im.size == 0:
break
im = resize(im, (128, 128), preserve_range=True)
cropped.append(im_to_blob(im))
else:
cropped_data = np.array(cropped, dtype=np.float32)
if on_gpu:
cropped_data = cuda.to_gpu(cropped_data)
x1 = Variable(cropped_data, volatile=not self.train)
return x1
def cb(self, msg):
bridge = cv_bridge.CvBridge()
depth_tmp = bridge.imgmsg_to_cv2(msg)
shape = list(depth_tmp.shape)
shape.append(3)
depth = np.zeros(shape, depth_tmp.dtype)
depth[:,:,0] = depth_tmp
depth[:,:,1] = depth_tmp
depth[:,:,2] = depth_tmp
depth = depth.astype(float)
depth = depth / depth.max() * 255
depth = depth.astype(np.uint8)
depth = resize(depth, (267, 178), preserve_range=True)
x_data = np.array([apc_od.im_to_blob(depth)], dtype=np.float32)
x = Variable(x_data, volatile='off')
z = self.model.encode(x)
initial_param = np.array([0.02])
scale = z.data
tolerance = (initial_param * scale)[0][0]
params = {'tolerance': tolerance}
self.reconfig_client.update_configuration(params)
def test_tile_ae_inout():
im = apc_od.data.doll()
x = apc_od.im_to_blob(im)
x_hat = x.copy()
apc_od.tile_ae_inout([x], [x_hat], '/tmp/tile_ae_inout.jpg')
def test_tile_ae_encoded():
im = apc_od.data.doll()
x = apc_od.im_to_blob(im)
apc_od.tile_ae_encoded([x, x], '/tmp/tile_ae_encoded.jpg')
def main():
model = CAEOnes(n_param=2)
optimizer = O.Adam()
optimizer.setup(model)
S.load_hdf5('bof_data/cae_ones_model.h5', model)
S.load_hdf5('bof_data/cae_ones_optimizer.h5', optimizer)
with gzip.open('bof_data/bof_berkeley.pkl.gz', 'rb') as f:
bof = pickle.load(f)
with gzip.open('bof_data/lgr_merged.pkl.gz', 'rb') as f:
lgr = pickle.load(f)
this_dir = os.path.dirname(os.path.abspath(__file__))
data_home = os.path.abspath(os.path.join(this_dir, '../data'))
train_data_dir = os.path.join(data_home, 'in_hand_recog_{}'.format('train'))
test_data_dir = os.path.join(data_home, 'in_hand_recog_{}'.format('test'))
train_data = load_files(train_data_dir, load_content=False)
test_data = load_files(test_data_dir, load_content=False)
N_train = len(train_data.filenames)
N_test = len(test_data.filenames)
train_imgs = []
for f in train_data.filenames:
if f.endswith('_0.jpg'):
# Skip mask file
continue
img_file = f
mask_file = img_file.split('_1.jpg')[0] + '_0.jpg'
img = cv2.imread(img_file, 0)
mask = cv2.imread(mask_file)
train_imgs.append((img, mask))
test_imgs = []
for f in test_data.filenames:
if f.endswith('_0.jpg'):
# Skip mask file
continue
img_file = f
mask_file = img_file.split('_1.jpg')[0] + '_0.jpg'
img = cv2.imread(img_file, 0)
mask = cv2.imread(mask_file)
test_imgs.append((img, mask))
y_true_0 = 12
# y_proba_true_0 = np.zeros(25, dtype=np.float32)
# y_proba_true_0[12] = 1
# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
# test
n_batch = 10
initial_params = [8, 4] # size, iterations
epoch = 0
model.train = False
accuracies = []
print('testing')
N = len(test_imgs)
perm = np.random.permutation(len(test_imgs))
for i in xrange(0, N, n_batch):
print('test_batch: ', i)
test_batch = [test_imgs[p_index] for p_index in perm[i:i+n_batch]]
y_true = np.repeat([y_true_0], len(test_batch), axis=0)
# y_proba_true = np.repeat(y_proba_true_0, len(test_batch), axis=0)
x_data = []
for img, mask in test_batch:
mask = resize(mask, (267, 178), preserve_range=True)
x_data.append(apc_od.im_to_blob(mask))
x_data = np.array(x_data, dtype=np.float32)
x = Variable(x_data, volatile='on')
z = model.encode(x)
param_scale = z.data
params = param_scale * initial_params
X = []
for k, param in enumerate(params):
size, iterations = map(int, param)
if size <= 0 or size > 50 or iterations <= 0 or iterations > 50:
rand = 1. * np.ones(2) / param_scale
params = rand * param_scale * initial_params
size, iterations = map(int, params[0])
print('test:', size, iterations)
if size <= 0 or size > 50 or iterations <= 0 or iterations > 50:
size, iterations = initial_params
kernel = np.ones((size, size), dtype=np.uint8)
img, mask = test_batch[k]
closed = cv2.morphologyEx(mask[:,:,0], cv2.MORPH_CLOSE, kernel, iterations=iterations)
cropped = bounding_rect_of_mask(img, closed)
frames, desc = get_sift_keypoints(cropped)
X.append(desc)
X = np.array(X)
if X.size == 0:
print('test: skipping')
N -= n_batch
continue
X_trans = bof.transform(X)
X_trans = normalize(X_trans)
y_pred = lgr.predict(X_trans)
accuracy = accuracy_score(y_true, y_pred)
accuracies.append(accuracy)
# y_proba = lgr.predict_proba(X_trans)
# square_error = np.sum(np.power(y_proba - y_true, 2))
# sum_error += square_error
mean_accuracy = np.array(accuracy).mean()
msg = 'epoch:{:02d}; test mean accuracy={};'.format(epoch, mean_accuracy)
write_log(msg)
print(msg)
# <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
n_batch = 10
learning_n_sample = 100
learning_rate = 0.1
initial_params = [8, 4] # size, iterations
for epoch in xrange(1, 11):
print('epoch:', epoch)
# train
model.train = True
sum_loss = 0
accuracies = []
N = len(train_imgs)
N_train = len(train_imgs)
perm = np.random.permutation(N_train)
for i in range(0, N_train, n_batch):
print('train_batch: ', i)
train_batch = [train_imgs[p_index] for p_index in perm[i:i+n_batch]]
y_true = np.repeat([y_true_0], len(train_batch), axis=0)
# y_proba_true = np.repeat(y_proba_true_0, len(train_batch), axis=0)
x_data = []
for img, mask in train_batch:
mask = resize(mask, (267, 178), preserve_range=True)
x_data.append(apc_od.im_to_blob(mask))
x_data = np.array(x_data, dtype=np.float32)
x = Variable(x_data, volatile='off')
z = model.encode(x)
param_scale = z.data
rands_shape = [learning_n_sample] + list(param_scale.shape)
rands = 1. * learning_rate * (11 - epoch) / 11 * (2 * np.random.random(rands_shape) - 1) + 1
rands[0] = np.ones(param_scale.shape) # ones
min_rand = None
max_accuracy = -np.inf
optimizer.zero_grads()
for j, rand in enumerate(rands):
params = rand * param_scale * initial_params
X = []
for k, param in enumerate(params):
size, iterations = map(int, param)
if size <= 0 or size > 50 or iterations <= 0 or iterations > 50:
size, iterations = initial_params
kernel = np.ones((size, size), dtype=np.uint8)
img, mask = train_batch[k]
closed = cv2.morphologyEx(mask[:,:,0], cv2.MORPH_CLOSE, kernel, iterations=iterations)
cropped = bounding_rect_of_mask(img, closed)
frames, desc = get_sift_keypoints(cropped)
X.append(desc)
X = np.array(X)
if X.size == 0:
continue
X_trans = bof.transform(X)
X_trans = normalize(X_trans)
y_pred = lgr.predict(X_trans)
accuracy = accuracy_score(y_true, y_pred)
# y_proba = lgr.predict_proba(X_trans)[0]
# square_error = 1. * np.sum(np.power(y_proba - y_true, 2)) / len(train_batch)
if accuracy > max_accuracy:
max_accuracy = accuracy
min_rand = rand
if min_rand is None:
print('train: skipping')
N -= n_batch
continue
t_data = np.array(min_rand * param_scale, dtype=np.float32)
t = Variable(t_data, volatile='off')
loss = F.mean_squared_error(t, z)
loss.backward()
optimizer.update()
sum_loss += float(loss.data) * len(train_batch)
accuracies.append(accuracy)
try:
mean_loss = 1. * sum_loss / N
except ZeroDivisionError:
mean_loss = np.inf
mean_accuracy = np.array(accuracies).mean()
msg = 'epoch:{:02d}; train mean loss={};'.format(epoch, mean_loss)
write_log(msg)
print(msg)
msg = 'epoch:{:02d}; train mean accuracy={};'.format(epoch, mean_accuracy)
write_log(msg)
print(msg)
# test
model.train = False
sum_error = 0
print('testing')
accuracies = []
N = len(test_imgs)
perm = np.random.permutation(len(test_imgs))
for i in xrange(0, N, n_batch):
print('test_batch: ', i)
test_batch = [test_imgs[p_index] for p_index in perm[i:i+n_batch]]
y_true = np.repeat([y_true_0], len(test_batch), axis=0)
x_data = []
for img, mask in test_batch:
mask = resize(mask, (267, 178), preserve_range=True)
x_data.append(apc_od.im_to_blob(mask))
x_data = np.array(x_data, dtype=np.float32)
x = Variable(x_data, volatile='on')
z = model.encode(x)
param_scale = z.data
params = param_scale * initial_params
X = []
for k, param in enumerate(params):
size, iterations = map(int, param)
if size <= 0 or size > 50 or iterations <= 0 or iterations > 50:
rand = 1. * np.ones(2) / param_scale
params = rand * param_scale * initial_params
size, iterations = map(int, params[0])
print('test:', size, iterations)
if size <= 0 or size > 50 or iterations <= 0 or iterations > 50:
size, iterations = initial_params
kernel = np.ones((size, size), dtype=np.uint8)
img, mask = test_batch[k]
closed = cv2.morphologyEx(mask[:,:,0], cv2.MORPH_CLOSE, kernel, iterations=iterations)
cropped = bounding_rect_of_mask(img, closed)
frames, desc = get_sift_keypoints(cropped)
X.append(desc)
X = np.array(X)
if X.size == 0:
print('test: skipping')
N -= n_batch
continue
X_trans = bof.transform(X)
X_trans = normalize(X_trans)
y_pred = lgr.predict(X_trans)
accuracy = accuracy_score(y_true, y_pred)
accuracies.append(accuracy)
# y_proba = lgr.predict_proba(X_trans)[0]
# square_error = np.sum(np.power(y_proba - y_true, 2))
# sum_error += square_error
mean_accuracy = np.array(accuracies).mean()
msg = 'epoch:{:02d}; test mean accuracy={};'.format(epoch, mean_accuracy)
write_log(msg)
print(msg)
S.save_hdf5('bof_data/cae_ones_model_trained_{}.h5'.format(epoch), model)
S.save_hdf5('bof_data/cae_ones_optimizer_trained_{}.h5'.format(epoch), optimizer)
def __call__(self, x, t=None):
on_gpu = isinstance(x.data, cupy.ndarray)
self.cae_ones1.train = self.train
self.vgg2.train = self.train
roi_scale = self.cae_ones1.encode(x)
if t is None:
assert self.train is False
if on_gpu:
roi_scale_data = cuda.to_cpu(roi_scale.data)
rois_data = self.initial_roi * roi_scale_data
if on_gpu:
rois_data = cuda.to_cpu(rois_data)
x_data = cuda.to_cpu(x.data)
rois_data = rois_data.astype(int)
cropped = []
for i in xrange(len(x_data)):
roi = rois_data[i]
im = blob_to_im(x_data[i])
im = im[roi[0]:roi[2], roi[1]:roi[3]]
im = resize(im, (128, 128), preserve_range=True)
cropped.append(im_to_blob(im))
cropped = np.array(cropped, dtype=np.float32)
if on_gpu:
cropped = cuda.to_gpu(cropped)
cropped = Variable(cropped, volatile=not self.train)
self.vgg2(cropped)
self.y = self.vgg2.y
return self.y
# randomly change the param and estimate good parameter for the task
min_y = None
rands_shape = [self.learning_n_sample] + list(roi_scale.data.shape)
rands = self.learning_rate * (2 * np.random.random(rands_shape) - 1) + 1
rands[0] = np.ones(roi_scale.data.shape)
for i, rand in enumerate(rands):
if on_gpu:
roi_scale_data = cuda.to_cpu(roi_scale.data)
rois_data = rand * (self.initial_roi * roi_scale_data)
x_data = cuda.to_cpu(x.data)
skip = False
rois_data = rois_data.astype(int)
cropped = []
for j in xrange(len(x_data)):
roi = rois_data[j]
im = blob_to_im(x_data[j])
im = im[roi[0]:roi[2], roi[1]:roi[3]]
if im.size == 0:
skip = True
break
im = resize(im, (128, 128), preserve_range=True)
cropped.append(im_to_blob(im))
if skip:
continue
cropped = np.array(cropped)
if on_gpu:
cropped = cuda.to_gpu(cropped)
cropped = Variable(cropped, volatile=not self.train)
self.vgg2(cropped, t)
h = self.vgg2.y
loss = F.softmax_cross_entropy(h, t)
if min_y is None:
min_loss_data = float(loss.data)
min_y = h
min_loss = loss
min_rand = rand
min_rois = rois_data
elif min_loss_data > float(loss.data):
min_loss_data = float(loss.data)
min_y = h
min_loss = loss
min_rand = rand
min_rois = rois_data
if on_gpu:
min_rand = cuda.to_gpu(min_rand)
rois_data = min_rand * roi_scale.data
xp = cuda.get_array_module(rois_data)
rois_data = rois_data.astype(xp.float32)
rois = Variable(rois_data, volatile=not self.train)
loss1 = F.mean_squared_error(roi_scale, rois)
loss2 = min_loss
self.loss = loss1 + loss2
self.accuracy = F.accuracy(min_y, t)
return self.loss
def main():
rospy.init_node('train_inbin_reconfig')
update_filename_client = rospy.ServiceProxy(
'pcd_to_pointcloud/update_filename', UpdateFilename)
parser = argparse.ArgumentParser()
parser.add_argument('container_dir')
args = parser.parse_args()
model = CAEOnes(n_param=1)
optimizer = O.Adam()
optimizer.setup(model)
print('loading hd5')
S.load_hdf5('cae_ones_model_inbin.h5', model)
S.load_hdf5('cae_ones_optimizer_inbin.h5', optimizer)
print('done loading')
directory = args.container_dir
# test
epoch = 0
sum_acc = 0
for i in [4, 5]:
pcd_file = os.path.realpath(glob.glob('{}/{}/*.pcd'.format(directory, i))[0])
req = UpdateFilenameRequest(filename=pcd_file)
res = update_filename_client(req)
print('update_filename to: {}, success: {}'
.format(pcd_file, res.success))
image_file = os.path.join(directory, str(i), 'image.jpg')
depth_file = os.path.join(directory, str(i), 'depth.jpg')
all_mask_file = os.path.join(directory, str(i), 'mask.jpg')
diffmask_file = os.path.join(directory, str(i), 'diffmask2.jpg')
image = cv2.imread(image_file)
depth = cv2.imread(depth_file)
all_mask = cv2.imread(all_mask_file, 0)
mask = cv2.imread(diffmask_file, 0)
depth = resize(depth, (267, 178), preserve_range=True)
x_data = np.array([apc_od.im_to_blob(depth)], dtype=np.float32)
x = Variable(x_data, volatile='on')
trainer = TrainInBinReconfig(model=model, optimizer=optimizer,
x=x, mask=mask, image=image, all_mask=all_mask)
model.train = False
param_scale = model.encode(x).data
params = (trainer.initial_param * param_scale)[0]
tolerance = params[0]
if params[0] < 0.0009:
params[0] = trainer.initial_param[0]
trainer.reconfigure(tolerance=params[0])
trainer.reconfigure(tolerance=tolerance)
val, label = trainer.evaluate()
save_label_and_img('{}_{}.jpg'.format(i, epoch), label, image)
sum_acc += (1. - val)
mean_acc = sum_acc / 2.
msg = 'epoch:{:02d}; test mean accuracy0={};'.format(epoch, mean_acc)
write_log(msg)
for epoch in xrange(1, 21):
sum_loss = 0
sum_acc = 0
for i in [1, 2, 3]:
pcd_file = os.path.realpath(glob.glob('{}/{}/*.pcd'.format(directory, i))[0])
req = UpdateFilenameRequest(filename=pcd_file)
res = update_filename_client(req)
print('update_filename to: {}, success: {}'
.format(pcd_file, res.success))
image_file = os.path.join(directory, str(i), 'image.jpg')
depth_file = os.path.join(directory, str(i), 'depth.jpg')
all_mask_file = os.path.join(directory, str(i), 'mask.jpg')
diffmask_file = os.path.join(directory, str(i), 'diffmask2.jpg')
image = cv2.imread(image_file)
depth = cv2.imread(depth_file)
all_mask = cv2.imread(all_mask_file, 0)
mask = cv2.imread(diffmask_file, 0)
depth = resize(depth, (267, 178), preserve_range=True)
x_data = np.array([apc_od.im_to_blob(depth)], dtype=np.float32)
x = Variable(x_data, volatile='off')
trainer = TrainInBinReconfig(
model=model, optimizer=optimizer, x=x, mask=mask, image=image, all_mask=all_mask)
model.train = True
loss_data, acc = trainer.random_sample(fname='{}_{}.jpg'.format(i, epoch), epoch=epoch)
sum_loss += loss_data
sum_acc += acc
mean_loss = sum_loss / 3.
mean_acc = sum_acc / 3.
msg = 'epoch:{:02d}; train mean loss0={};'.format(epoch, mean_loss)
write_log(msg)
msg = 'epoch:{:02d}; train mean accuracy0={};'.format(epoch, mean_acc)
write_log(msg)
# test
sum_acc = 0
for i in [4, 5]:
pcd_file = os.path.realpath(glob.glob('{}/{}/*.pcd'.format(directory, i))[0])
req = UpdateFilenameRequest(filename=pcd_file)
res = update_filename_client(req)
print('update_filename to: {}, success: {}'
.format(pcd_file, res.success))
image_file = os.path.join(directory, str(i), 'image.jpg')
depth_file = os.path.join(directory, str(i), 'depth.jpg')
all_mask_file = os.path.join(directory, str(i), 'mask.jpg')
diffmask_file = os.path.join(directory, str(i), 'diffmask2.jpg')
image = cv2.imread(image_file)
depth = cv2.imread(depth_file)
all_mask = cv2.imread(all_mask_file, 0)
mask = cv2.imread(diffmask_file, 0)
depth = resize(depth, (267, 178), preserve_range=True)
x_data = np.array([apc_od.im_to_blob(depth)], dtype=np.float32)
x = Variable(x_data, volatile='on')
trainer = TrainInBinReconfig(model=model, optimizer=optimizer,
x=x, mask=mask, image=image, all_mask=all_mask)
model.train = False
param_scale = model.encode(x).data
params = (trainer.initial_param * param_scale)[0]
if params[0] < 0.0009:
params[0] = trainer.initial_param[0]
tolerance = params[0]
trainer.reconfigure(tolerance=tolerance)
val, label = trainer.evaluate()
save_label_and_img('{}_{}.jpg'.format(i, epoch), label, image)
sum_acc += (1. - val)
mean_acc = sum_acc / 2.
msg = 'epoch:{:02d}; test mean accuracy0={};'.format(epoch, mean_acc)
write_log(msg)
S.save_hdf5('cae_ones_model_inbin_trained.h5', model)
S.save_hdf5('cae_ones_optimizer_inbin_trained.h5', optimizer)
