def main(): parser = argparse.ArgumentParser() parser.add_argument('raw_path') parser.add_argument('model_path') args = parser.parse_args() raw_path = args.raw_path model_path = args.model_path mask_path = raw_to_mask_path(raw_path) raw = imread(raw_path) mask = imread(mask_path) y_min, x_min, y_max, x_max = mask_to_roi(mask) im = raw[y_min:y_max, x_min:x_max] model = VGG_mini_ABN() serializers.load_hdf5(model_path, model) im = resize(im, (128, 128), preserve_range=True) x_data = np.array([im_to_blob(im)], dtype=np.float32) x = Variable(x_data, volatile=True) model.train = False y = model(x) y_data = y.data print(OBJECT_CLASSES[np.argmax(y_data[0])])
def setUp(self): self.model = CAEPool() self.optimizer = O.Adam() self.optimizer.setup(self.model) img = doll() x_data = np.array([im_to_blob(im_preprocess(img))]) self.x = Variable(x_data)
def setUp(self): self.model = VGG_mini_ABN() self.optimizer = O.Adam() self.optimizer.setup(self.model) img = doll() img = resize(img, (128, 128), preserve_range=True) x_data = np.array([im_to_blob(img)]) y_data = np.array([0], dtype=np.int32) self.x = Variable(x_data) self.y = Variable(y_data)
def dataset_to_xt_data(dataset, crop_roi): """Convert dataset to x_data and t_data""" x_data = [] for raw_path in dataset.filenames: import cv2 raw = cv2.imread(raw_path) raw = resize(raw, (267, 178), preserve_range=True) x_data.append(im_to_blob(raw)) x_data = np.array(x_data, dtype=np.float32) t_data = dataset.target.astype(np.int32) return x_data, t_data
def setUp(self): cae_ones_h5 = os.path.join(here, 'data/cae_ones.h5') vgg_h5 = os.path.join(here, 'data/vgg.h5') self.model = CAEOnesRoiVGG( initial_roi=[100, 100, 300, 300], cae_ones_h5=cae_ones_h5, vgg_h5=vgg_h5) self.optimizer = O.Adam() self.optimizer.setup(self.model) img = doll() x_data = np.array([im_to_blob(im_preprocess(img))]) self.x = Variable(x_data) t_data = np.array([0], dtype=np.int32) self.t = Variable(t_data)
def dataset_to_xt_data(dataset, crop_roi): """Convert dataset to x_data and t_data""" x_data = [] for raw_path in dataset.filenames: raw = im_preprocess(imread(raw_path)) if crop_roi: mask_path = raw_to_mask_path(raw_path) roi = mask_to_roi(im_preprocess(imread(mask_path))) raw = raw[roi[0]:roi[2], roi[1]:roi[3]] raw = resize(raw, (128, 128), preserve_range=True) x_data.append(im_to_blob(raw)) x_data = np.array(x_data, dtype=np.float32) t_data = dataset.target.astype(np.int32) return x_data, t_data
def x0_to_x1(self, x0, roi_scale): on_gpu = isinstance(x0.data, cupy.ndarray) roi_scale_data = cuda.to_cpu(roi_scale.data) \ if on_gpu else roi_scale.data rois_data = (self.initial_roi * roi_scale_data).astype(int) x0_data = cuda.to_cpu(x0.data) if on_gpu else x0.data cropped = [] for i in xrange(len(x0_data)): roi = rois_data[i] im = blob_to_im(x0_data[i]) im = im[roi[0]:roi[2], roi[1]:roi[3]] if im.size == 0: break im = resize(im, (128, 128), preserve_range=True) cropped.append(im_to_blob(im)) else: cropped_data = np.array(cropped, dtype=np.float32) if on_gpu: cropped_data = cuda.to_gpu(cropped_data) x1 = Variable(cropped_data, volatile=not self.train) return x1
def cb(self, msg): bridge = cv_bridge.CvBridge() depth_tmp = bridge.imgmsg_to_cv2(msg) shape = list(depth_tmp.shape) shape.append(3) depth = np.zeros(shape, depth_tmp.dtype) depth[:,:,0] = depth_tmp depth[:,:,1] = depth_tmp depth[:,:,2] = depth_tmp depth = depth.astype(float) depth = depth / depth.max() * 255 depth = depth.astype(np.uint8) depth = resize(depth, (267, 178), preserve_range=True) x_data = np.array([apc_od.im_to_blob(depth)], dtype=np.float32) x = Variable(x_data, volatile='off') z = self.model.encode(x) initial_param = np.array([0.02]) scale = z.data tolerance = (initial_param * scale)[0][0] params = {'tolerance': tolerance} self.reconfig_client.update_configuration(params)
def test_tile_ae_inout(): im = apc_od.data.doll() x = apc_od.im_to_blob(im) x_hat = x.copy() apc_od.tile_ae_inout([x], [x_hat], '/tmp/tile_ae_inout.jpg')
def test_tile_ae_encoded(): im = apc_od.data.doll() x = apc_od.im_to_blob(im) apc_od.tile_ae_encoded([x, x], '/tmp/tile_ae_encoded.jpg')
def main(): model = CAEOnes(n_param=2) optimizer = O.Adam() optimizer.setup(model) S.load_hdf5('bof_data/cae_ones_model.h5', model) S.load_hdf5('bof_data/cae_ones_optimizer.h5', optimizer) with gzip.open('bof_data/bof_berkeley.pkl.gz', 'rb') as f: bof = pickle.load(f) with gzip.open('bof_data/lgr_merged.pkl.gz', 'rb') as f: lgr = pickle.load(f) this_dir = os.path.dirname(os.path.abspath(__file__)) data_home = os.path.abspath(os.path.join(this_dir, '../data')) train_data_dir = os.path.join(data_home, 'in_hand_recog_{}'.format('train')) test_data_dir = os.path.join(data_home, 'in_hand_recog_{}'.format('test')) train_data = load_files(train_data_dir, load_content=False) test_data = load_files(test_data_dir, load_content=False) N_train = len(train_data.filenames) N_test = len(test_data.filenames) train_imgs = [] for f in train_data.filenames: if f.endswith('_0.jpg'): # Skip mask file continue img_file = f mask_file = img_file.split('_1.jpg')[0] + '_0.jpg' img = cv2.imread(img_file, 0) mask = cv2.imread(mask_file) train_imgs.append((img, mask)) test_imgs = [] for f in test_data.filenames: if f.endswith('_0.jpg'): # Skip mask file continue img_file = f mask_file = img_file.split('_1.jpg')[0] + '_0.jpg' img = cv2.imread(img_file, 0) mask = cv2.imread(mask_file) test_imgs.append((img, mask)) y_true_0 = 12 # y_proba_true_0 = np.zeros(25, dtype=np.float32) # y_proba_true_0[12] = 1 # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> # test n_batch = 10 initial_params = [8, 4] # size, iterations epoch = 0 model.train = False accuracies = [] print('testing') N = len(test_imgs) perm = np.random.permutation(len(test_imgs)) for i in xrange(0, N, n_batch): print('test_batch: ', i) test_batch = [test_imgs[p_index] for p_index in perm[i:i+n_batch]] y_true = np.repeat([y_true_0], len(test_batch), axis=0) # y_proba_true = np.repeat(y_proba_true_0, len(test_batch), axis=0) x_data = [] for img, mask in test_batch: mask = resize(mask, (267, 178), preserve_range=True) x_data.append(apc_od.im_to_blob(mask)) x_data = np.array(x_data, dtype=np.float32) x = Variable(x_data, volatile='on') z = model.encode(x) param_scale = z.data params = param_scale * initial_params X = [] for k, param in enumerate(params): size, iterations = map(int, param) if size <= 0 or size > 50 or iterations <= 0 or iterations > 50: rand = 1. * np.ones(2) / param_scale params = rand * param_scale * initial_params size, iterations = map(int, params[0]) print('test:', size, iterations) if size <= 0 or size > 50 or iterations <= 0 or iterations > 50: size, iterations = initial_params kernel = np.ones((size, size), dtype=np.uint8) img, mask = test_batch[k] closed = cv2.morphologyEx(mask[:,:,0], cv2.MORPH_CLOSE, kernel, iterations=iterations) cropped = bounding_rect_of_mask(img, closed) frames, desc = get_sift_keypoints(cropped) X.append(desc) X = np.array(X) if X.size == 0: print('test: skipping') N -= n_batch continue X_trans = bof.transform(X) X_trans = normalize(X_trans) y_pred = lgr.predict(X_trans) accuracy = accuracy_score(y_true, y_pred) accuracies.append(accuracy) # y_proba = lgr.predict_proba(X_trans) # square_error = np.sum(np.power(y_proba - y_true, 2)) # sum_error += square_error mean_accuracy = np.array(accuracy).mean() msg = 'epoch:{:02d}; test mean accuracy={};'.format(epoch, mean_accuracy) write_log(msg) print(msg) # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< n_batch = 10 learning_n_sample = 100 learning_rate = 0.1 initial_params = [8, 4] # size, iterations for epoch in xrange(1, 11): print('epoch:', epoch) # train model.train = True sum_loss = 0 accuracies = [] N = len(train_imgs) N_train = len(train_imgs) perm = np.random.permutation(N_train) for i in range(0, N_train, n_batch): print('train_batch: ', i) train_batch = [train_imgs[p_index] for p_index in perm[i:i+n_batch]] y_true = np.repeat([y_true_0], len(train_batch), axis=0) # y_proba_true = np.repeat(y_proba_true_0, len(train_batch), axis=0) x_data = [] for img, mask in train_batch: mask = resize(mask, (267, 178), preserve_range=True) x_data.append(apc_od.im_to_blob(mask)) x_data = np.array(x_data, dtype=np.float32) x = Variable(x_data, volatile='off') z = model.encode(x) param_scale = z.data rands_shape = [learning_n_sample] + list(param_scale.shape) rands = 1. * learning_rate * (11 - epoch) / 11 * (2 * np.random.random(rands_shape) - 1) + 1 rands[0] = np.ones(param_scale.shape) # ones min_rand = None max_accuracy = -np.inf optimizer.zero_grads() for j, rand in enumerate(rands): params = rand * param_scale * initial_params X = [] for k, param in enumerate(params): size, iterations = map(int, param) if size <= 0 or size > 50 or iterations <= 0 or iterations > 50: size, iterations = initial_params kernel = np.ones((size, size), dtype=np.uint8) img, mask = train_batch[k] closed = cv2.morphologyEx(mask[:,:,0], cv2.MORPH_CLOSE, kernel, iterations=iterations) cropped = bounding_rect_of_mask(img, closed) frames, desc = get_sift_keypoints(cropped) X.append(desc) X = np.array(X) if X.size == 0: continue X_trans = bof.transform(X) X_trans = normalize(X_trans) y_pred = lgr.predict(X_trans) accuracy = accuracy_score(y_true, y_pred) # y_proba = lgr.predict_proba(X_trans)[0] # square_error = 1. * np.sum(np.power(y_proba - y_true, 2)) / len(train_batch) if accuracy > max_accuracy: max_accuracy = accuracy min_rand = rand if min_rand is None: print('train: skipping') N -= n_batch continue t_data = np.array(min_rand * param_scale, dtype=np.float32) t = Variable(t_data, volatile='off') loss = F.mean_squared_error(t, z) loss.backward() optimizer.update() sum_loss += float(loss.data) * len(train_batch) accuracies.append(accuracy) try: mean_loss = 1. * sum_loss / N except ZeroDivisionError: mean_loss = np.inf mean_accuracy = np.array(accuracies).mean() msg = 'epoch:{:02d}; train mean loss={};'.format(epoch, mean_loss) write_log(msg) print(msg) msg = 'epoch:{:02d}; train mean accuracy={};'.format(epoch, mean_accuracy) write_log(msg) print(msg) # test model.train = False sum_error = 0 print('testing') accuracies = [] N = len(test_imgs) perm = np.random.permutation(len(test_imgs)) for i in xrange(0, N, n_batch): print('test_batch: ', i) test_batch = [test_imgs[p_index] for p_index in perm[i:i+n_batch]] y_true = np.repeat([y_true_0], len(test_batch), axis=0) x_data = [] for img, mask in test_batch: mask = resize(mask, (267, 178), preserve_range=True) x_data.append(apc_od.im_to_blob(mask)) x_data = np.array(x_data, dtype=np.float32) x = Variable(x_data, volatile='on') z = model.encode(x) param_scale = z.data params = param_scale * initial_params X = [] for k, param in enumerate(params): size, iterations = map(int, param) if size <= 0 or size > 50 or iterations <= 0 or iterations > 50: rand = 1. * np.ones(2) / param_scale params = rand * param_scale * initial_params size, iterations = map(int, params[0]) print('test:', size, iterations) if size <= 0 or size > 50 or iterations <= 0 or iterations > 50: size, iterations = initial_params kernel = np.ones((size, size), dtype=np.uint8) img, mask = test_batch[k] closed = cv2.morphologyEx(mask[:,:,0], cv2.MORPH_CLOSE, kernel, iterations=iterations) cropped = bounding_rect_of_mask(img, closed) frames, desc = get_sift_keypoints(cropped) X.append(desc) X = np.array(X) if X.size == 0: print('test: skipping') N -= n_batch continue X_trans = bof.transform(X) X_trans = normalize(X_trans) y_pred = lgr.predict(X_trans) accuracy = accuracy_score(y_true, y_pred) accuracies.append(accuracy) # y_proba = lgr.predict_proba(X_trans)[0] # square_error = np.sum(np.power(y_proba - y_true, 2)) # sum_error += square_error mean_accuracy = np.array(accuracies).mean() msg = 'epoch:{:02d}; test mean accuracy={};'.format(epoch, mean_accuracy) write_log(msg) print(msg) S.save_hdf5('bof_data/cae_ones_model_trained_{}.h5'.format(epoch), model) S.save_hdf5('bof_data/cae_ones_optimizer_trained_{}.h5'.format(epoch), optimizer)
def __call__(self, x, t=None): on_gpu = isinstance(x.data, cupy.ndarray) self.cae_ones1.train = self.train self.vgg2.train = self.train roi_scale = self.cae_ones1.encode(x) if t is None: assert self.train is False if on_gpu: roi_scale_data = cuda.to_cpu(roi_scale.data) rois_data = self.initial_roi * roi_scale_data if on_gpu: rois_data = cuda.to_cpu(rois_data) x_data = cuda.to_cpu(x.data) rois_data = rois_data.astype(int) cropped = [] for i in xrange(len(x_data)): roi = rois_data[i] im = blob_to_im(x_data[i]) im = im[roi[0]:roi[2], roi[1]:roi[3]] im = resize(im, (128, 128), preserve_range=True) cropped.append(im_to_blob(im)) cropped = np.array(cropped, dtype=np.float32) if on_gpu: cropped = cuda.to_gpu(cropped) cropped = Variable(cropped, volatile=not self.train) self.vgg2(cropped) self.y = self.vgg2.y return self.y # randomly change the param and estimate good parameter for the task min_y = None rands_shape = [self.learning_n_sample] + list(roi_scale.data.shape) rands = self.learning_rate * (2 * np.random.random(rands_shape) - 1) + 1 rands[0] = np.ones(roi_scale.data.shape) for i, rand in enumerate(rands): if on_gpu: roi_scale_data = cuda.to_cpu(roi_scale.data) rois_data = rand * (self.initial_roi * roi_scale_data) x_data = cuda.to_cpu(x.data) skip = False rois_data = rois_data.astype(int) cropped = [] for j in xrange(len(x_data)): roi = rois_data[j] im = blob_to_im(x_data[j]) im = im[roi[0]:roi[2], roi[1]:roi[3]] if im.size == 0: skip = True break im = resize(im, (128, 128), preserve_range=True) cropped.append(im_to_blob(im)) if skip: continue cropped = np.array(cropped) if on_gpu: cropped = cuda.to_gpu(cropped) cropped = Variable(cropped, volatile=not self.train) self.vgg2(cropped, t) h = self.vgg2.y loss = F.softmax_cross_entropy(h, t) if min_y is None: min_loss_data = float(loss.data) min_y = h min_loss = loss min_rand = rand min_rois = rois_data elif min_loss_data > float(loss.data): min_loss_data = float(loss.data) min_y = h min_loss = loss min_rand = rand min_rois = rois_data if on_gpu: min_rand = cuda.to_gpu(min_rand) rois_data = min_rand * roi_scale.data xp = cuda.get_array_module(rois_data) rois_data = rois_data.astype(xp.float32) rois = Variable(rois_data, volatile=not self.train) loss1 = F.mean_squared_error(roi_scale, rois) loss2 = min_loss self.loss = loss1 + loss2 self.accuracy = F.accuracy(min_y, t) return self.loss
def main(): rospy.init_node('train_inbin_reconfig') update_filename_client = rospy.ServiceProxy( 'pcd_to_pointcloud/update_filename', UpdateFilename) parser = argparse.ArgumentParser() parser.add_argument('container_dir') args = parser.parse_args() model = CAEOnes(n_param=1) optimizer = O.Adam() optimizer.setup(model) print('loading hd5') S.load_hdf5('cae_ones_model_inbin.h5', model) S.load_hdf5('cae_ones_optimizer_inbin.h5', optimizer) print('done loading') directory = args.container_dir # test epoch = 0 sum_acc = 0 for i in [4, 5]: pcd_file = os.path.realpath(glob.glob('{}/{}/*.pcd'.format(directory, i))[0]) req = UpdateFilenameRequest(filename=pcd_file) res = update_filename_client(req) print('update_filename to: {}, success: {}' .format(pcd_file, res.success)) image_file = os.path.join(directory, str(i), 'image.jpg') depth_file = os.path.join(directory, str(i), 'depth.jpg') all_mask_file = os.path.join(directory, str(i), 'mask.jpg') diffmask_file = os.path.join(directory, str(i), 'diffmask2.jpg') image = cv2.imread(image_file) depth = cv2.imread(depth_file) all_mask = cv2.imread(all_mask_file, 0) mask = cv2.imread(diffmask_file, 0) depth = resize(depth, (267, 178), preserve_range=True) x_data = np.array([apc_od.im_to_blob(depth)], dtype=np.float32) x = Variable(x_data, volatile='on') trainer = TrainInBinReconfig(model=model, optimizer=optimizer, x=x, mask=mask, image=image, all_mask=all_mask) model.train = False param_scale = model.encode(x).data params = (trainer.initial_param * param_scale)[0] tolerance = params[0] if params[0] < 0.0009: params[0] = trainer.initial_param[0] trainer.reconfigure(tolerance=params[0]) trainer.reconfigure(tolerance=tolerance) val, label = trainer.evaluate() save_label_and_img('{}_{}.jpg'.format(i, epoch), label, image) sum_acc += (1. - val) mean_acc = sum_acc / 2. msg = 'epoch:{:02d}; test mean accuracy0={};'.format(epoch, mean_acc) write_log(msg) for epoch in xrange(1, 21): sum_loss = 0 sum_acc = 0 for i in [1, 2, 3]: pcd_file = os.path.realpath(glob.glob('{}/{}/*.pcd'.format(directory, i))[0]) req = UpdateFilenameRequest(filename=pcd_file) res = update_filename_client(req) print('update_filename to: {}, success: {}' .format(pcd_file, res.success)) image_file = os.path.join(directory, str(i), 'image.jpg') depth_file = os.path.join(directory, str(i), 'depth.jpg') all_mask_file = os.path.join(directory, str(i), 'mask.jpg') diffmask_file = os.path.join(directory, str(i), 'diffmask2.jpg') image = cv2.imread(image_file) depth = cv2.imread(depth_file) all_mask = cv2.imread(all_mask_file, 0) mask = cv2.imread(diffmask_file, 0) depth = resize(depth, (267, 178), preserve_range=True) x_data = np.array([apc_od.im_to_blob(depth)], dtype=np.float32) x = Variable(x_data, volatile='off') trainer = TrainInBinReconfig( model=model, optimizer=optimizer, x=x, mask=mask, image=image, all_mask=all_mask) model.train = True loss_data, acc = trainer.random_sample(fname='{}_{}.jpg'.format(i, epoch), epoch=epoch) sum_loss += loss_data sum_acc += acc mean_loss = sum_loss / 3. mean_acc = sum_acc / 3. msg = 'epoch:{:02d}; train mean loss0={};'.format(epoch, mean_loss) write_log(msg) msg = 'epoch:{:02d}; train mean accuracy0={};'.format(epoch, mean_acc) write_log(msg) # test sum_acc = 0 for i in [4, 5]: pcd_file = os.path.realpath(glob.glob('{}/{}/*.pcd'.format(directory, i))[0]) req = UpdateFilenameRequest(filename=pcd_file) res = update_filename_client(req) print('update_filename to: {}, success: {}' .format(pcd_file, res.success)) image_file = os.path.join(directory, str(i), 'image.jpg') depth_file = os.path.join(directory, str(i), 'depth.jpg') all_mask_file = os.path.join(directory, str(i), 'mask.jpg') diffmask_file = os.path.join(directory, str(i), 'diffmask2.jpg') image = cv2.imread(image_file) depth = cv2.imread(depth_file) all_mask = cv2.imread(all_mask_file, 0) mask = cv2.imread(diffmask_file, 0) depth = resize(depth, (267, 178), preserve_range=True) x_data = np.array([apc_od.im_to_blob(depth)], dtype=np.float32) x = Variable(x_data, volatile='on') trainer = TrainInBinReconfig(model=model, optimizer=optimizer, x=x, mask=mask, image=image, all_mask=all_mask) model.train = False param_scale = model.encode(x).data params = (trainer.initial_param * param_scale)[0] if params[0] < 0.0009: params[0] = trainer.initial_param[0] tolerance = params[0] trainer.reconfigure(tolerance=tolerance) val, label = trainer.evaluate() save_label_and_img('{}_{}.jpg'.format(i, epoch), label, image) sum_acc += (1. - val) mean_acc = sum_acc / 2. msg = 'epoch:{:02d}; test mean accuracy0={};'.format(epoch, mean_acc) write_log(msg) S.save_hdf5('cae_ones_model_inbin_trained.h5', model) S.save_hdf5('cae_ones_optimizer_inbin_trained.h5', optimizer)