def test_pickle(self): # test tensor for dtype in [np.float32, np.float64, np.int32, np.int64]: array = np.array( [[[3, 21, 1], [22, 1, 5]], [[2, 2, 2], [7, 7, 7]]], dtype=dtype) b1 = buffers.as_tensor_buffer(array) pickle.dump(b1, open('tmp.pkl', 'wb')) b2 = pickle.load(open('tmp.pkl', 'rb')) array2 = buffers.as_numpy_array(b2) self.assertTrue((array == array2).all()) # test matrix for dtype in [np.float32, np.float64, np.int32, np.int64]: array = np.array([[3, 21, 1], [22, 1, 5], [2, 2, 2], [7, 7, 7]], dtype=dtype) b1 = buffers.as_matrix_buffer(array) pickle.dump(b1, open('tmp.pkl', 'wb')) b2 = pickle.load(open('tmp.pkl', 'rb')) array2 = buffers.as_numpy_array(b2) self.assertTrue((array == array2).all()) # test vector for dtype in [np.float32, np.float64, np.int32, np.int64]: array = np.array([3, 21, 1], dtype=dtype) b1 = buffers.as_vector_buffer(array) pickle.dump(b1, open('tmp.pkl', 'wb')) b2 = pickle.load(open('tmp.pkl', 'rb')) array2 = buffers.as_numpy_array(b2) self.assertTrue((array == array2).all())
def test_pickle(self): # test tensor for dtype in [np.float32, np.float64, np.int32, np.int64]: array = np.array([[[3, 21, 1], [22, 1, 5]], [[2, 2, 2], [7, 7, 7]]], dtype=dtype) b1 = buffers.as_tensor_buffer(array) pickle.dump(b1, open("tmp.pkl", "wb")) b2 = pickle.load(open("tmp.pkl", "rb")) array2 = buffers.as_numpy_array(b2) self.assertTrue((array == array2).all()) # test matrix for dtype in [np.float32, np.float64, np.int32, np.int64]: array = np.array([[3, 21, 1], [22, 1, 5], [2, 2, 2], [7, 7, 7]], dtype=dtype) b1 = buffers.as_matrix_buffer(array) pickle.dump(b1, open("tmp.pkl", "wb")) b2 = pickle.load(open("tmp.pkl", "rb")) array2 = buffers.as_numpy_array(b2) self.assertTrue((array == array2).all()) # test vector for dtype in [np.float32, np.float64, np.int32, np.int64]: array = np.array([3, 21, 1], dtype=dtype) b1 = buffers.as_vector_buffer(array) pickle.dump(b1, open("tmp.pkl", "wb")) b2 = pickle.load(open("tmp.pkl", "rb")) array2 = buffers.as_numpy_array(b2) self.assertTrue((array == array2).all())
def matrix_buffer_flatten_helper(self, X, buffer_type): n = len(X.shape) matrix_buffer = buffers.as_matrix_buffer(X) assert isinstance(matrix_buffer, buffer_type) X_back = buffers.as_numpy_array(matrix_buffer) self.assertTrue((X == X_back.flatten()).all()) X_back = buffers.as_numpy_array(buffer=matrix_buffer, flatten=True) self.assertTrue((X == X_back).all())
def img_buffer_flatten_helper(self, X, buffer_type): m,n = X.shape img_buffer = buffers.as_tensor_buffer(X) assert isinstance(img_buffer, buffer_type) X_back = buffers.as_numpy_array(img_buffer) self.assertTrue((X.reshape(1,m,n) == X_back).all()) X_back = buffers.as_numpy_array(buffer=img_buffer, flatten=True) self.assertTrue((X == X_back).all())
def test_tree_pickle(self): forest = self.construct_axis_aligned_forest() tree = forest.GetTree(1) pickle.dump(tree, open('tmp.pkl', 'wb')) tree2 = pickle.load(open('tmp.pkl', 'rb')) self.assertAlmostEqual(tree2.GetYs().Get(2,0), 0.8, places=7) self.assertEqual(tree2.GetCounts().Get(3), 5) b1 = buffers.as_numpy_array(tree.GetExtraInfo().GetBuffer("first")) b2 = buffers.as_numpy_array(tree2.GetExtraInfo().GetBuffer("first")) self.assertTrue((b1 == b2).all())
def test_tree_pickle(self): forest = self.construct_axis_aligned_forest() tree = forest.GetTree(1) pickle.dump(tree, open('tmp.pkl', 'wb')) tree2 = pickle.load(open('tmp.pkl', 'rb')) self.assertAlmostEqual(tree2.GetYs().Get(2, 0), 0.8, places=7) self.assertEqual(tree2.GetCounts().Get(3), 5) b1 = buffers.as_numpy_array(tree.GetExtraInfo().GetBuffer("first")) b2 = buffers.as_numpy_array(tree2.GetExtraInfo().GetBuffer("first")) self.assertTrue((b1 == b2).all())
def classify_body_pixels(depth, ground_labels, forest): assert(depth.ndim == 2) # setup test data pixel_indices = to_indices(0, np.where(ground_labels != background)) buffer_collection = buffers.BufferCollection() buffer_collection.AddInt32MatrixBuffer(buffers.PIXEL_INDICES, buffers.as_matrix_buffer(pixel_indices)) buffer_collection.AddFloat32Tensor3Buffer(buffers.DEPTH_IMAGES, buffers.as_tensor_buffer(depth)) # setup predictor all_samples_step = pipeline.AllSamplesStep_i32f32i32(buffers.PIXEL_INDICES) depth_delta_feature = image_features.ScaledDepthDeltaFeature_f32i32(all_samples_step.IndicesBufferId, buffers.PIXEL_INDICES, buffers.DEPTH_IMAGES) combiner = classification.ClassProbabilityCombiner_f32(number_of_body_parts) forest_predictor = predict.ScaledDepthDeltaClassificationPredictin_f32i32(forest, depth_delta_feature, combiner, all_samples_step) # predict yprobs_buffer = buffers.Float32MatrixBuffer() forest_predictor.PredictYs(buffer_collection, yprobs_buffer) # convert to image space yprobs = buffers.as_numpy_array(yprobs_buffer) (_, ydim) = yprobs.shape m,n = depth.shape img_yprobs = np.zeros((m,n), dtype=np.float32) img_yprobs[ground_labels != background].shape yprobs.max(axis=1).shape img_yprobs[ground_labels != background] = yprobs.max(axis=1) img_yhat = np.zeros((m,n), dtype=np.int32) img_yhat[ground_labels != background] = np.argmax(yprobs, axis=1) return img_yhat, img_yprobs
def classify_pixels(depth, forest): assert(depth.ndim == 2) # setup test data pixel_indices = np.array( list(itertools.product( np.zeros(1), range(m), range(n) )), dtype=np.int32 ) buffer_collection = buffers.BufferCollection() buffer_collection.AddInt32MatrixBuffer(buffers.PIXEL_INDICES, buffers.as_matrix_buffer(pixel_indices)) buffer_collection.AddFloat32Tensor3Buffer(buffers.DEPTH_IMAGES, buffers.as_tensor_buffer(depth)) # setup predictor all_samples_step = pipeline.AllSamplesStep_i32f32i32(buffers.PIXEL_INDICES) depth_delta_feature = image_features.ScaledDepthDeltaFeature_f32i32(all_samples_step.IndicesBufferId, buffers.PIXEL_INDICES, buffers.DEPTH_IMAGES) combiner = classification.ClassProbabilityCombiner_f32(number_of_body_parts) forest_predictor = predict.ScaledDepthDeltaClassificationPredictin_f32i32(forest, depth_delta_feature, combiner, all_samples_step) # predict yprobs_buffer = buffers.Float32MatrixBuffer() forest_predictor.PredictYs(buffer_collection, yprobs_buffer) # convert to image space yprobs = buffers.as_numpy_array(yprobs_buffer) (_, ydim) = yprobs.shape img_yprobs = yprobs.reshape((m,n,ydim)) img_yhat = np.argmax(img_yprobs, axis=2) return img_yhat, img_yprobs.max(axis=2)
def test_vector(self): data_1 = np.array([3,21,1,22,1,5], dtype=np.float32) collection = buffers.BufferCollection(first=data_1) self.assertEqual(collection.HasBuffer("first"), True) self.assertEqual(collection.HasFloat32VectorBuffer("first"), True) self.assertEqual(collection.HasFloat64VectorBuffer("first"), False) self.assertEqual(collection.HasInt32VectorBuffer("first"), False) self.assertEqual(collection.HasInt64VectorBuffer("first"), False) self.assertEqual(collection.HasBuffer("second"), False) data_2 = np.array([3,21,1], dtype=np.float64 ) collection.AddBuffer("second", data_2) self.assertEqual(collection.HasBuffer("second"), True) self.assertEqual(collection.HasFloat32VectorBuffer("second"), False) self.assertEqual(collection.HasFloat64VectorBuffer("second"), True) self.assertEqual(collection.HasInt32VectorBuffer("second"), False) self.assertEqual(collection.HasInt64VectorBuffer("second"), False) self.assertEqual(collection.HasBuffer("third"), False) data_3 = np.array([22,1,5], dtype=np.int32 ) collection.AddBuffer("third", data_3) self.assertEqual(collection.HasFloat32VectorBuffer("third"), False) self.assertEqual(collection.HasFloat64VectorBuffer("third"), False) self.assertEqual(collection.HasInt32VectorBuffer("third"), True) self.assertEqual(collection.HasInt64VectorBuffer("third"), False) self.assertEqual(collection.HasBuffer("forth"), False) data_4 = np.array([1,1,1], dtype=np.int64 ) collection.AddBuffer("forth", data_4) self.assertEqual(collection.HasFloat32VectorBuffer("forth"), False) self.assertEqual(collection.HasFloat64VectorBuffer("forth"), False) self.assertEqual(collection.HasInt32VectorBuffer("forth"), False) self.assertEqual(collection.HasInt64VectorBuffer("forth"), True) data_1_out = buffers.as_numpy_array(collection.GetBuffer("first")) self.assertTrue((data_1 == data_1_out).all()) data_2_out = buffers.as_numpy_array(collection.GetBuffer("second")) self.assertTrue((data_2 == data_2_out).all()) data_3_out = buffers.as_numpy_array(collection.GetBuffer("third")) self.assertTrue((data_3 == data_3_out).all()) data_4_out = buffers.as_numpy_array(collection.GetBuffer("forth")) self.assertTrue((data_4 == data_4_out).all())
def predict(self, **kwargs): result = buffers.Float32MatrixBuffer() bufferCollection = self.prepare_data(**kwargs) if 'tree_weights' in kwargs: tree_weights = kwargs.get('tree_weights') self.forest_predictor.PredictYs(bufferCollection, tree_weights, result) else: self.forest_predictor.PredictYs(bufferCollection, result) return buffers.as_numpy_array(result)
def test_pickle(self): import pickle data_1 = np.array([3,21,1,22,1,5], dtype=np.float32) collection = buffers.BufferCollection(first=data_1) data_2 = np.array([[3,21,1],[22,1,5]], dtype=np.float64 ) collection.AddBuffer("second", data_2) data_3 = np.array([[[3,21,1]],[[22,1,5]]], dtype=np.int32 ) collection.AddBuffer("third", data_3) pickle.dump(collection, open('tmp.pkl', 'wb')) collection2 = pickle.load(open('tmp.pkl', 'rb')) data_1_out = buffers.as_numpy_array(collection2.GetBuffer("first")) self.assertTrue((data_1 == data_1_out).all()) data_2_out = buffers.as_numpy_array(collection2.GetBuffer("second")) self.assertTrue((data_2 == data_2_out).all()) data_3_out = buffers.as_numpy_array(collection2.GetBuffer("third")) self.assertTrue((data_3 == data_3_out).all())
def predict_proba(self, x): buffer_collection = buffers.BufferCollection() buffer_collection.AddFloat32MatrixBuffer(buffers.X_FLOAT_DATA, buffers.as_matrix_buffer(x)) number_of_classes = self.forest_data.GetTree(0).GetYs().GetN() all_samples_step = pipeline.AllSamplesStep_f32f32i32(buffers.X_FLOAT_DATA) combiner = classification.ClassProbabilityCombiner_f32(number_of_classes) matrix_feature = matrix_features.LinearFloat32MatrixFeature_f32i32(all_samples_step.IndicesBufferId, buffers.X_FLOAT_DATA) forest_predicter = predict.LinearMatrixClassificationPredictin_f32i32(forest_data, matrix_feature, combiner, all_samples_step) result = buffers.Float32MatrixBuffer() forest_predicter.PredictYs(bufferCollection, result) return buffers.as_numpy_array(result)
def as_pyforest(native_forest): pytrees = [] for tree_id in range(native_forest.GetNumberOfTrees()): native_tree = native_forest.GetTree(tree_id) native_tree.Compact() pytree = PyTree() pytree.path = buffers.as_numpy_array(native_tree.mPath) pytree.int_features_params = buffers.as_numpy_array(native_tree.mIntFeatureParams) pytree.float_features_params = buffers.as_numpy_array(native_tree.mFloatFeatureParams) pytree.depths = buffers.as_numpy_array(native_tree.mDepths, flatten=True) pytree.counts = buffers.as_numpy_array(native_tree.mCounts, flatten=True) pytree.ys = buffers.as_numpy_array(native_tree.mYs) pytrees.append(pytree) return pytrees
def as_pyforest(native_forest): pytrees = [] for tree_id in range(native_forest.GetNumberOfTrees()): native_tree = native_forest.GetTree(tree_id) native_tree.Compact() pytree = PyTree() pytree.native_tree = native_tree pytree.path = buffers.as_numpy_array(native_tree.GetPath()) pytree.int_features_params = buffers.as_numpy_array(native_tree.GetIntFeatureParams()) pytree.float_features_params = buffers.as_numpy_array(native_tree.GetFloatFeatureParams()) pytree.depths = buffers.as_numpy_array(native_tree.GetDepths(), flatten=True) pytree.counts = buffers.as_numpy_array(native_tree.GetCounts(), flatten=True) pytree.ys = buffers.as_numpy_array(native_tree.GetYs()) pytree.extra_info = native_tree.GetExtraInfo() pytrees.append(pytree) return pytrees
def as_pyforest(native_forest): pytrees = [] for tree_id in range(native_forest.GetNumberOfTrees()): native_tree = native_forest.GetTree(tree_id) native_tree.Compact() pytree = PyTree() pytree.native_tree = native_tree pytree.path = buffers.as_numpy_array(native_tree.GetPath()) pytree.int_features_params = buffers.as_numpy_array( native_tree.GetIntFeatureParams()) pytree.float_features_params = buffers.as_numpy_array( native_tree.GetFloatFeatureParams()) pytree.depths = buffers.as_numpy_array(native_tree.GetDepths(), flatten=True) pytree.counts = buffers.as_numpy_array(native_tree.GetCounts(), flatten=True) pytree.ys = buffers.as_numpy_array(native_tree.GetYs()) pytree.extra_info = native_tree.GetExtraInfo() pytrees.append(pytree) return pytrees
def predict(self, **kwargs): result = buffers.Float32MatrixBuffer() bufferCollection = self.prepare_data(**kwargs) self.forest_predictor.PredictYs(bufferCollection, result) return buffers.as_numpy_array(result)
def predict_leafs_ys(self, **kwargs): oob_weights = buffers.Float32MatrixBuffer() leaf_ys = buffers.Float32Tensor3Buffer() bufferCollection = self.prepare_data(**kwargs) self.forest_predictor.PredictLeafYs(bufferCollection, oob_weights, leaf_ys) return buffers.as_numpy_array(leaf_ys), buffers.as_numpy_array(oob_weights)
def convert_img_buffer_both_directions_helper(self, X, buffer_type): buf = buffers.as_tensor_buffer(X) assert isinstance(buf, buffer_type) X_back = buffers.as_numpy_array(buf) self.assertTrue((X == X_back).all())
def y_hat_sum(self, tree_weights, oob_weights, leaf_ys, **kwargs): tree_weights = buffers.as_numpy_array(tree_weights) weights = oob_weights * tree_weights weighted_ys = weights * leaf_ys y_hat_sum = weighted_ys.sum(axis=2) return y_hat_sum, weights, weights.sum(axis=1)