def test_pickle(self):
# test tensor
for dtype in [np.float32, np.float64, np.int32, np.int64]:
array = np.array(
[[[3, 21, 1], [22, 1, 5]], [[2, 2, 2], [7, 7, 7]]],
dtype=dtype)
b1 = buffers.as_tensor_buffer(array)
pickle.dump(b1, open('tmp.pkl', 'wb'))
b2 = pickle.load(open('tmp.pkl', 'rb'))
array2 = buffers.as_numpy_array(b2)
self.assertTrue((array == array2).all())
# test matrix
for dtype in [np.float32, np.float64, np.int32, np.int64]:
array = np.array([[3, 21, 1], [22, 1, 5], [2, 2, 2], [7, 7, 7]],
dtype=dtype)
b1 = buffers.as_matrix_buffer(array)
pickle.dump(b1, open('tmp.pkl', 'wb'))
b2 = pickle.load(open('tmp.pkl', 'rb'))
array2 = buffers.as_numpy_array(b2)
self.assertTrue((array == array2).all())
# test vector
for dtype in [np.float32, np.float64, np.int32, np.int64]:
array = np.array([3, 21, 1], dtype=dtype)
b1 = buffers.as_vector_buffer(array)
pickle.dump(b1, open('tmp.pkl', 'wb'))
b2 = pickle.load(open('tmp.pkl', 'rb'))
array2 = buffers.as_numpy_array(b2)
self.assertTrue((array == array2).all())
def test_pickle(self):
# test tensor
for dtype in [np.float32, np.float64, np.int32, np.int64]:
array = np.array([[[3, 21, 1], [22, 1, 5]], [[2, 2, 2], [7, 7, 7]]], dtype=dtype)
b1 = buffers.as_tensor_buffer(array)
pickle.dump(b1, open("tmp.pkl", "wb"))
b2 = pickle.load(open("tmp.pkl", "rb"))
array2 = buffers.as_numpy_array(b2)
self.assertTrue((array == array2).all())
# test matrix
for dtype in [np.float32, np.float64, np.int32, np.int64]:
array = np.array([[3, 21, 1], [22, 1, 5], [2, 2, 2], [7, 7, 7]], dtype=dtype)
b1 = buffers.as_matrix_buffer(array)
pickle.dump(b1, open("tmp.pkl", "wb"))
b2 = pickle.load(open("tmp.pkl", "rb"))
array2 = buffers.as_numpy_array(b2)
self.assertTrue((array == array2).all())
# test vector
for dtype in [np.float32, np.float64, np.int32, np.int64]:
array = np.array([3, 21, 1], dtype=dtype)
b1 = buffers.as_vector_buffer(array)
pickle.dump(b1, open("tmp.pkl", "wb"))
b2 = pickle.load(open("tmp.pkl", "rb"))
array2 = buffers.as_numpy_array(b2)
self.assertTrue((array == array2).all())
def matrix_buffer_flatten_helper(self, X, buffer_type):
n = len(X.shape)
matrix_buffer = buffers.as_matrix_buffer(X)
assert isinstance(matrix_buffer, buffer_type)
X_back = buffers.as_numpy_array(matrix_buffer)
self.assertTrue((X == X_back.flatten()).all())
X_back = buffers.as_numpy_array(buffer=matrix_buffer, flatten=True)
self.assertTrue((X == X_back).all())
def img_buffer_flatten_helper(self, X, buffer_type):
m,n = X.shape
img_buffer = buffers.as_tensor_buffer(X)
assert isinstance(img_buffer, buffer_type)
X_back = buffers.as_numpy_array(img_buffer)
self.assertTrue((X.reshape(1,m,n) == X_back).all())
X_back = buffers.as_numpy_array(buffer=img_buffer, flatten=True)
self.assertTrue((X == X_back).all())
def test_tree_pickle(self):
forest = self.construct_axis_aligned_forest()
tree = forest.GetTree(1)
pickle.dump(tree, open('tmp.pkl', 'wb'))
tree2 = pickle.load(open('tmp.pkl', 'rb'))
self.assertAlmostEqual(tree2.GetYs().Get(2,0), 0.8, places=7)
self.assertEqual(tree2.GetCounts().Get(3), 5)
b1 = buffers.as_numpy_array(tree.GetExtraInfo().GetBuffer("first"))
b2 = buffers.as_numpy_array(tree2.GetExtraInfo().GetBuffer("first"))
self.assertTrue((b1 == b2).all())
def test_tree_pickle(self):
forest = self.construct_axis_aligned_forest()
tree = forest.GetTree(1)
pickle.dump(tree, open('tmp.pkl', 'wb'))
tree2 = pickle.load(open('tmp.pkl', 'rb'))
self.assertAlmostEqual(tree2.GetYs().Get(2, 0), 0.8, places=7)
self.assertEqual(tree2.GetCounts().Get(3), 5)
b1 = buffers.as_numpy_array(tree.GetExtraInfo().GetBuffer("first"))
b2 = buffers.as_numpy_array(tree2.GetExtraInfo().GetBuffer("first"))
self.assertTrue((b1 == b2).all())
def classify_body_pixels(depth, ground_labels, forest):
assert(depth.ndim == 2)
# setup test data
pixel_indices = to_indices(0, np.where(ground_labels != background))
buffer_collection = buffers.BufferCollection()
buffer_collection.AddInt32MatrixBuffer(buffers.PIXEL_INDICES, buffers.as_matrix_buffer(pixel_indices))
buffer_collection.AddFloat32Tensor3Buffer(buffers.DEPTH_IMAGES, buffers.as_tensor_buffer(depth))
# setup predictor
all_samples_step = pipeline.AllSamplesStep_i32f32i32(buffers.PIXEL_INDICES)
depth_delta_feature = image_features.ScaledDepthDeltaFeature_f32i32(all_samples_step.IndicesBufferId,
buffers.PIXEL_INDICES,
buffers.DEPTH_IMAGES)
combiner = classification.ClassProbabilityCombiner_f32(number_of_body_parts)
forest_predictor = predict.ScaledDepthDeltaClassificationPredictin_f32i32(forest, depth_delta_feature, combiner, all_samples_step)
# predict
yprobs_buffer = buffers.Float32MatrixBuffer()
forest_predictor.PredictYs(buffer_collection, yprobs_buffer)
# convert to image space
yprobs = buffers.as_numpy_array(yprobs_buffer)
(_, ydim) = yprobs.shape
m,n = depth.shape
img_yprobs = np.zeros((m,n), dtype=np.float32)
img_yprobs[ground_labels != background].shape
yprobs.max(axis=1).shape
img_yprobs[ground_labels != background] = yprobs.max(axis=1)
img_yhat = np.zeros((m,n), dtype=np.int32)
img_yhat[ground_labels != background] = np.argmax(yprobs, axis=1)
return img_yhat, img_yprobs
def classify_pixels(depth, forest):
assert(depth.ndim == 2)
# setup test data
pixel_indices = np.array( list(itertools.product( np.zeros(1), range(m), range(n) )), dtype=np.int32 )
buffer_collection = buffers.BufferCollection()
buffer_collection.AddInt32MatrixBuffer(buffers.PIXEL_INDICES, buffers.as_matrix_buffer(pixel_indices))
buffer_collection.AddFloat32Tensor3Buffer(buffers.DEPTH_IMAGES, buffers.as_tensor_buffer(depth))
# setup predictor
all_samples_step = pipeline.AllSamplesStep_i32f32i32(buffers.PIXEL_INDICES)
depth_delta_feature = image_features.ScaledDepthDeltaFeature_f32i32(all_samples_step.IndicesBufferId,
buffers.PIXEL_INDICES,
buffers.DEPTH_IMAGES)
combiner = classification.ClassProbabilityCombiner_f32(number_of_body_parts)
forest_predictor = predict.ScaledDepthDeltaClassificationPredictin_f32i32(forest, depth_delta_feature, combiner, all_samples_step)
# predict
yprobs_buffer = buffers.Float32MatrixBuffer()
forest_predictor.PredictYs(buffer_collection, yprobs_buffer)
# convert to image space
yprobs = buffers.as_numpy_array(yprobs_buffer)
(_, ydim) = yprobs.shape
img_yprobs = yprobs.reshape((m,n,ydim))
img_yhat = np.argmax(img_yprobs, axis=2)
return img_yhat, img_yprobs.max(axis=2)
def test_vector(self):
data_1 = np.array([3,21,1,22,1,5], dtype=np.float32)
collection = buffers.BufferCollection(first=data_1)
self.assertEqual(collection.HasBuffer("first"), True)
self.assertEqual(collection.HasFloat32VectorBuffer("first"), True)
self.assertEqual(collection.HasFloat64VectorBuffer("first"), False)
self.assertEqual(collection.HasInt32VectorBuffer("first"), False)
self.assertEqual(collection.HasInt64VectorBuffer("first"), False)
self.assertEqual(collection.HasBuffer("second"), False)
data_2 = np.array([3,21,1], dtype=np.float64 )
collection.AddBuffer("second", data_2)
self.assertEqual(collection.HasBuffer("second"), True)
self.assertEqual(collection.HasFloat32VectorBuffer("second"), False)
self.assertEqual(collection.HasFloat64VectorBuffer("second"), True)
self.assertEqual(collection.HasInt32VectorBuffer("second"), False)
self.assertEqual(collection.HasInt64VectorBuffer("second"), False)
self.assertEqual(collection.HasBuffer("third"), False)
data_3 = np.array([22,1,5], dtype=np.int32 )
collection.AddBuffer("third", data_3)
self.assertEqual(collection.HasFloat32VectorBuffer("third"), False)
self.assertEqual(collection.HasFloat64VectorBuffer("third"), False)
self.assertEqual(collection.HasInt32VectorBuffer("third"), True)
self.assertEqual(collection.HasInt64VectorBuffer("third"), False)
self.assertEqual(collection.HasBuffer("forth"), False)
data_4 = np.array([1,1,1], dtype=np.int64 )
collection.AddBuffer("forth", data_4)
self.assertEqual(collection.HasFloat32VectorBuffer("forth"), False)
self.assertEqual(collection.HasFloat64VectorBuffer("forth"), False)
self.assertEqual(collection.HasInt32VectorBuffer("forth"), False)
self.assertEqual(collection.HasInt64VectorBuffer("forth"), True)
data_1_out = buffers.as_numpy_array(collection.GetBuffer("first"))
self.assertTrue((data_1 == data_1_out).all())
data_2_out = buffers.as_numpy_array(collection.GetBuffer("second"))
self.assertTrue((data_2 == data_2_out).all())
data_3_out = buffers.as_numpy_array(collection.GetBuffer("third"))
self.assertTrue((data_3 == data_3_out).all())
data_4_out = buffers.as_numpy_array(collection.GetBuffer("forth"))
self.assertTrue((data_4 == data_4_out).all())
def predict(self, **kwargs):
result = buffers.Float32MatrixBuffer()
bufferCollection = self.prepare_data(**kwargs)
if 'tree_weights' in kwargs:
tree_weights = kwargs.get('tree_weights')
self.forest_predictor.PredictYs(bufferCollection, tree_weights, result)
else:
self.forest_predictor.PredictYs(bufferCollection, result)
return buffers.as_numpy_array(result)
def test_pickle(self):
import pickle
data_1 = np.array([3,21,1,22,1,5], dtype=np.float32)
collection = buffers.BufferCollection(first=data_1)
data_2 = np.array([[3,21,1],[22,1,5]], dtype=np.float64 )
collection.AddBuffer("second", data_2)
data_3 = np.array([[[3,21,1]],[[22,1,5]]], dtype=np.int32 )
collection.AddBuffer("third", data_3)
pickle.dump(collection, open('tmp.pkl', 'wb'))
collection2 = pickle.load(open('tmp.pkl', 'rb'))
data_1_out = buffers.as_numpy_array(collection2.GetBuffer("first"))
self.assertTrue((data_1 == data_1_out).all())
data_2_out = buffers.as_numpy_array(collection2.GetBuffer("second"))
self.assertTrue((data_2 == data_2_out).all())
data_3_out = buffers.as_numpy_array(collection2.GetBuffer("third"))
self.assertTrue((data_3 == data_3_out).all())
def predict_proba(self, x):
buffer_collection = buffers.BufferCollection()
buffer_collection.AddFloat32MatrixBuffer(buffers.X_FLOAT_DATA, buffers.as_matrix_buffer(x))
number_of_classes = self.forest_data.GetTree(0).GetYs().GetN()
all_samples_step = pipeline.AllSamplesStep_f32f32i32(buffers.X_FLOAT_DATA)
combiner = classification.ClassProbabilityCombiner_f32(number_of_classes)
matrix_feature = matrix_features.LinearFloat32MatrixFeature_f32i32(all_samples_step.IndicesBufferId,
buffers.X_FLOAT_DATA)
forest_predicter = predict.LinearMatrixClassificationPredictin_f32i32(forest_data, matrix_feature, combiner, all_samples_step)
result = buffers.Float32MatrixBuffer()
forest_predicter.PredictYs(bufferCollection, result)
return buffers.as_numpy_array(result)
def as_pyforest(native_forest):
pytrees = []
for tree_id in range(native_forest.GetNumberOfTrees()):
native_tree = native_forest.GetTree(tree_id)
native_tree.Compact()
pytree = PyTree()
pytree.path = buffers.as_numpy_array(native_tree.mPath)
pytree.int_features_params = buffers.as_numpy_array(native_tree.mIntFeatureParams)
pytree.float_features_params = buffers.as_numpy_array(native_tree.mFloatFeatureParams)
pytree.depths = buffers.as_numpy_array(native_tree.mDepths, flatten=True)
pytree.counts = buffers.as_numpy_array(native_tree.mCounts, flatten=True)
pytree.ys = buffers.as_numpy_array(native_tree.mYs)
pytrees.append(pytree)
return pytrees
def as_pyforest(native_forest):
pytrees = []
for tree_id in range(native_forest.GetNumberOfTrees()):
native_tree = native_forest.GetTree(tree_id)
native_tree.Compact()
pytree = PyTree()
pytree.native_tree = native_tree
pytree.path = buffers.as_numpy_array(native_tree.GetPath())
pytree.int_features_params = buffers.as_numpy_array(native_tree.GetIntFeatureParams())
pytree.float_features_params = buffers.as_numpy_array(native_tree.GetFloatFeatureParams())
pytree.depths = buffers.as_numpy_array(native_tree.GetDepths(), flatten=True)
pytree.counts = buffers.as_numpy_array(native_tree.GetCounts(), flatten=True)
pytree.ys = buffers.as_numpy_array(native_tree.GetYs())
pytree.extra_info = native_tree.GetExtraInfo()
pytrees.append(pytree)
return pytrees
def as_pyforest(native_forest):
pytrees = []
for tree_id in range(native_forest.GetNumberOfTrees()):
native_tree = native_forest.GetTree(tree_id)
native_tree.Compact()
pytree = PyTree()
pytree.native_tree = native_tree
pytree.path = buffers.as_numpy_array(native_tree.GetPath())
pytree.int_features_params = buffers.as_numpy_array(
native_tree.GetIntFeatureParams())
pytree.float_features_params = buffers.as_numpy_array(
native_tree.GetFloatFeatureParams())
pytree.depths = buffers.as_numpy_array(native_tree.GetDepths(),
flatten=True)
pytree.counts = buffers.as_numpy_array(native_tree.GetCounts(),
flatten=True)
pytree.ys = buffers.as_numpy_array(native_tree.GetYs())
pytree.extra_info = native_tree.GetExtraInfo()
pytrees.append(pytree)
return pytrees
def predict(self, **kwargs):
result = buffers.Float32MatrixBuffer()
bufferCollection = self.prepare_data(**kwargs)
self.forest_predictor.PredictYs(bufferCollection, result)
return buffers.as_numpy_array(result)
def predict_leafs_ys(self, **kwargs):
oob_weights = buffers.Float32MatrixBuffer()
leaf_ys = buffers.Float32Tensor3Buffer()
bufferCollection = self.prepare_data(**kwargs)
self.forest_predictor.PredictLeafYs(bufferCollection, oob_weights, leaf_ys)
return buffers.as_numpy_array(leaf_ys), buffers.as_numpy_array(oob_weights)
def convert_img_buffer_both_directions_helper(self, X, buffer_type):
buf = buffers.as_tensor_buffer(X)
assert isinstance(buf, buffer_type)
X_back = buffers.as_numpy_array(buf)
self.assertTrue((X == X_back).all())
def y_hat_sum(self, tree_weights, oob_weights, leaf_ys, **kwargs):
tree_weights = buffers.as_numpy_array(tree_weights)
weights = oob_weights * tree_weights
weighted_ys = weights * leaf_ys
y_hat_sum = weighted_ys.sum(axis=2)
return y_hat_sum, weights, weights.sum(axis=1)
