def setUp(self):
self.feature_histogram = FeatureHistogram()
session.init("test_feature_histogram")
data_insts = []
for i in range(1000):
indices = []
data = []
for j in range(10):
x = random.randint(0, 5)
if x != 0:
data.append(x)
indices.append(j)
sparse_vec = SparseVector(indices, data, shape=10)
data_insts.append(
(Instance(features=sparse_vec), (1, random.randint(0, 3))))
self.node_map = {0: 0, 1: 1, 2: 2, 3: 3}
self.data_insts = data_insts
self.data_bin = session.parallelize(data_insts,
include_key=False,
partition=16)
self.grad_and_hess_list = [(random.random(), random.random())
for i in range(1000)]
self.grad_and_hess = session.parallelize(self.grad_and_hess_list,
include_key=False,
partition=16)
bin_split_points = []
for i in range(10):
bin_split_points.append(np.array([i for i in range(6)]))
self.bin_split_points = np.array(bin_split_points)
self.bin_sparse = [0 for i in range(10)]
def get_left_node_local_histogram(self, cur_nodes: List[Node],
tree: List[Node], g_h, table_with_assign,
split_points, sparse_point,
valid_feature):
node_map = self.get_node_map(cur_nodes, left_node_only=True)
LOGGER.info("start to get node histograms")
histograms = FeatureHistogram.calculate_histogram(
table_with_assign, g_h, split_points, sparse_point, valid_feature,
node_map, self.use_missing, self.zero_as_missing)
hist_bags = []
for hist_list in histograms:
hist_bags.append(HistogramBag(hist_list))
left_nodes = []
for node in cur_nodes:
if node.is_left_node or node.id == 0:
left_nodes.append(node)
# set histogram id and parent histogram id
for node, hist_bag in zip(left_nodes, hist_bags):
# LOGGER.debug('node id {}, node parent id {}, cur tree {}'.format(node.id, node.parent_nodeid, len(tree)))
hist_bag.hid = node.id
hist_bag.p_hid = node.parent_nodeid
return hist_bags
def get_local_histogram(self, cur_to_split: List[Node], g_h,
table_with_assign, split_points, sparse_point,
valid_feature):
LOGGER.info("start to get node histograms")
node_map = self.get_node_map(nodes=cur_to_split)
histograms = FeatureHistogram.calculate_histogram(
table_with_assign, g_h, split_points, sparse_point, valid_feature,
node_map, self.use_missing, self.zero_as_missing)
hist_bags = []
for hist_list in histograms:
hist_bags.append(HistogramBag(hist_list))
return hist_bags
class TestFeatureHistogram(unittest.TestCase):
def setUp(self):
self.feature_histogram = FeatureHistogram()
session.init("test_feature_histogram")
data_insts = []
for i in range(1000):
indices = []
data = []
for j in range(10):
x = random.randint(0, 5)
if x != 0:
data.append(x)
indices.append(j)
sparse_vec = SparseVector(indices, data, shape=10)
data_insts.append(
(Instance(features=sparse_vec), (1, random.randint(0, 3))))
self.node_map = {0: 0, 1: 1, 2: 2, 3: 3}
self.data_insts = data_insts
self.data_bin = session.parallelize(data_insts,
include_key=False,
partition=16)
self.grad_and_hess_list = [(random.random(), random.random())
for i in range(1000)]
self.grad_and_hess = session.parallelize(self.grad_and_hess_list,
include_key=False,
partition=16)
bin_split_points = []
for i in range(10):
bin_split_points.append(np.array([i for i in range(6)]))
self.bin_split_points = np.array(bin_split_points)
self.bin_sparse = [0 for i in range(10)]
def test_accumulate_histogram(self):
data = [[[[random.randint(0, 10) for i in range(2)] for j in range(3)]
for k in range(4)] for r in range(5)]
histograms = copy.deepcopy(data)
for i in range(len(data)):
for j in range(len(data[i])):
histograms[i][
j] = self.feature_histogram._tensor_histogram_cumsum(
histograms[i][j])
for k in range(1, len(data[i][j])):
for r in range(len(data[i][j][k])):
data[i][j][k][r] += data[i][j][k - 1][r]
self.assertTrue(
data[i][j][k][r] == histograms[i][j][k][r])
def test_calculate_histogram(self):
histograms = self.feature_histogram.calculate_histogram(
self.data_bin,
self.grad_and_hess,
self.bin_split_points,
self.bin_sparse,
node_map=self.node_map)
his2 = [[[[0 for i in range(3)] for j in range(6)] for k in range(10)]
for r in range(4)]
for i in range(1000):
grad, hess = self.grad_and_hess_list[i]
id = self.node_map[self.data_insts[i][1][1]]
for fid, bid in self.data_insts[i][0].features.get_all_data():
his2[id][fid][bid][0] += grad
his2[id][fid][bid][1] += hess
his2[id][fid][bid][2] += 1
for i in range(len(his2)):
for j in range(len(his2[i])):
his2[i][j] = self.feature_histogram._tensor_histogram_cumsum(
his2[i][j])
for k in range(len(his2[i][j])):
for r in range(len(his2[i][j][k])):
self.assertTrue(
np.fabs(his2[i][j][k][r] - histograms[i][j][k][r])
< consts.FLOAT_ZERO)
def test_aggregate_histogram(self):
fake_fid = 114
data1 = [[random.randint(0, 10) for i in range(2)] for j in range(3)]
data2 = [[random.randint(0, 10) for i in range(2)] for j in range(3)]
fid, agg_histograms = self.feature_histogram._hist_aggregate(
(fake_fid, data1), (fake_fid, data2))
for i in range(len(data1)):
for j in range(len(data1[i])):
data1[i][j] += data2[i][j]
self.assertTrue(data1[i][j] == agg_histograms[i][j])
def tearDown(self):
session.stop()