def test_apply_axis(self, x): """ Tests apply along axis """ x1 = ds.apply_along_axis(_sum_and_mult, 0, x) self.assertTrue(x1.shape, (1, 3)) self.assertTrue(x1._reg_shape, (1, 2)) self.assertTrue(_equal_arrays(x1.collect(), np.array([12, 15, 18]))) self.assertTrue(_validate_array(x1)) x1 = ds.apply_along_axis(_sum_and_mult, 1, x) self.assertTrue(x1.shape, (3, 1)) self.assertTrue(x1._reg_shape, (2, 1)) self.assertTrue(_equal_arrays(x1.collect(False), np.array([[6], [15], [24]]))) self.assertTrue(_validate_array(x1)) x1 = ds.apply_along_axis(_sum_and_mult, 1, x, 2) self.assertTrue(x1.shape, (3, 1)) self.assertTrue(x1._reg_shape, (2, 1)) self.assertTrue(_equal_arrays(x1.collect(False), np.array([[8], [17], [26]]))) self.assertTrue(_validate_array(x1)) x1 = ds.apply_along_axis(_sum_and_mult, 1, x, b=2) self.assertTrue(x1.shape, (3, 1)) self.assertTrue(x1._reg_shape, (2, 1)) self.assertTrue(_equal_arrays(x1.collect(False), np.array([[12], [30], [48]]))) self.assertTrue(_validate_array(x1)) x1 = ds.apply_along_axis(_sum_and_mult, 1, x, 1, b=2) self.assertTrue(x1.shape, (3, 1)) self.assertTrue(x1._reg_shape, (2, 1)) self.assertTrue(_equal_arrays(x1.collect(False), np.array([[14], [32], [50]]))) self.assertTrue(_validate_array(x1))
def test_apply_sparse(self): """ Tests apply with sparse data """ x_d = ds.array(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]), block_size=(2, 2)) x_sp = ds.array(csr_matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]]), block_size=(2, 2)) x1 = ds.apply_along_axis(_sum_and_mult, 0, x_d) x2 = ds.apply_along_axis(_sum_and_mult, 0, x_sp) self.assertTrue(np.array_equal(x1.collect(), x2.collect()))
def fit(self, x): """ Compute the min and max values for later scaling. Parameters ---------- x : ds-array, shape=(n_samples, n_features) Returns ------- self : MinMaxScaler """ self.data_min_ = ds.apply_along_axis(np.min, 0, x) self.data_max_ = ds.apply_along_axis(np.max, 0, x) return self
def test_apply_axis(self): """ Tests apply along axis""" x = ds.array(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]), block_size=(2, 2)) x1 = ds.apply_along_axis(_sum_and_mult, 0, x) self.assertTrue(x1.shape, (1, 3)) self.assertTrue(x1._reg_shape, (1, 2)) self.assertTrue(np.array_equal(x1.collect(), np.array([12, 15, 18]))) x1 = ds.apply_along_axis(_sum_and_mult, 1, x) self.assertTrue(x1.shape, (3, 1)) self.assertTrue(x1._reg_shape, (2, 1)) self.assertTrue(np.array_equal(x1.collect(), np.array([6, 15, 24]))) x1 = ds.apply_along_axis(_sum_and_mult, 1, x, 2) self.assertTrue(x1.shape, (3, 1)) self.assertTrue(x1._reg_shape, (2, 1)) self.assertTrue(np.array_equal(x1.collect(), np.array([8, 17, 26]))) x1 = ds.apply_along_axis(_sum_and_mult, 1, x, b=2) self.assertTrue(x1.shape, (3, 1)) self.assertTrue(x1._reg_shape, (2, 1)) self.assertTrue(np.array_equal(x1.collect(), np.array([12, 30, 48]))) x1 = ds.apply_along_axis(_sum_and_mult, 1, x, 1, b=2) self.assertTrue(x1.shape, (3, 1)) self.assertTrue(x1._reg_shape, (2, 1)) self.assertTrue(np.array_equal(x1.collect(), np.array([14, 32, 50]))) x = ds.array(sp.csr_matrix([[1, 0, -1], [0, 5, 0], [7, 8, 0]]), block_size=(2, 2)) x1 = ds.apply_along_axis(_sum_and_mult, 0, x, 1, b=2) self.assertTrue(x1.shape, (1, 3)) self.assertTrue(x1._reg_shape, (1, 2)) self.assertTrue(np.array_equal(x1.collect(), np.array([18, 28, 0]))) x = ds.array(sp.csr_matrix([[1, 0, -1], [0, 5, 0], [7, 8, 0]]), block_size=(2, 2)) x1 = ds.apply_along_axis(_sum_and_mult, 0, x, 1, b=2) self.assertTrue(x1.shape, (1, 3)) self.assertTrue(x1._reg_shape, (1, 2)) self.assertTrue((x1.collect() == np.array([18, 28, 0])).all())
def fit(self, x): """ Compute the mean and std to be used for later scaling. Parameters ---------- x : ds-array, shape=(n_samples, n_features) Returns ------- self : StandardScaler """ self.mean_ = ds.apply_along_axis(np.mean, 0, x) var_blocks = [[]] for row, m_row in zip(x._iterator(1), self.mean_._iterator(1)): var_blocks[0].append(_compute_var(row._blocks, m_row._blocks)) self.var_ = Array(var_blocks, top_left_shape=self.mean_._top_left_shape, reg_shape=self.mean_._reg_shape, shape=self.mean_.shape, sparse=x._sparse) return self