def test_write_01(self):
original = np.ones([10, 10])
X = Matrix(self.sds, original)
X.write(self.temp_dir + "01").compute()
NX = self.sds.read(self.temp_dir + "01")
res = NX.compute()
self.assertTrue(np.allclose(original, res))
def test_write_02(self):
original = np.array([[1, 2, 3, 4, 5]])
X = Matrix(self.sds, original)
X.write(self.temp_dir + "02").compute()
NX = self.sds.read(self.temp_dir + "02")
res = NX.compute()
self.assertTrue(np.allclose(original, res))
def test_matmul_chain(self):
m3 = np.ones((m2.shape[1], 10), dtype=np.uint8)
m = Matrix(self.sds, m1) @ Matrix(self.sds, m2) @ Matrix(self.sds, m3)
res = (m).compute()
np_res = m1.dot(m2).dot(m3)
self.assertTrue(np.allclose(res, np_res))
self.assertTrue(np.allclose(m.shape, np_res.shape))
def generate_matrices_for_l2svm(dims: int, seed: int = 1234) -> Tuple[Matrix, Matrix]:
np.random.seed(seed)
m1 = np.array(np.random.randint(100, size=dims * dims) + 1.01, dtype=np.double)
m1.shape = (dims, dims)
m2 = np.zeros((dims, 1))
for i in range(dims):
if np.random.random() > 0.5:
m2[i][0] = 1
return Matrix(m1), Matrix(m2)
def test_using_predict(self):
"""
Test the algorithm using the predict function.
With builtin classification
"""
[X, labels, Y] = self.gen_data()
# Call algorithm
bias = multiLogReg(Matrix(self.sds,X),Matrix(self.sds,Y)).compute()
[m, y_pred, acc] = multiLogRegPredict(Matrix(self.sds,X),Matrix(self.sds,bias), Matrix(self.sds,Y)).compute()
self.assertTrue(acc > 98)
def test_simple(self):
"""
Test simple, if the log reg splits a dataset where everything over 1 is label 2 and under 1 is 1.
With manual classification.
"""
[X, labels, Y] = self.gen_data()
# Call algorithm
bias = multiLogReg(Matrix(self.sds,X),Matrix(self.sds,Y)).compute()
# Calculate result.
res = np.reshape(np.dot(X, bias[:len(X[0])]) + bias[len(X[0])], (250))
f2 = lambda x: (x < 0) + 1
accuracy = np.sum(labels == f2(res)) / 250 * 100
self.assertTrue(accuracy > 98)
def test_lm_simple(self):
# if the dimensions of the input is 1, then the
X = np.random.rand(30, 1)
Y = np.random.rand(30, 1)
regressor = LinearRegression(fit_intercept=False)
model = regressor.fit(X, Y).coef_
X_sds = Matrix(self.sds, X)
Y_sds = Matrix(self.sds, Y)
sds_model_weights = lm(X_sds, Y_sds).compute()
model = model.reshape(sds_model_weights.shape)
eps = 1e-03
self.assertTrue(np.allclose(sds_model_weights, model, eps),
"All elements are not close")
def generate_matrices_for_k_means(self, dims: (int, int), seed: int = 1234):
np.random.seed(seed)
mu, sigma = 0, 0.1
s = np.random.normal(mu, sigma, dims[0] * dims[1])
m1 = np.array(s, dtype=np.double)
m1 = np.reshape(m1, (dims[0], dims[1]))
return Matrix(self.sds, m1)
def test_simple(self):
"""
line of numbers. Here the pca should return values that are double or close to double of the last value
"""
m1 = np.array([[1, 1], [2, 2], [3, 3], [4, 4], [5, 5]])
[res, model] = pca(Matrix(self.sds, m1), K=1,
scale=False, center=False).compute()
for x in range(len(m1) - 1):
self.assertTrue(abs(res[x + 1] - res[0] * (x + 2)) < 0.001)
def matrix(self, mat: Union[np.array, os.PathLike],
*args: Sequence[VALID_INPUT_TYPES],
**kwargs: Dict[str, VALID_INPUT_TYPES]) -> 'Matrix':
""" Create matrix.
:param mat: Matrix given by numpy array or path to matrix file
:param args: additional arguments
:param kwargs: additional named arguments
:return: the OperationNode representing this operation
"""
return Matrix(self, mat, *args, **kwargs)
def test_multi_log_reg(self):
# Reduced because we want the tests to finish a bit faster.
train_count = 15000
test_count = 5000
# Train data
X = Matrix(self.sds, self.d.get_train_data().reshape(
(60000, 28*28))[:train_count])
Y = Matrix(self.sds, self.d.get_train_labels()[:train_count])
Y = Y + 1.0
# Test data
Xt = Matrix(self.sds, self.d.get_test_data().reshape(
(10000, 28*28))[:test_count])
Yt = Matrix(self.sds, self.d.get_test_labels()[:test_count])
Yt = Yt + 1.0
bias = multiLogReg(X, Y)
[_, _, acc] = multiLogRegPredict(Xt, bias, Yt).compute()
self.assertGreater(acc, 80)
def test_500x2(self):
"""
This test is based on statistics, that if we run kmeans, on a normal distributed dataset, centered around 0
and use 4 clusters then they will be located in each one corner.
This test uses the prediction builtin.
"""
features = self.generate_matrices_for_k_means((500, 2), seed=1304)
[c, _] = kmeans(features, k=4).compute()
C = Matrix(self.sds, c)
elm = Matrix(self.sds, np.array([[1, 1], [-1, 1], [-1, -1], [1, -1]]))
res = kmeansPredict(elm, C).compute()
corners = set()
for x in res:
if x == 1:
corners.add("pp")
elif x == 2:
corners.add("pn")
elif x == 3:
corners.add("np")
else:
corners.add("nn")
self.assertTrue(len(corners) == 4)
def test_500x2(self):
"""
This test constructs a line of values in 2d space.
That if fit correctly maps perfectly to 1d space.
The check is simply if the input value was positive
then the output value should be similar.
"""
m1 = self.generate_matrices_for_pca(30, seed=1304)
X = Matrix(self.sds, m1)
# print(features)
[res, model] = pca(X, K=1, scale="FALSE", center="FALSE").compute()
for (x, y) in zip(m1, res):
self.assertTrue((x[0] > 0 and y > 0) or (x[0] < 0 and y < 0))
def test_multi_log_reg_with_read(self):
train_count = 100
test_count = 100
X = Matrix(self.sds, self.d.get_train_data().reshape(
(60000, 28*28))[:train_count])
X.write(self.base_path + "train_data").compute()
Y = Matrix(self.sds, self.d.get_train_labels()[:train_count]) + 1
Y.write(self.base_path + "train_labels").compute()
Xr = self.sds.read(self.base_path + "train_data")
Yr = self.sds.read(self.base_path + "train_labels")
bias = multiLogReg(Xr, Yr, verbose=False)
# Test data
Xt = Matrix(self.sds, self.d.get_test_data().reshape(
(10000, 28*28))[:test_count])
Yt = Matrix(self.sds, self.d.get_test_labels()[:test_count])
Yt = Yt + 1.0
[_, _, acc] = multiLogRegPredict(Xt, bias, Yt).compute(verbose=True)
self.assertGreater(acc, 70)
def test_mean3(self):
self.assertTrue(
np.allclose(
Matrix(self.sds, m1).mean(axis=1).compute(),
m1.mean(axis=1).reshape(dim, 1)))
def test_mean2(self):
self.assertTrue(
np.allclose(
Matrix(self.sds, m1).mean(axis=0).compute(), m1.mean(axis=0)))
def test_sum2(self):
self.assertTrue(
np.allclose(
Matrix(self.sds, m1).sum(axis=0).compute(), m1.sum(axis=0)))
def test_empty(self):
m_empty = np.asarray([[]])
r = Matrix(self.sds, np.asarray(m_empty)).rev().compute()
self.assertTrue(np.allclose(r, m_empty))
def test_x_axis(self):
self.assertTrue(np.allclose(Matrix(self.sds, mx).rev().compute(), mx))
def test_tanh(self):
self.assertTrue(
np.allclose(Matrix(self.sds, m1).tanh().compute(), np.tanh(m1)))
def test_atan(self):
self.assertTrue(
np.allclose(Matrix(self.sds, m2).atan().compute(), np.arctan(m2)))
def test_acos(self):
self.assertTrue(
np.allclose(Matrix(self.sds, m2).acos().compute(), np.arccos(m2)))
def test_var3(self):
self.assertTrue(
np.allclose(
Matrix(self.sds, m1).var(axis=1).compute(),
m1.var(axis=1, ddof=1).reshape(dim, 1)))
def test_var1(self):
self.assertTrue(
np.allclose(Matrix(self.sds, m1).var().compute(), m1.var(ddof=1)))
def test_c_bind(self):
m1 = Matrix(self.sds, np.zeros((10, 6)))
m2 = Matrix(self.sds, np.ones((10, 7)))
res = m1.cbind(m2).compute()
npres = np.hstack((np.zeros((10, 6)), np.ones((10, 7))))
self.assertTrue(np.allclose(res, npres))
def test_y_axis(self):
self.assertTrue(
np.allclose(Matrix(self.sds, my).rev().compute(), np.flip(my, 0)))
def test_var2(self):
self.assertTrue(
np.allclose(
Matrix(self.sds, m1).var(axis=0).compute(),
m1.var(axis=0, ddof=1)))
def test_sin(self):
self.assertTrue(
np.allclose(Matrix(self.sds, m1).sin().compute(), np.sin(m1)))
def test_c_bind_shape(self):
m1 = Matrix(self.sds, np.zeros((10, 3)))
m2 = Matrix(self.sds, np.ones((10, 4)))
res = m1.cbind(m2).shape
npres = np.hstack((np.zeros((10, 3)), np.ones((10, 4)))).shape
self.assertTrue(np.allclose(res, npres))
def test_cos(self):
self.assertTrue(
np.allclose(Matrix(self.sds, m1).cos().compute(), np.cos(m1)))