def setUpClass(cls):
cls.sds = SystemDSContext()
sleep(2.0)
# Clear stdout ...
cls.sds.get_stdout()
cls.sds.get_stdout()
sleep(1.0)
def train(self):
with SystemDSContext() as sds_train:
a = sds_train.rand(500, 10, -100, 100, pdf="normal", seed=10)
features = a # training data all not outliers
n_gaussian = 4
[_, _, _, _, mu, precision_cholesky,
weight] = gmm(features, n_components=n_gaussian, seed=10)
model = sds_train.list(mu, precision_cholesky, weight)
model.write(self.model_path).compute()
def setUpClass(cls):
cls.sds = SystemDSContext()
cls.sds.federated([fed1], [([0, 0], [dim, dim])]).write(
fed1_file, format="federated").compute()
cls.sds.federated([fed1, fed2], [([0, 0], [dim, dim]),
([0, dim], [dim, dim * 2])]).write(
fed2_file,
format="federated").compute()
cls.sds.federated([fed1, fed2, fed3],
[([0, 0], [dim, dim]), ([0, dim], [dim, dim * 2]),
([0, dim * 2], [dim, dim * 3])]).write(
fed3_file, format="federated").compute()
def predict(self):
with SystemDSContext() as sds_predict:
model = sds_predict.read(self.model_path)
mu = model[1].as_matrix()
precision_cholesky = model[2].as_matrix()
weight = model[3].as_matrix()
notOutliers = sds_predict.rand(
10, 10, -1, 1, seed=10) # inside a
outliers = sds_predict.rand(
10, 10, 1150, 1200, seed=10) # outliers
test = outliers.rbind(notOutliers) # testing data half outliers
[_, pp] = gmmPredict(
test, weight, mu, precision_cholesky, model=sds_predict.scalar("VVV"))
outliers = pp.max(axis=1) < 0.99
ret = outliers.compute()
self.assertTrue(ret.sum() == 10)
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# -------------------------------------------------------------
# Import SystemDS
from systemds.context import SystemDSContext
from systemds.operator.algorithm import l2svm
with SystemDSContext() as sds:
# Generate 10 by 10 matrix with values in range 0 to 100.
features = sds.rand(10, 10, 0, 100)
# Add value to all cells in features
features += 1.1
# Generate labels of all ones and zeros
labels = sds.rand(10, 1, 1, 1, sparsity=0.5)
model = l2svm(features, labels).compute()
print(model)
def test_create_multiple_context(self):
# Creating multiple contexts in sequence but open at the same time is okay.
a = SystemDSContext()
b = SystemDSContext()
c = SystemDSContext()
d = SystemDSContext()
a.close()
b.close()
c.close()
d.close()
def test_create_10_contexts(self):
# Creating multiple contexts and closing them should be no problem.
for _ in range(0, 10):
SystemDSContext().close()
def test_same_port(self):
# Same port should graciously change port
sds1 = SystemDSContext(port=9415)
sds2 = SystemDSContext(port=9415)
sds1.close()
sds2.close()
def setUpClass(cls):
cls.sds = SystemDSContext()
cls.d = DataManager()
def get_train_data(self, sds: SystemDSContext) -> 'Frame':
self._get_data(self._train_data_loc)
return sds.read(self._train_data_loc)[:, 0:14]
def setUpClass(cls):
cls.sds = SystemDSContext()
sleep(1)
cls.sds.get_stdout()
cls.sds.get_stdout()
# under the License.
#
# -------------------------------------------------------------
import warnings
import unittest
import os
import shutil
import sys
import re
path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../")
sys.path.insert(0, path)
from systemds.context import SystemDSContext
sds = SystemDSContext()
test_dir = os.path.join("tests", "lineage")
temp_dir = os.path.join(test_dir, "temp")
class TestLineageTrace(unittest.TestCase):
def setUp(self):
warnings.filterwarnings(
action="ignore", message="unclosed", category=ResourceWarning
)
def tearDown(self):
warnings.filterwarnings(
action="ignore", message="unclosed", category=ResourceWarning
)
def setUpClass(cls):
cls.sds = SystemDSContext()
cls.source_reuse = cls.sds.source("./tests/source/source_01.dml",
"test")
def get_jspec(self, sds: SystemDSContext) -> 'Scalar':
self._get_data(self._jspec_loc)
return sds.read(self._jspec_loc,
data_type="scalar",
value_type="string")
def get_test_labels(self, sds: SystemDSContext) -> 'Frame':
self._get_data(self._test_data_loc)
return sds.read(self._test_data_loc)[:, 14]
def setUpClass(cls):
cls.sds = SystemDSContext()
import warnings
import unittest
import os
import sys
import numpy as np
path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../")
sys.path.insert(0, path)
from systemds.context import SystemDSContext
from sklearn.linear_model import LinearRegression
import random
sds = SystemDSContext()
regressor = LinearRegression(fit_intercept=False)
shape = (random.randrange(1, 30), random.randrange(1, 30))
eps = 1e-03
class TestLm(unittest.TestCase):
def setUp(self):
warnings.filterwarnings(action="ignore",
message="unclosed",
category=ResourceWarning)
def tearDown(self):
warnings.filterwarnings(action="ignore",
message="unclosed",
def setUpClass(cls):
cls.sds = SystemDSContext()
cls.jvm = cls.sds.java_gateway.jvm
