def setUpClass(cls): cls.sds = SystemDSContext() sleep(2.0) # Clear stdout ... cls.sds.get_stdout() cls.sds.get_stdout() sleep(1.0)
def train(self): with SystemDSContext() as sds_train: a = sds_train.rand(500, 10, -100, 100, pdf="normal", seed=10) features = a # training data all not outliers n_gaussian = 4 [_, _, _, _, mu, precision_cholesky, weight] = gmm(features, n_components=n_gaussian, seed=10) model = sds_train.list(mu, precision_cholesky, weight) model.write(self.model_path).compute()
def setUpClass(cls): cls.sds = SystemDSContext() cls.sds.federated([fed1], [([0, 0], [dim, dim])]).write( fed1_file, format="federated").compute() cls.sds.federated([fed1, fed2], [([0, 0], [dim, dim]), ([0, dim], [dim, dim * 2])]).write( fed2_file, format="federated").compute() cls.sds.federated([fed1, fed2, fed3], [([0, 0], [dim, dim]), ([0, dim], [dim, dim * 2]), ([0, dim * 2], [dim, dim * 3])]).write( fed3_file, format="federated").compute()
def predict(self): with SystemDSContext() as sds_predict: model = sds_predict.read(self.model_path) mu = model[1].as_matrix() precision_cholesky = model[2].as_matrix() weight = model[3].as_matrix() notOutliers = sds_predict.rand( 10, 10, -1, 1, seed=10) # inside a outliers = sds_predict.rand( 10, 10, 1150, 1200, seed=10) # outliers test = outliers.rbind(notOutliers) # testing data half outliers [_, pp] = gmmPredict( test, weight, mu, precision_cholesky, model=sds_predict.scalar("VVV")) outliers = pp.max(axis=1) < 0.99 ret = outliers.compute() self.assertTrue(ret.sum() == 10)
# distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # ------------------------------------------------------------- # Import SystemDS from systemds.context import SystemDSContext from systemds.operator.algorithm import l2svm with SystemDSContext() as sds: # Generate 10 by 10 matrix with values in range 0 to 100. features = sds.rand(10, 10, 0, 100) # Add value to all cells in features features += 1.1 # Generate labels of all ones and zeros labels = sds.rand(10, 1, 1, 1, sparsity=0.5) model = l2svm(features, labels).compute() print(model)
def test_create_multiple_context(self): # Creating multiple contexts in sequence but open at the same time is okay. a = SystemDSContext() b = SystemDSContext() c = SystemDSContext() d = SystemDSContext() a.close() b.close() c.close() d.close()
def test_create_10_contexts(self): # Creating multiple contexts and closing them should be no problem. for _ in range(0, 10): SystemDSContext().close()
def test_same_port(self): # Same port should graciously change port sds1 = SystemDSContext(port=9415) sds2 = SystemDSContext(port=9415) sds1.close() sds2.close()
def setUpClass(cls): cls.sds = SystemDSContext() cls.d = DataManager()
def get_train_data(self, sds: SystemDSContext) -> 'Frame': self._get_data(self._train_data_loc) return sds.read(self._train_data_loc)[:, 0:14]
def setUpClass(cls): cls.sds = SystemDSContext() sleep(1) cls.sds.get_stdout() cls.sds.get_stdout()
# under the License. # # ------------------------------------------------------------- import warnings import unittest import os import shutil import sys import re path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../") sys.path.insert(0, path) from systemds.context import SystemDSContext sds = SystemDSContext() test_dir = os.path.join("tests", "lineage") temp_dir = os.path.join(test_dir, "temp") class TestLineageTrace(unittest.TestCase): def setUp(self): warnings.filterwarnings( action="ignore", message="unclosed", category=ResourceWarning ) def tearDown(self): warnings.filterwarnings( action="ignore", message="unclosed", category=ResourceWarning )
def setUpClass(cls): cls.sds = SystemDSContext() cls.source_reuse = cls.sds.source("./tests/source/source_01.dml", "test")
def get_jspec(self, sds: SystemDSContext) -> 'Scalar': self._get_data(self._jspec_loc) return sds.read(self._jspec_loc, data_type="scalar", value_type="string")
def get_test_labels(self, sds: SystemDSContext) -> 'Frame': self._get_data(self._test_data_loc) return sds.read(self._test_data_loc)[:, 14]
def setUpClass(cls): cls.sds = SystemDSContext()
import warnings import unittest import os import sys import numpy as np path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../") sys.path.insert(0, path) from systemds.context import SystemDSContext from sklearn.linear_model import LinearRegression import random sds = SystemDSContext() regressor = LinearRegression(fit_intercept=False) shape = (random.randrange(1, 30), random.randrange(1, 30)) eps = 1e-03 class TestLm(unittest.TestCase): def setUp(self): warnings.filterwarnings(action="ignore", message="unclosed", category=ResourceWarning) def tearDown(self): warnings.filterwarnings(action="ignore", message="unclosed",
def setUpClass(cls): cls.sds = SystemDSContext() cls.jvm = cls.sds.java_gateway.jvm