Example #1
0
    def testRunWithoutFetch(self):
        with new_cluster(scheduler_n_process=2, worker_n_process=2,
                         shared_memory='20M') as cluster:
            session = cluster.session

            a = mt.ones((10, 20)) + 1
            self.assertIsNone(session.run(a, fetch=False))
            np.testing.assert_array_equal(a.execute(session=session), np.ones((10, 20)) + 1)
Example #2
0
 def testExecutableTuple(self):
     with new_cluster(scheduler_n_process=2, worker_n_process=2,
                      web=True) as cluster:
         with new_session('http://' + cluster._web_endpoint).as_default():
             a = mt.ones((20, 10), chunk_size=10)
             u, s, v = (mt.linalg.svd(a)).execute()
             np.testing.assert_allclose(u.dot(np.diag(s).dot(v)),
                                        np.ones((20, 10)))
Example #3
0
    def testGraphFail(self):
        op = SerializeMustFailOperand(f=3)
        tensor = op.new_tensor(None, (3, 3))

        with new_cluster(scheduler_n_process=2, worker_n_process=2,
                         shared_memory='20M') as cluster:
            with self.assertRaises(SystemError):
                cluster.session.run(tensor)
Example #4
0
    def testGraphFail(self, *_):
        op = SerializeMustFailOperand(f=3)
        tensor = op.new_tensor(None, (3, 3))

        with new_cluster(scheduler_n_process=2, worker_n_process=2,
                         shared_memory='20M') as cluster:
            with self.assertRaises(ExecutionFailed):
                cluster.session.run(tensor, timeout=_exec_timeout)
Example #5
0
    def testFetchDataFrameSlices(self, *_):
        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M',
                         web=True) as cluster:
            session = cluster.session
            a = mt.random.rand(10, 10, chunk_size=3)
            df = md.DataFrame(a)

            r = session.run(df)

            r_slice1 = session.fetch(df.iloc[:2])
            pd.testing.assert_frame_equal(r.iloc[:2], r_slice1)

            r_slice2 = session.fetch(df.iloc[2:8, 2:8])
            pd.testing.assert_frame_equal(r.iloc[2:8, 2:8], r_slice2)

            r_slice3 = session.fetch(df.iloc[:, 2:])
            pd.testing.assert_frame_equal(r.iloc[:, 2:], r_slice3)

            r_slice4 = session.fetch(df.iloc[:, -5:])
            pd.testing.assert_frame_equal(r.iloc[:, -5:], r_slice4)

            r_slice5 = session.fetch(df.iloc[4])
            pd.testing.assert_series_equal(r.iloc[4], r_slice5)

            r_slice6 = session.fetch(df.iloc[6:9])
            pd.testing.assert_frame_equal(r.iloc[6:9], r_slice6)

            # test repr
            pdf = pd.DataFrame(np.random.randint(1000, size=(80, 10)))
            df2 = md.DataFrame(pdf, chunk_size=41)
            self.assertEqual(repr(df2.execute(session=session)), repr(pdf))

            ps = pdf[0]
            s = md.Series(ps, chunk_size=41)
            self.assertEqual(repr(s.execute(session=session)), repr(ps))

            web_session = new_session('http://' + cluster._web_endpoint)
            r = web_session.run(df)

            r_slice1 = web_session.fetch(df.iloc[:2])
            pd.testing.assert_frame_equal(r.iloc[:2], r_slice1)

            r_slice2 = web_session.fetch(df.iloc[2:8, 2:8])
            pd.testing.assert_frame_equal(r.iloc[2:8, 2:8], r_slice2)

            r_slice3 = web_session.fetch(df.iloc[:, 2:])
            pd.testing.assert_frame_equal(r.iloc[:, 2:], r_slice3)

            r_slice4 = web_session.fetch(df.iloc[:, -5:])
            pd.testing.assert_frame_equal(r.iloc[:, -5:], r_slice4)

            r_slice5 = web_session.fetch(df.iloc[4])
            pd.testing.assert_series_equal(r.iloc[4], r_slice5)

            r_slice6 = web_session.fetch(df.iloc[6:9])
            pd.testing.assert_frame_equal(r.iloc[6:9], r_slice6)
Example #6
0
    def testIndexTensorExecute(self, *_):
        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M') as cluster:
            session = cluster.session

            a = mt.random.rand(10, 5)
            idx = slice(0, 5), slice(0, 5)
            a[idx] = 2
            a_splits = mt.split(a, 2)
            r1, r2 = session.run(a_splits[0], a[idx], timeout=_exec_timeout)

            np.testing.assert_array_equal(r1, r2)
            np.testing.assert_array_equal(r1, np.ones((5, 5)) * 2)

            with new_session(cluster.endpoint) as session2:
                a = mt.random.rand(10, 5)
                idx = slice(0, 5), slice(0, 5)

                a[idx] = mt.ones((5, 5)) * 2
                r = session2.run(a[idx], timeout=_exec_timeout)

                np.testing.assert_array_equal(r, np.ones((5, 5)) * 2)

            with new_session(cluster.endpoint) as session3:
                a = mt.random.rand(100, 5)

                slice1 = a[:10]
                slice2 = a[10:20]
                r1, r2, expected = session3.run(slice1,
                                                slice2,
                                                a,
                                                timeout=_exec_timeout)

                np.testing.assert_array_equal(r1, expected[:10])
                np.testing.assert_array_equal(r2, expected[10:20])

            with new_session(cluster.endpoint) as session4:
                a = mt.random.rand(100, 5)

                a[:10] = mt.ones((10, 5))
                a[10:20] = 2
                r = session4.run(a, timeout=_exec_timeout)

                np.testing.assert_array_equal(r[:10], np.ones((10, 5)))
                np.testing.assert_array_equal(r[10:20], np.ones((10, 5)) * 2)

            with new_session(cluster.endpoint) as session5:
                raw = np.random.rand(10, 10)
                a = mt.tensor(raw, chunk_size=(5, 4))
                b = a[a.argmin(axis=1), mt.tensor(np.arange(10))]
                r = session5.run(b, timeout=_exec_timeout, compose=False)

                np.testing.assert_array_equal(
                    r, raw[raw.argmin(axis=1),
                           np.arange(10)])
Example #7
0
    def testRunWithoutCompose(self, *_):
        with new_cluster(scheduler_n_process=2, worker_n_process=2,
                         shared_memory='20M', web=False) as cluster:
            session = cluster.session

            arr1 = (mt.ones((10, 10), chunk_size=3) + 1) * 2
            r1 = session.run(arr1, timeout=_exec_timeout)
            arr2 = (mt.ones((10, 10), chunk_size=4) + 1) * 2
            r2 = session.run(arr2, compose=False, timeout=_exec_timeout)
            np.testing.assert_array_equal(r1, r2)
Example #8
0
    def testMutableTensorCtor(self):
        def testWithGivenSession(session):
            from mars.tensor.core import mutable_tensor

            # cannot get non-existing mutable tensor
            with self.assertRaises(ValueError):
                mutable_tensor("test")

            # should be create
            mut1 = mutable_tensor("test", (4, 5), dtype='int32', chunk_size=3)

            # should be get
            mut2 = mutable_tensor("test")

            # mut1 should equal to mut2, but are not the same object
            self.assertEqual(mut1.shape, mut2.shape)
            self.assertEqual(mut1.dtype, mut2.dtype)

            # LocalSession return the same MutableTensor instance when `get_mutable_tensor`.
            if isinstance(session._sess, LocalSession):
                self.assertTrue(mut1 is mut2)
            else:
                self.assertTrue(mut1 is not mut2)

            mut2[1:4, 2] = 8
            mut2[2:4] = np.arange(10).reshape(2, 5)

            expected = np.zeros((4, 5), dtype='int32')
            expected[1:4, 2] = 8
            expected[2:4] = np.arange(10).reshape(2, 5)

            # cannot be sealed twice
            #
            # Note that we operate on `mut2`, if we seal `mut1`, the result may not be correct.
            #
            # When we operate both on `mut1` and `mut2`, the result may not correct since the
            # two MutableTensor instances both main their own local buffers, but they cannot
            # be both sealed.
            arr = mut2.seal()
            with self.assertRaises(ValueError):
                mut1.seal()

            # check value
            np.testing.assert_array_equal(session.fetch(arr), expected)

        with new_session().as_default() as session:
            testWithGivenSession(session)

        with new_cluster(scheduler_n_process=2, worker_n_process=2,
                         shared_memory='20M', web=True) as cluster:
            session = cluster.session.as_default()
            testWithGivenSession(session)

            with new_session('http://' + cluster._web_endpoint).as_default() as web_session:
                testWithGivenSession(web_session)
Example #9
0
    def testEagerMode(self):
        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M',
                         web=True) as cluster:

            self.assertIsInstance(Session.default_or_local()._sess,
                                  LocalClusterSession)

            with option_context({'eager_mode': True}):
                a_data = np.random.rand(10, 10)

                a = mt.tensor(a_data, chunk_size=3)
                np.testing.assert_array_equal(a, a_data)

                r1 = a + 1
                expected1 = a_data + 1
                np.testing.assert_array_equal(r1, expected1)

                r2 = r1.dot(r1)
                expected2 = expected1.dot(expected1)
                np.testing.assert_array_almost_equal(r2, expected2)

            a = mt.ones((10, 10), chunk_size=3)
            with self.assertRaises(ValueError):
                a.fetch()

            r = a.dot(a)
            np.testing.assert_array_equal(r.execute(), np.ones((10, 10)) * 10)

            with new_session('http://' + cluster._web_endpoint).as_default():
                self.assertIsInstance(Session.default_or_local()._sess,
                                      WebSession)

                with option_context({'eager_mode': True}):
                    a_data = np.random.rand(10, 10)

                    a = mt.tensor(a_data, chunk_size=3)
                    np.testing.assert_array_equal(a, a_data)

                    r1 = a + 1
                    expected1 = a_data + 1
                    np.testing.assert_array_equal(r1, expected1)

                    r2 = r1.dot(r1)
                    expected2 = expected1.dot(expected1)
                    np.testing.assert_array_almost_equal(r2, expected2)

                a = mt.ones((10, 10), chunk_size=3)
                with self.assertRaises(ValueError):
                    a.fetch()

                r = a.dot(a)
                np.testing.assert_array_equal(r.execute(),
                                              np.ones((10, 10)) * 10)
Example #10
0
    def testCudaCluster(self, *_):
        from mars.dataframe.datasource.dataframe import from_pandas as from_pandas_df

        with new_cluster(scheduler_n_process=2, worker_n_process=2,
                         shared_memory='20M', web=True) as cluster:
            session = cluster.session
            pdf = pd.DataFrame(np.random.rand(20, 30), index=np.arange(20, 0, -1))
            df = from_pandas_df(pdf, chunk_size=(13, 21))
            cdf = df.to_gpu()
            result = session.run(cdf)
            pd.testing.assert_frame_equal(pdf, result)
Example #11
0
    def testMultipleOutputTensorExecute(self, *_):
        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M') as cluster:
            session = cluster.session

            t = mt.random.rand(20, 5, chunk_size=5)
            r = mt.linalg.svd(t)

            res = session.run((t, ) + r, timeout=_exec_timeout)

            U, s, V = res[1:]
            np.testing.assert_allclose(res[0], U.dot(np.diag(s).dot(V)))

            raw = np.random.rand(20, 5)

            # to test the fuse, the graph should be fused
            t = mt.array(raw)
            U, s, V = mt.linalg.svd(t)
            r = U.dot(mt.diag(s).dot(V))

            res = r.execute()
            np.testing.assert_allclose(raw, res)

            # test submit part of svd outputs
            t = mt.array(raw)
            U, s, V = mt.linalg.svd(t)

            with new_session(cluster.endpoint) as session2:
                U_result, s_result = session2.run(U, s, timeout=_exec_timeout)
                U_expected, s_expectd, _ = np.linalg.svd(raw,
                                                         full_matrices=False)

                np.testing.assert_allclose(U_result, U_expected)
                np.testing.assert_allclose(s_result, s_expectd)

            with new_session(cluster.endpoint) as session2:
                U_result, s_result = session2.run(U + 1,
                                                  s + 1,
                                                  timeout=_exec_timeout)
                U_expected, s_expectd, _ = np.linalg.svd(raw,
                                                         full_matrices=False)

                np.testing.assert_allclose(U_result, U_expected + 1)
                np.testing.assert_allclose(s_result, s_expectd + 1)

            with new_session(cluster.endpoint) as session2:
                t = mt.array(raw)
                _, s, _ = mt.linalg.svd(t)
                del _

                s_result = session2.run(s, timeout=_exec_timeout)
                s_expected = np.linalg.svd(raw, full_matrices=False)[1]
                np.testing.assert_allclose(s_result, s_expected)
Example #12
0
    def testSparse(self):
        import scipy.sparse as sps

        with new_cluster(scheduler_n_process=2, worker_n_process=2,
                         shared_memory='20M', web=True) as cluster:
            session = cluster.session

            # calculate sparse with no element in matrix
            a = sps.csr_matrix((10000, 10000))
            b = sps.csr_matrix((10000, 1))
            t1 = mt.tensor(a)
            t2 = mt.tensor(b)
            session.run(t1 * t2)
Example #13
0
    def testLocalClusterWithWeb(self):
        with new_cluster(scheduler_n_process=2, worker_n_process=3, web=True) as cluster:
            with cluster.session as session:
                t = mt.ones((3, 3), chunk_size=2)
                result = session.run(t)

                np.testing.assert_array_equal(result, np.ones((3, 3)))

            with new_session('http://' + cluster._web_endpoint) as session:
                t = mt.ones((3, 3), chunk_size=2)
                result = session.run(t)

                np.testing.assert_array_equal(result, np.ones((3, 3)))
Example #14
0
    def testTileContextInLocalCluster(self):
        with new_cluster(scheduler_n_process=2, worker_n_process=2,
                         shared_memory='20M', modules=[__name__], web=True) as cluster:
            session = cluster.session

            raw = np.random.rand(10, 20)
            data = mt.tensor(raw)

            session.run(data)

            data2 = TileWithContextOperand().new_tensor([data], shape=data.shape)

            result = session.run(data2)
            np.testing.assert_array_equal(raw * raw.nbytes, result)
Example #15
0
    def testBoolIndexingExecute(self, *_):
        mock_nsplits = ((4, 4, 4, 4), )
        MarsAPI.get_tensor_nsplits.return_value = mock_nsplits
        with new_cluster(scheduler_n_process=2, worker_n_process=2) as cluster:
            a = mt.random.rand(8, 8, chunk_size=4)
            a[2:6, 2:6] = mt.ones((4, 4)) * 2
            b = a[a > 1]
            self.assertEqual(b.shape, (np.nan, ))

            cluster.session.run(b, fetch=False)
            self.assertEqual(b.shape, (16, ))

            c = b.reshape((4, 4))
            self.assertEqual(c.shape, (4, 4))
Example #16
0
    def testMutableTensorString(self):
        def testWithGivenSession(session):
            from mars.tensor.core import mutable_tensor

            # simple dtype.
            mut1 = mutable_tensor("test", (4, ), dtype='<U16', chunk_size=3)
            mut1[0] = 'a'
            mut1[1] = 'bb'
            mut1[2] = 'cccc'
            mut1[3] = 'dddddddd'
            arr1 = mut1.seal()

            expected = np.empty((4, ), dtype='<U16')
            expected[0] = 'a'
            expected[1] = 'bb'
            expected[2] = 'cccc'
            expected[3] = 'dddddddd'
            np.testing.assert_array_equal(session.fetch(arr1), expected)

            # structured array that contains string
            dtype = np.dtype([('x', np.int32), ('y', '<U16')])
            mut2 = mutable_tensor("test", (4, ), dtype=dtype, chunk_size=3)
            mut2[0] = (0, 'a')
            mut2[1] = (1, 'bb')
            mut2[2] = (2, 'cccc')
            mut2[3] = (3, 'dddddddd')
            arr2 = mut2.seal()

            expected = np.empty((4, ), dtype=dtype)
            expected[0] = (0, 'a')
            expected[1] = (1, 'bb')
            expected[2] = (2, 'cccc')
            expected[3] = (3, 'dddddddd')
            np.testing.assert_array_equal(session.fetch(arr2), expected)

        with new_session().as_default() as session:
            testWithGivenSession(session)

        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M',
                         web=True) as cluster:
            session = cluster.session.as_default()
            testWithGivenSession(session)

            with new_session(
                    'http://' +
                    cluster._web_endpoint).as_default() as web_session:
                testWithGivenSession(web_session)
Example #17
0
    def testFetchDataFrameSlices(self, *_):
        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M',
                         web=True) as cluster:
            session = cluster.session
            a = mt.random.rand(10, 10, chunk_size=3)
            df = md.DataFrame(a)

            r = session.run(df)

            r_slice1 = session.fetch(df.iloc[:2])
            pd.testing.assert_frame_equal(r.iloc[:2], r_slice1)

            r_slice2 = session.fetch(df.iloc[2:8, 2:8])
            pd.testing.assert_frame_equal(r.iloc[2:8, 2:8], r_slice2)

            r_slice3 = session.fetch(df.iloc[:, 2:])
            pd.testing.assert_frame_equal(r.iloc[:, 2:], r_slice3)

            r_slice4 = session.fetch(df.iloc[:, -5:])
            pd.testing.assert_frame_equal(r.iloc[:, -5:], r_slice4)

            r_slice5 = session.fetch(df.iloc[4])
            pd.testing.assert_series_equal(r.iloc[4], r_slice5)

            r_slice6 = session.fetch(df.iloc[6:9])
            pd.testing.assert_frame_equal(r.iloc[6:9], r_slice6)

            web_session = new_session('http://' + cluster._web_endpoint)
            r = web_session.run(df)

            r_slice1 = web_session.fetch(df.iloc[:2])
            pd.testing.assert_frame_equal(r.iloc[:2], r_slice1)

            r_slice2 = web_session.fetch(df.iloc[2:8, 2:8])
            pd.testing.assert_frame_equal(r.iloc[2:8, 2:8], r_slice2)

            r_slice3 = web_session.fetch(df.iloc[:, 2:])
            pd.testing.assert_frame_equal(r.iloc[:, 2:], r_slice3)

            r_slice4 = web_session.fetch(df.iloc[:, -5:])
            pd.testing.assert_frame_equal(r.iloc[:, -5:], r_slice4)

            r_slice5 = web_session.fetch(df.iloc[4])
            pd.testing.assert_series_equal(r.iloc[4], r_slice5)

            r_slice6 = web_session.fetch(df.iloc[6:9])
            pd.testing.assert_frame_equal(r.iloc[6:9], r_slice6)
Example #18
0
    def testMutableTensorFillValue(self):
        def testWithGivenSession(session):
            from mars.tensor.core import mutable_tensor

            # simple dtype.
            mut1 = mutable_tensor("test", (4, 5),
                                  dtype='double',
                                  fill_value=123.456,
                                  chunk_size=3)

            mut1[1:4, 2] = 8
            mut1[2:4] = np.arange(10).reshape(2, 5)
            arr1 = mut1.seal()

            expected = np.full((4, 5), fill_value=123.456, dtype='double')
            expected[1:4, 2] = 8
            expected[2:4] = np.arange(10).reshape(2, 5)
            np.testing.assert_array_equal(session.fetch(arr1), expected)

            # structured dtype, but the `fill_value` cannot be tuple (consistent with np.full).
            dtype = np.dtype([('x', np.int32), ('y', np.double)])
            mut2 = mutable_tensor("test", (4, 5),
                                  dtype=dtype,
                                  fill_value=123.456,
                                  chunk_size=3)

            mut2[1:4, 2] = (1, 2.)
            mut2[2:4] = np.arange(10).reshape(2, 5)
            arr2 = mut2.seal()

            expected = np.full((4, 5), fill_value=123.456, dtype=dtype)
            expected[1:4, 2] = (1, 2.)
            expected[2:4] = np.arange(10).reshape(2, 5)
            np.testing.assert_array_equal(session.fetch(arr2), expected)

        with new_session().as_default() as session:
            testWithGivenSession(session)

        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M',
                         web=True) as cluster:
            session = cluster.session.as_default()
            testWithGivenSession(session)

            with new_session(
                    'http://' +
                    cluster._web_endpoint).as_default() as web_session:
                testWithGivenSession(web_session)
Example #19
0
    def testFetchSlices(self, *_):
        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M',
                         web=True) as cluster:
            session = cluster.session
            a = mt.random.rand(10, 10, 10, chunk_size=3)

            r = session.run(a)

            r_slice1 = session.fetch(a[:2])
            np.testing.assert_array_equal(r[:2], r_slice1)

            r_slice2 = session.fetch(a[2:8, 2:8])
            np.testing.assert_array_equal(r[2:8, 2:8], r_slice2)

            r_slice3 = session.fetch(a[:, 2:])
            np.testing.assert_array_equal(r[:, 2:], r_slice3)

            r_slice4 = session.fetch(a[:, 2:, -5:])
            np.testing.assert_array_equal(r[:, 2:, -5:], r_slice4)

            r_slice5 = session.fetch(a[0])
            np.testing.assert_array_equal(r[0], r_slice5)

            # test repr
            with np.printoptions(threshold=100):
                raw = np.random.randint(1000, size=(3, 4, 6))
                b = mt.tensor(raw, chunk_size=3)
                self.assertEqual(repr(b.execute(session=session)), repr(raw))

            web_session = new_session('http://' + cluster._web_endpoint)
            r = web_session.run(a)

            r_slice1 = web_session.fetch(a[:2])
            np.testing.assert_array_equal(r[:2], r_slice1)

            r_slice2 = web_session.fetch(a[2:8, 2:8])
            np.testing.assert_array_equal(r[2:8, 2:8], r_slice2)

            r_slice3 = web_session.fetch(a[:, 2:])
            np.testing.assert_array_equal(r[:, 2:], r_slice3)

            r_slice4 = web_session.fetch(a[:, 2:, -5:])
            np.testing.assert_array_equal(r[:, 2:, -5:], r_slice4)

            r_slice5 = web_session.fetch(a[4])
            np.testing.assert_array_equal(r[4], r_slice5)
Example #20
0
    def testSingleOutputTensorExecute(self):
        with new_cluster(scheduler_n_process=2, worker_n_process=2) as cluster:
            self.assertIs(cluster.session, Session.default_or_local())

            t = mt.random.rand(10)
            r = t.sum()

            res = r.execute()
            self.assertTrue(np.isscalar(res))
            self.assertLess(res, 10)

            t = mt.random.rand(10)
            r = t.sum() * 4 - 1

            res = r.execute()
            self.assertLess(res, 39)
Example #21
0
    def testTiledTensor(self, *_):
        with new_cluster(scheduler_n_process=2, worker_n_process=2,
                         shared_memory='20M') as cluster:
            session = cluster.session
            a = mt.ones((10, 10), chunk_size=3)
            b = a.dot(a)
            b = b.tiles()

            r = session.run(b, timeout=_exec_timeout)
            np.testing.assert_array_equal(r, np.ones((10, 10)) * 10)

            a = a.tiles()
            b = a + 1

            r = session.run(b, timeout=_exec_timeout)
            np.testing.assert_array_equal(r, np.ones((10, 10)) + 1)
Example #22
0
    def testClusterSession(self):
        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M',
                         web=True) as cluster:
            sess1 = cluster.session
            sess2 = new_session(cluster.endpoint, session_id=sess1.session_id)

            self.assertNotEqual(sess1, sess2)
            self.assertEqual(sess1.session_id, sess2.session_id)

            session_id = str(uuid.uuid4())
            with self.assertRaises(ValueError) as cm:
                new_session(cluster.endpoint, session_id=session_id)

            expected_msg = "The session with id = %s doesn't exist" % session_id
            self.assertEqual(cm.exception.args[0], expected_msg)

            sess1.close()
            with self.assertRaises(ValueError) as cm:
                new_session(cluster.endpoint, session_id=sess1.session_id)

            expected_msg = "The session with id = %s doesn't exist" % sess1.session_id
            self.assertEqual(cm.exception.args[0], expected_msg)

            web_sess1 = new_session('http://' + cluster._web_endpoint)
            web_sess2 = new_session('http://' + cluster._web_endpoint,
                                    session_id=web_sess1.session_id)

            self.assertNotEqual(web_sess1, web_sess2)
            self.assertEqual(web_sess1.session_id, web_sess2.session_id)

            session_id = str(uuid.uuid4())
            with self.assertRaises(ValueError) as cm:
                new_session('http://' + cluster._web_endpoint,
                            session_id=session_id)

            expected_msg = "The session with id = %s doesn't exist" % session_id
            self.assertEqual(cm.exception.args[0], expected_msg)

            web_sess1.close()
            with self.assertRaises(ValueError) as cm:
                new_session('http://' + cluster._web_endpoint,
                            session_id=web_sess1.session_id)

            expected_msg = "The session with id = %s doesn't exist" % web_sess1.session_id
            self.assertEqual(cm.exception.args[0], expected_msg)
Example #23
0
    def testRerunTensor(self):
        with new_cluster(scheduler_n_process=2, worker_n_process=2) as cluster:
            session = cluster.session

            a = mt.ones((10, 10)) + 1
            result1 = session.run(a)
            np.testing.assert_array_equal(result1, np.ones((10, 10)) + 1)
            result2 = session.run(a)
            np.testing.assert_array_equal(result1, result2)

            with new_session(cluster.endpoint) as session2:
                a = mt.random.rand(10, 10)
                a_result1 = session2.run(a)
                b = mt.ones((10, 10))
                a_result2, b_result = session2.run(a, b)
                np.testing.assert_array_equal(a_result1, a_result2)
                np.testing.assert_array_equal(b_result, np.ones((10, 10)))
Example #24
0
    def testRemoteFunctionInLocalCluster(self):
        with new_cluster(scheduler_n_process=2,
                         worker_n_process=3,
                         shared_memory='20M',
                         modules=[__name__],
                         web=True) as cluster:
            session = cluster.session

            def f(x):
                return x + 1

            def g(x, y):
                return x * y

            a = mr.spawn(f, 3)
            b = mr.spawn(f, 4)
            c = mr.spawn(g, (a, b))

            r = session.run(c, timeout=_exec_timeout)
            self.assertEqual(r, 20)

            e = mr.spawn(f, mr.spawn(f, 2))

            r = session.run(e, timeout=_exec_timeout)
            self.assertEqual(r, 4)

            session2 = new_session(cluster.endpoint)
            expect_session_id = session2.session_id

            def f2():
                session = Session.default
                assert isinstance(session._sess, ClusterSession)
                assert session._sess.session_id == expect_session_id

                t = mt.ones((3, 2))
                return t.sum().to_numpy()

            self.assertEqual(
                cloudpickle.loads(cloudpickle.dumps(
                    Session.default)).session_id, session.session_id)
            self.assertIsInstance(serialize_function(f2), bytes)

            d = mr.spawn(f2, retry_when_fail=False)

            r = session2.run(d, timeout=_exec_timeout)
            self.assertEqual(r, 6)
Example #25
0
    def testTileContextInLocalCluster(self):
        class FakeOp(TensorAbs):
            _op_type_ = 9870102948

            _multiplier = Int64Field('multiplier')

            @classmethod
            def tile(cls, op):
                context = get_context()

                self.assertEqual(context.running_mode,
                                 RunningMode.local_cluster)

                inp_chunk = op.inputs[0].chunks[0]
                inp_size = context.get_chunk_metas([inp_chunk.key
                                                    ])[0].chunk_size
                chunk_op = op.copy().reset_key()
                chunk_op._multiplier = inp_size
                chunk = chunk_op.new_chunk([inp_chunk], shape=inp_chunk.shape)

                new_op = op.copy()
                return new_op.new_tensors(op.inputs,
                                          shape=op.outputs[0].shape,
                                          order=op.outputs[0].order,
                                          nsplits=op.inputs[0].nsplits,
                                          chunks=[chunk])

            @classmethod
            def execute(cls, ctx, op):
                ctx[op.outputs[0].key] = ctx[op.inputs[0].key] * op._multiplier

        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M',
                         web=True) as cluster:
            session = cluster.session

            raw = np.random.rand(10, 20)
            data = mt.tensor(raw)

            session.run(data)

            data2 = FakeOp().new_tensor([data], shape=data.shape)

            result = session.run(data2)
            np.testing.assert_array_equal(raw * raw.nbytes, result)
Example #26
0
    def testLearnInLocalCluster(self, *_):
        from mars.learn.neighbors import NearestNeighbors
        from sklearn.neighbors import NearestNeighbors as SkNearestNeighbors
        from mars.learn.metrics import roc_curve, auc
        from sklearn.metrics import roc_curve as sklearn_roc_curve, auc as sklearn_auc

        with new_cluster(scheduler_n_process=2,
                         worker_n_process=3,
                         shared_memory='20M') as cluster:
            rs = np.random.RandomState(0)
            raw_X = rs.rand(10, 5)
            raw_Y = rs.rand(8, 5)

            X = mt.tensor(raw_X, chunk_size=7)
            Y = mt.tensor(raw_Y, chunk_size=(5, 3))
            nn = NearestNeighbors(n_neighbors=3)
            nn.fit(X)

            ret = nn.kneighbors(Y, session=cluster.session)

            snn = SkNearestNeighbors(n_neighbors=3)
            snn.fit(raw_X)
            expected = snn.kneighbors(raw_Y)

            result = [r.fetch() for r in ret]
            np.testing.assert_almost_equal(result[0], expected[0])
            np.testing.assert_almost_equal(result[1], expected[1])

            rs = np.random.RandomState(0)
            raw = pd.DataFrame({
                'a': rs.randint(0, 10, (10, )),
                'b': rs.rand(10)
            })

            df = md.DataFrame(raw)
            y = df['a'].to_tensor().astype('int')
            pred = df['b'].to_tensor().astype('float')
            fpr, tpr, thresholds = roc_curve(y, pred, pos_label=2)
            m = auc(fpr, tpr)

            sk_fpr, sk_tpr, sk_threshod = sklearn_roc_curve(
                raw['a'].to_numpy().astype('int'),
                raw['b'].to_numpy().astype('float'),
                pos_label=2)
            expect_m = sklearn_auc(sk_fpr, sk_tpr)
            self.assertAlmostEqual(m.fetch(), expect_m)
Example #27
0
    def testRerunTensor(self, *_):
        with new_cluster(scheduler_n_process=2, worker_n_process=2,
                         shared_memory='20M') as cluster:
            session = cluster.session

            a = mt.ones((10, 10)) + 1
            result1 = session.run(a, timeout=_exec_timeout)
            np.testing.assert_array_equal(result1, np.ones((10, 10)) + 1)
            result2 = session.run(a, timeout=_exec_timeout)
            np.testing.assert_array_equal(result1, result2)

            with new_session(cluster.endpoint) as session2:
                a = mt.random.rand(10, 10)
                a_result1 = session2.run(a, timeout=_exec_timeout)
                b = mt.ones((10, 10))
                a_result2, b_result = session2.run(a, b, timeout=_exec_timeout)
                np.testing.assert_array_equal(a_result1, a_result2)
                np.testing.assert_array_equal(b_result, np.ones((10, 10)))
Example #28
0
    def testIndexTensorExecute(self):
        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M') as cluster:
            session = cluster.session

            a = mt.random.rand(10, 5)
            idx = slice(0, 5), slice(0, 5)
            a[idx] = 2
            a_splits = mt.split(a, 2)
            r1, r2 = session.run(a_splits[0], a[idx])

            np.testing.assert_array_equal(r1, r2)
            np.testing.assert_array_equal(r1, np.ones((5, 5)) * 2)

            with new_session(cluster.endpoint) as session2:
                a = mt.random.rand(10, 5)
                idx = slice(0, 5), slice(0, 5)

                a[idx] = mt.ones((5, 5)) * 2
                r = session2.run(a[idx])

                np.testing.assert_array_equal(r, np.ones((5, 5)) * 2)

            with new_session(cluster.endpoint) as session3:
                a = mt.random.rand(100, 5)

                slice1 = a[:10]
                slice2 = a[10:20]
                r1, r2, expected = session3.run(slice1, slice2, a)

                np.testing.assert_array_equal(r1, expected[:10])
                np.testing.assert_array_equal(r2, expected[10:20])

            with new_session(cluster.endpoint) as session4:
                a = mt.random.rand(100, 5)

                a[:10] = mt.ones((10, 5))
                a[10:20] = 2
                r = session4.run(a)

                np.testing.assert_array_equal(r[:10], np.ones((10, 5)))
                np.testing.assert_array_equal(r[10:20], np.ones((10, 5)) * 2)
Example #29
0
    def testMutableTensorSeal(self):
        def testWithGivenSession(session):
            mut = session.create_mutable_tensor("test", (4, 5),
                                                dtype='int32',
                                                chunk_size=3)
            mut[1:4, 2] = 8
            mut[2:4] = np.arange(10).reshape(2, 5)
            mut[1] = np.arange(5)
            arr = mut.seal()

            expected = np.zeros((4, 5), dtype='int32')
            expected[1:4, 2] = 8
            expected[2:4] = np.arange(10).reshape(2, 5)
            expected[1] = np.arange(5)

            # check chunk properties
            for chunk1, chunk2 in zip(mut.chunks, arr.chunks):
                self.assertEqual(chunk1.key, chunk2.key)
                self.assertEqual(chunk1.index, chunk2.index)
                self.assertEqual(chunk1.shape, chunk2.shape)
                self.assertEqual(chunk1.dtype, chunk2.dtype)

            # check value
            np.testing.assert_array_equal(session.fetch(arr), expected)

            # check operations on the sealed tensor
            np.testing.assert_array_equal(session.run(arr + 1), expected + 1)
            np.testing.assert_array_equal(session.run(arr + arr),
                                          expected + expected)
            np.testing.assert_array_equal(session.run(arr.sum()),
                                          expected.sum())

        with new_cluster(scheduler_n_process=2,
                         worker_n_process=2,
                         shared_memory='20M',
                         web=True) as cluster:
            session = cluster.session.as_default()
            testWithGivenSession(session)

            with new_session(
                    'http://' +
                    cluster._web_endpoint).as_default() as web_session:
                testWithGivenSession(web_session)
Example #30
0
    def testStoreHDF5ForLocalCluster(self):
        with new_cluster(worker_n_process=2,
                         shared_memory='20M', web=True) as cluster:
            session = cluster.session

            raw = np.random.RandomState(0).rand(10, 20)
            t = mt.tensor(raw, chunk_size=11)

            dataset = 'test_dataset'
            with tempfile.TemporaryDirectory() as d:
                filename = os.path.join(d, 'test_read_{}.hdf5'.format(int(time.time())))

                r = mt.tohdf5(filename, t, dataset=dataset)

                session.run(r, timeout=_exec_timeout)

                with h5py.File(filename, 'r') as f:
                    result = np.asarray(f[dataset])
                    np.testing.assert_array_equal(result, raw)