def testLazyImport(self):
old_sys_path = sys.path
mock_mod = textwrap.dedent("""
__version__ = '0.1.0b1'
""".strip())
temp_dir = tempfile.mkdtemp(prefix='mars-utils-test-')
sys.path += [temp_dir]
try:
with open(os.path.join(temp_dir, 'test_mod.py'), 'w') as outf:
outf.write(mock_mod)
non_exist_mod = utils.lazy_import('non_exist_mod', locals=locals())
self.assertIsNone(non_exist_mod)
mod = utils.lazy_import('test_mod',
globals=globals(),
locals=locals(),
rename='mod')
self.assertIsNotNone(mod)
self.assertEqual(mod.__version__, '0.1.0b1')
glob = globals().copy()
mod = utils.lazy_import('test_mod',
globals=glob,
locals=locals(),
rename='mod')
glob['mod'] = mod
self.assertIsNotNone(mod)
self.assertEqual(mod.__version__, '0.1.0b1')
self.assertEqual(type(glob['mod']).__name__, 'module')
finally:
shutil.rmtree(temp_dir)
sys.path = old_sys_path
from mars.executor import Executor, GraphExecution
from mars.graph import SerializableGraph
from mars.optimizes.chunk_graph.fuse import Fusion
from mars.serialize import serializes, deserializes, \
ProtobufSerializeProvider, JsonSerializeProvider
from mars.utils import lazy_import
try:
import pytest
except ImportError:
pytest = None
from unittest import mock
_mock = mock
cupy = lazy_import('cupy', globals=globals())
cudf = lazy_import('cudf', globals=globals())
logger = logging.getLogger(__name__)
class TestCase(unittest.TestCase):
pass
class MultiGetDict(dict):
def __getitem__(self, item):
if isinstance(item, tuple):
return tuple(
super(MultiGetDict, self).__getitem__(it) for it in item)
return super(MultiGetDict, self).__getitem__(item)
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import numpy as np
import mars.tensor as mt
from mars.learn.tests.integrated.base import LearnIntegrationTestBase
from mars.learn.contrib.pytorch import MarsDataset, MarsDistributedSampler
from mars.context import DistributedContext
from mars.session import new_session
from mars.utils import lazy_import
torch_installed = lazy_import('torch', globals=globals()) is not None
@unittest.skipIf(not torch_installed, 'pytorch not installed')
class Test(LearnIntegrationTestBase):
def testDistributedDataset(self):
service_ep = 'http://127.0.0.1:' + self.web_port
scheduler_ep = '127.0.0.1:' + self.scheduler_port
with new_session(service_ep) as sess:
raw = np.random.rand(100, 200)
data = mt.tensor(raw, chunk_size=40)
data.execute(name='data', session=sess)
with DistributedContext(scheduler_address=scheduler_ep, session_id=sess.session_id):
dataset = MarsDataset('data')
self.assertEqual(len(dataset), 100)
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import subprocess
import pytest
from mars.config import option_context
from mars.lib.aio import stop_isolation
from mars.oscar.backends.router import Router
from mars.oscar.backends.ray.communication import RayServer
from mars.serialization.ray import register_ray_serializers, unregister_ray_serializers
from mars.utils import lazy_import
ray = lazy_import('ray')
@pytest.fixture
def ray_start_regular(request):
param = getattr(request, "param", {})
if not param.get('enable', True):
yield
else:
register_ray_serializers()
try:
yield ray.init(num_cpus=20)
finally:
ray.shutdown()
unregister_ray_serializers()
Router.set_instance(None)
import uuid
import weakref
import numpy as np
import pandas as pd
from numpy.testing import assert_allclose
from mars.serialize import dataserializer
from mars.tests.core import require_cupy, require_cudf
from mars.utils import get_next_port, lazy_import
from mars.worker import WorkerDaemonActor, QuotaActor, MemQuotaActor
from mars.worker.tests.base import WorkerCase
from mars.worker.storage import StorageManagerActor, CudaHolderActor, PlasmaKeyMapActor, \
SharedHolderActor, DataStorageDevice
cp = lazy_import('cupy', globals=globals(), rename='cp')
cudf = lazy_import('cudf', globals=globals())
@require_cupy
@require_cudf
class Test(WorkerCase):
def testCudaMemPutAndGet(self):
test_addr = f'127.0.0.1:{get_next_port()}'
with self.create_pool(n_process=1, address=test_addr) as pool, \
self.run_actor_test(pool) as test_actor:
pool.create_actor(WorkerDaemonActor, uid=WorkerDaemonActor.default_uid())
storage_manager_ref = pool.create_actor(
StorageManagerActor, uid=StorageManagerActor.default_uid())
pool.create_actor(QuotaActor, 1024 ** 2, uid=MemQuotaActor.default_uid())
pool.create_actor(CudaHolderActor)
# limitations under the License.
import os
import tempfile
import unittest
import numpy as np
import pandas as pd
import mars.tensor as mt
import mars.dataframe as md
from mars.session import new_session
from mars.tests.core import flaky, require_ray
from mars.utils import lazy_import
ray = lazy_import('ray', globals=globals())
@require_ray
class Test(unittest.TestCase):
@classmethod
def tearDownClass(cls) -> None:
ray.shutdown()
def testRayTask(self):
with new_session(backend='ray').as_default():
# test tensor task
raw = np.random.rand(100, 100)
t = (mt.tensor(raw, chunk_size=30) + 1).sum().to_numpy()
self.assertAlmostEqual(t, (raw + 1).sum())
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import numpy as np
import pandas as pd
import mars.tensor as mt
import mars.dataframe as md
from mars.session import new_session
from mars.utils import lazy_import
ray_installed = lazy_import('ray', globals=globals()) is not None
@unittest.skipIf(not ray_installed, 'ray not installed')
class Test(unittest.TestCase):
def testRayTask(self):
with new_session(backend='ray').as_default():
# test tensor task
raw = np.random.rand(100, 100)
t = mt.tensor(raw, chunk_size=30).sum().to_numpy()
self.assertAlmostEqual(t, raw.sum())
# test DataFrame task
raw = pd.DataFrame(np.random.random((20, 4)), columns=list('abcd'))
df = md.DataFrame(raw, chunk_size=5)
r = df.describe().to_pandas()
import pandas as pd
try:
import pyarrow as pa
except ImportError: # pragma: no cover
pa = None
import mars.dataframe as md
from mars.config import option_context
from mars.core.session import get_default_session, SyncSession
from mars.dataframe import CustomReduction, NamedAgg
from mars.dataframe.base import to_gpu
from mars.tests import setup
from mars.tests.core import require_cudf, require_cupy
from mars.utils import lazy_import
cp = lazy_import('cupy', rename='cp', globals=globals())
setup = setup
@pytest.fixture
def check_ref_counts():
yield
sess = get_default_session()
assert len(SyncSession(sess)._get_ref_counts()) == 0
class FunctionOptions(NamedTuple):
has_min_count: bool = False
# limitations under the License.
import os
import subprocess
import pytest
from mars.config import option_context
from mars.core.mode import is_kernel_mode, is_build_mode
from mars.lib.aio import stop_isolation
from mars.oscar.backends.router import Router
from mars.oscar.backends.ray.communication import RayServer
from mars.serialization.ray import register_ray_serializers, unregister_ray_serializers
from mars.utils import lazy_import
ray = lazy_import("ray")
@pytest.fixture
def ray_start_regular(request):
param = getattr(request, "param", {})
if not param.get("enable", True):
yield
else:
register_ray_serializers()
try:
yield ray.init(num_cpus=20)
finally:
ray.shutdown()
unregister_ray_serializers()
Router.set_instance(None)
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from mars.contrib.dask import convert_dask_collection, mars_scheduler
from mars.utils import lazy_import
dask_installed = lazy_import('dask', globals=globals()) is not None
mimesis_installed = lazy_import('mimesis', globals=globals()) is not None
@pytest.mark.skipif(not dask_installed, reason='dask not installed')
def test_delayed(setup_cluster):
from dask import delayed
import numpy as np
def calc_chunk(n: int, i: int):
rs = np.random.RandomState(i)
a = rs.uniform(-1, 1, size=(n, 2))
d = np.linalg.norm(a, axis=1)
return (d < 1).sum()
def calc_pi(fs, N):