def simulate_cloud(request): mode = request.config.getoption("--simulate-cloud").lower() if mode == "off": yield return if mode not in ("normal", "experimental"): raise ValueError(f"Unsupported --simulate-cloud mode: {mode}") assert IsExperimental.get( ), "Simulated cloud must be started in experimental mode" from modin.experimental.cloud import create_cluster, get_connection import modin.pandas.test.utils with create_cluster("local", cluster_type="local"): get_connection().teleport(set_experimental_env)(mode) with Patcher( get_connection(), (modin.pandas.test.utils, "assert_index_equal"), (modin.pandas.test.utils, "assert_series_equal"), (modin.pandas.test.utils, "assert_frame_equal"), (modin.pandas.test.utils, "assert_extension_array_equal"), (modin.pandas.test.utils, "assert_empty_frame_equal"), ): yield
def simulate_cloud(request): mode = request.config.getoption("--simulate-cloud").lower() if mode == "off": yield return if mode not in ("normal", "experimental"): raise ValueError(f"Unsupported --simulate-cloud mode: {mode}") assert IsExperimental.get(), "Simulated cloud must be started in experimental mode" from modin.experimental.cloud import create_cluster, get_connection import pandas._testing import pandas._libs.testing as cyx_testing with create_cluster("local", cluster_type="local"): get_connection().teleport(set_experimental_env)(mode) with Patcher( get_connection(), (pandas._testing, "assert_class_equal"), (cyx_testing, "assert_almost_equal"), ): yield
def simulate_cloud(request): mode = request.config.getoption("--simulate-cloud").lower() if mode == "off": yield return if mode not in ("normal", "experimental"): raise ValueError(f"Unsupported --simulate-cloud mode: {mode}") os.environ["MODIN_EXPERIMENTAL"] = "True" from modin.experimental.cloud import create_cluster, get_connection with create_cluster("local", __spawner__="local"): def set_env(mode): import os os.environ["MODIN_EXPERIMENTAL"] = ( "True" if mode == "experimental" else "False" ) get_connection().teleport(set_env)(mode) yield
# pip install git+https://github.com/intel-ai/ibis.git@develop # pip install braceexpand # NOTE: expects https://github.com/intel-ai/omniscripts checked out and in PYTHONPATH # the following import turns on experimental mode in Modin, # including enabling running things in remote cloud import modin.experimental.pandas as pd # noqa: F401 from modin.experimental.cloud import create_cluster from h2o import run_benchmark test_cluster = create_cluster( "aws", "aws_credentials", cluster_name="rayscale-test", region="eu-north-1", zone="eu-north-1b", image="ami-00e1e82d7d4ca80d3", ) with test_cluster: parameters = { "no_pandas": False, "pandas_mode": "Modin_on_ray", "ray_tmpdir": "/tmp", "ray_memory": 1024 * 1024 * 1024, "extended_functionality": False, } # G1... - for groupby queries; J1... - for join queries; # Additional required files inside h2o-data folder: # - J1_1e6_1e0_0_0.csv
# the following import turns on experimental mode in Modin, # including enabling running things in remote cloud import modin.experimental.pandas as pd # noqa: F401 from modin.experimental.cloud import create_cluster from taxi import run_benchmark as run_benchmark cluster_params = {} if USE_OMNISCI: cluster_params["cluster_type"] = "omnisci" test_cluster = create_cluster( "aws", "aws_credentials", cluster_name="rayscale-test", region="eu-central-1", zone="eu-central-1b", image="ami-05f7491af5eef733a", **cluster_params, ) with test_cluster: if USE_OMNISCI: from modin.experimental.cloud import get_connection # We should move omniscripts trigger in remote conext # https://github.com/intel-ai/omniscripts/blob/7d4599bcacf51de876952c658048571d32275ac1/taxi/taxibench_pandas_ibis.py#L482 import modin.experimental.engines.omnisci_on_native.frame.omnisci_worker OmnisciServer = (get_connection().modules[ "modin.experimental.engines.omnisci_on_native.frame.omnisci_worker"] .OmnisciServer)