def simulate_cloud(request):
mode = request.config.getoption("--simulate-cloud").lower()
if mode == "off":
yield
return
if mode not in ("normal", "experimental"):
raise ValueError(f"Unsupported --simulate-cloud mode: {mode}")
assert IsExperimental.get(
), "Simulated cloud must be started in experimental mode"
from modin.experimental.cloud import create_cluster, get_connection
import modin.pandas.test.utils
with create_cluster("local", cluster_type="local"):
get_connection().teleport(set_experimental_env)(mode)
with Patcher(
get_connection(),
(modin.pandas.test.utils, "assert_index_equal"),
(modin.pandas.test.utils, "assert_series_equal"),
(modin.pandas.test.utils, "assert_frame_equal"),
(modin.pandas.test.utils, "assert_extension_array_equal"),
(modin.pandas.test.utils, "assert_empty_frame_equal"),
):
yield
def simulate_cloud(request):
mode = request.config.getoption("--simulate-cloud").lower()
if mode == "off":
yield
return
if mode not in ("normal", "experimental"):
raise ValueError(f"Unsupported --simulate-cloud mode: {mode}")
assert IsExperimental.get(), "Simulated cloud must be started in experimental mode"
from modin.experimental.cloud import create_cluster, get_connection
import pandas._testing
import pandas._libs.testing as cyx_testing
with create_cluster("local", cluster_type="local"):
get_connection().teleport(set_experimental_env)(mode)
with Patcher(
get_connection(),
(pandas._testing, "assert_class_equal"),
(cyx_testing, "assert_almost_equal"),
):
yield
def simulate_cloud(request):
mode = request.config.getoption("--simulate-cloud").lower()
if mode == "off":
yield
return
if mode not in ("normal", "experimental"):
raise ValueError(f"Unsupported --simulate-cloud mode: {mode}")
os.environ["MODIN_EXPERIMENTAL"] = "True"
from modin.experimental.cloud import create_cluster, get_connection
with create_cluster("local", __spawner__="local"):
def set_env(mode):
import os
os.environ["MODIN_EXPERIMENTAL"] = (
"True" if mode == "experimental" else "False"
)
get_connection().teleport(set_env)(mode)
yield
# pip install git+https://github.com/intel-ai/ibis.git@develop
# pip install braceexpand
# NOTE: expects https://github.com/intel-ai/omniscripts checked out and in PYTHONPATH
# the following import turns on experimental mode in Modin,
# including enabling running things in remote cloud
import modin.experimental.pandas as pd # noqa: F401
from modin.experimental.cloud import create_cluster
from h2o import run_benchmark
test_cluster = create_cluster(
"aws",
"aws_credentials",
cluster_name="rayscale-test",
region="eu-north-1",
zone="eu-north-1b",
image="ami-00e1e82d7d4ca80d3",
)
with test_cluster:
parameters = {
"no_pandas": False,
"pandas_mode": "Modin_on_ray",
"ray_tmpdir": "/tmp",
"ray_memory": 1024 * 1024 * 1024,
"extended_functionality": False,
}
# G1... - for groupby queries; J1... - for join queries;
# Additional required files inside h2o-data folder:
# - J1_1e6_1e0_0_0.csv
# the following import turns on experimental mode in Modin,
# including enabling running things in remote cloud
import modin.experimental.pandas as pd # noqa: F401
from modin.experimental.cloud import create_cluster
from taxi import run_benchmark as run_benchmark
cluster_params = {}
if USE_OMNISCI:
cluster_params["cluster_type"] = "omnisci"
test_cluster = create_cluster(
"aws",
"aws_credentials",
cluster_name="rayscale-test",
region="eu-central-1",
zone="eu-central-1b",
image="ami-05f7491af5eef733a",
**cluster_params,
)
with test_cluster:
if USE_OMNISCI:
from modin.experimental.cloud import get_connection
# We should move omniscripts trigger in remote conext
# https://github.com/intel-ai/omniscripts/blob/7d4599bcacf51de876952c658048571d32275ac1/taxi/taxibench_pandas_ibis.py#L482
import modin.experimental.engines.omnisci_on_native.frame.omnisci_worker
OmnisciServer = (get_connection().modules[
"modin.experimental.engines.omnisci_on_native.frame.omnisci_worker"]
.OmnisciServer)
