def initialize_dask():
"""Initialize Dask environment."""
from distributed.client import default_client
try:
client = default_client()
except ValueError:
from distributed import Client
# The indentation here is intentional, we want the code to be indented.
ErrorMessage.not_initialized(
"Dask",
"""
from distributed import Client
client = Client()
""",
)
num_cpus = CpuCount.get()
memory_limit = Memory.get()
worker_memory_limit = memory_limit // num_cpus if memory_limit else "auto"
client = Client(n_workers=num_cpus, memory_limit=worker_memory_limit)
num_cpus = len(client.ncores())
NPartitions._put(num_cpus)
def initialize_ray(
override_is_cluster=False,
override_redis_address: str = None,
override_redis_password: str = None,
):
"""
Initialize Ray based on parameters, ``modin.config`` variables and internal defaults.
Parameters
----------
override_is_cluster : bool, default: False
Whether to override the detection of Modin being run in a cluster
and always assume this runs on cluster head node.
This also overrides Ray worker detection and always runs the initialization
function (runs from main thread only by default).
If not specified, ``modin.config.IsRayCluster`` variable is used.
override_redis_address : str, optional
What Redis address to connect to when running in Ray cluster.
If not specified, ``modin.config.RayRedisAddress`` is used.
override_redis_password : str, optional
What password to use when connecting to Redis.
If not specified, ``modin.config.RayRedisPassword`` is used.
"""
import ray
if not ray.is_initialized() or override_is_cluster:
cluster = override_is_cluster or IsRayCluster.get()
redis_address = override_redis_address or RayRedisAddress.get()
redis_password = override_redis_password or RayRedisPassword.get()
if cluster:
# We only start ray in a cluster setting for the head node.
ray.init(
address=redis_address or "auto",
include_dashboard=False,
ignore_reinit_error=True,
_redis_password=redis_password,
)
else:
from modin.error_message import ErrorMessage
# This string is intentionally formatted this way. We want it indented in
# the warning message.
ErrorMessage.not_initialized(
"Ray",
"""
import ray
ray.init()
""",
)
object_store_memory = Memory.get()
# In case anything failed above, we can still improve the memory for Modin.
if object_store_memory is None:
virtual_memory = psutil.virtual_memory().total
if sys.platform.startswith("linux"):
shm_fd = os.open("/dev/shm", os.O_RDONLY)
try:
shm_stats = os.fstatvfs(shm_fd)
system_memory = shm_stats.f_bsize * shm_stats.f_bavail
if system_memory / (virtual_memory / 2) < 0.99:
warnings.warn(
f"The size of /dev/shm is too small ({system_memory} bytes). The required size "
f"at least half of RAM ({virtual_memory // 2} bytes). Please, delete files in /dev/shm or "
"increase size of /dev/shm with --shm-size in Docker. Also, you can set "
"the required memory size for each Ray worker in bytes to MODIN_MEMORY environment variable."
)
finally:
os.close(shm_fd)
else:
system_memory = virtual_memory
object_store_memory = int(0.6 * system_memory // 1e9 * 1e9)
# If the memory pool is smaller than 2GB, just use the default in ray.
if object_store_memory == 0:
object_store_memory = None
else:
object_store_memory = int(object_store_memory)
ray_init_kwargs = {
"num_cpus": CpuCount.get(),
"num_gpus": GpuCount.get(),
"include_dashboard": False,
"ignore_reinit_error": True,
"object_store_memory": object_store_memory,
"address": redis_address,
"_redis_password": redis_password,
"_memory": object_store_memory,
}
ray.init(**ray_init_kwargs)
if Backend.get() == "Cudf":
from modin.engines.ray.cudf_on_ray.frame.gpu_manager import GPUManager
from modin.engines.ray.cudf_on_ray.frame.partition_manager import (
GPU_MANAGERS, )
# Check that GPU_MANAGERS is empty because _update_engine can be called multiple times
if not GPU_MANAGERS:
for i in range(GpuCount.get()):
GPU_MANAGERS.append(GPUManager.remote(i))
_move_stdlib_ahead_of_site_packages()
ray.worker.global_worker.run_function_on_all_workers(
_move_stdlib_ahead_of_site_packages)
ray.worker.global_worker.run_function_on_all_workers(_import_pandas)
num_cpus = int(ray.cluster_resources()["CPU"])
num_gpus = int(ray.cluster_resources().get("GPU", 0))
if Backend.get() == "Cudf":
NPartitions._put(num_gpus)
else:
NPartitions._put(num_cpus)
def initialize_ray(
override_is_cluster=False,
override_redis_address: str = None,
override_redis_password: str = None,
):
"""
Initializes ray based on parameters, environment variables and internal defaults.
Parameters
----------
override_is_cluster: bool, optional
Whether to override the detection of Moding being run in a cluster
and always assume this runs on cluster head node.
This also overrides Ray worker detection and always runs the function,
not only from main thread.
If not specified, $MODIN_RAY_CLUSTER env variable is used.
override_redis_address: str, optional
What Redis address to connect to when running in Ray cluster.
If not specified, $MODIN_REDIS_ADDRESS is used.
override_redis_password: str, optional
What password to use when connecting to Redis.
If not specified, a new random one is generated.
"""
import ray
if not ray.is_initialized() or override_is_cluster:
import secrets
cluster = override_is_cluster or IsRayCluster.get()
redis_address = override_redis_address or RayRedisAddress.get()
redis_password = override_redis_password or secrets.token_hex(32)
if cluster:
# We only start ray in a cluster setting for the head node.
ray.init(
address=redis_address or "auto",
include_dashboard=False,
ignore_reinit_error=True,
_redis_password=redis_password,
logging_level=100,
)
else:
from modin.error_message import ErrorMessage
# This string is intentionally formatted this way. We want it indented in
# the warning message.
ErrorMessage.not_initialized(
"Ray",
"""
import ray
ray.init()
""",
)
object_store_memory = Memory.get()
plasma_directory = RayPlasmaDir.get()
if IsOutOfCore.get():
if plasma_directory is None:
from tempfile import gettempdir
plasma_directory = gettempdir()
# We may have already set the memory from the environment variable, we don't
# want to overwrite that value if we have.
if object_store_memory is None:
# Round down to the nearest Gigabyte.
try:
system_memory = ray._private.utils.get_system_memory()
except AttributeError: # Compatibility with Ray <= 1.2
system_memory = ray.utils.get_system_memory()
mem_bytes = system_memory // 10**9 * 10**9
# Default to 8x memory for out of core
object_store_memory = 8 * mem_bytes
# In case anything failed above, we can still improve the memory for Modin.
if object_store_memory is None:
# Round down to the nearest Gigabyte.
try:
system_memory = ray._private.utils.get_system_memory()
except AttributeError: # Compatibility with Ray <= 1.2
system_memory = ray.utils.get_system_memory()
object_store_memory = int(0.6 * system_memory // 10**9 * 10**9)
# If the memory pool is smaller than 2GB, just use the default in ray.
if object_store_memory == 0:
object_store_memory = None
else:
object_store_memory = int(object_store_memory)
ray_init_kwargs = {
"num_cpus": CpuCount.get(),
"include_dashboard": False,
"ignore_reinit_error": True,
"_plasma_directory": plasma_directory,
"object_store_memory": object_store_memory,
"address": redis_address,
"_redis_password": redis_password,
"logging_level": 100,
"_memory": object_store_memory,
"_lru_evict": True,
}
from packaging import version
# setting of `_lru_evict` parameter raises DeprecationWarning since ray 2.0.0.dev0
if version.parse(ray.__version__) >= version.parse("2.0.0.dev0"):
ray_init_kwargs.pop("_lru_evict")
ray.init(**ray_init_kwargs)
_move_stdlib_ahead_of_site_packages()
ray.worker.global_worker.run_function_on_all_workers(
_move_stdlib_ahead_of_site_packages)
ray.worker.global_worker.run_function_on_all_workers(_import_pandas)
num_cpus = int(ray.cluster_resources()["CPU"])
NPartitions._put(num_cpus)
def initialize_ray(
override_is_cluster=False,
override_redis_address: str = None,
override_redis_password: str = None,
):
"""
Initializes ray based on parameters, environment variables and internal defaults.
Parameters
----------
override_is_cluster: bool, optional
Whether to override the detection of Moding being run in a cluster
and always assume this runs on cluster head node.
This also overrides Ray worker detection and always runs the function,
not only from main thread.
If not specified, $MODIN_RAY_CLUSTER env variable is used.
override_redis_address: str, optional
What Redis address to connect to when running in Ray cluster.
If not specified, $MODIN_REDIS_ADDRESS is used.
override_redis_password: str, optional
What password to use when connecting to Redis.
If not specified, a new random one is generated.
"""
import ray
if threading.current_thread().name == "MainThread" or override_is_cluster:
import secrets
cluster = override_is_cluster or IsRayCluster.get()
redis_address = override_redis_address or RayRedisAddress.get()
redis_password = override_redis_password or secrets.token_hex(16)
if cluster:
# We only start ray in a cluster setting for the head node.
ray.init(
address=redis_address or "auto",
include_dashboard=False,
ignore_reinit_error=True,
_redis_password=redis_password,
logging_level=100,
)
else:
object_store_memory = Memory.get()
plasma_directory = RayPlasmaDir.get()
if IsOutOfCore.get():
if plasma_directory is None:
from tempfile import gettempdir
plasma_directory = gettempdir()
# We may have already set the memory from the environment variable, we don't
# want to overwrite that value if we have.
if object_store_memory is None:
# Round down to the nearest Gigabyte.
mem_bytes = ray.utils.get_system_memory() // 10**9 * 10**9
# Default to 8x memory for out of core
object_store_memory = 8 * mem_bytes
# In case anything failed above, we can still improve the memory for Modin.
if object_store_memory is None:
# Round down to the nearest Gigabyte.
object_store_memory = int(
0.6 * ray.utils.get_system_memory() // 10**9 * 10**9)
# If the memory pool is smaller than 2GB, just use the default in ray.
if object_store_memory == 0:
object_store_memory = None
else:
object_store_memory = int(object_store_memory)
ray.init(
num_cpus=CpuCount.get(),
include_dashboard=False,
ignore_reinit_error=True,
_plasma_directory=plasma_directory,
object_store_memory=object_store_memory,
address=redis_address,
_redis_password=redis_password,
logging_level=100,
_memory=object_store_memory,
_lru_evict=True,
)
_move_stdlib_ahead_of_site_packages()
ray.worker.global_worker.run_function_on_all_workers(
_move_stdlib_ahead_of_site_packages)
ray.worker.global_worker.run_function_on_all_workers(_import_pandas)
def initialize_ray(
override_is_cluster=False,
override_redis_address: str = None,
override_redis_password: str = None,
):
"""
Initialize Ray based on parameters, ``modin.config`` variables and internal defaults.
Parameters
----------
override_is_cluster : bool, default: False
Whether to override the detection of Modin being run in a cluster
and always assume this runs on cluster head node.
This also overrides Ray worker detection and always runs the initialization
function (runs from main thread only by default).
If not specified, ``modin.config.IsRayCluster`` variable is used.
override_redis_address : str, optional
What Redis address to connect to when running in Ray cluster.
If not specified, ``modin.config.RayRedisAddress`` is used.
override_redis_password : str, optional
What password to use when connecting to Redis.
If not specified, ``modin.config.RayRedisPassword`` is used.
"""
if not ray.is_initialized() or override_is_cluster:
cluster = override_is_cluster or IsRayCluster.get()
redis_address = override_redis_address or RayRedisAddress.get()
redis_password = (
(ray.ray_constants.REDIS_DEFAULT_PASSWORD if cluster else
RayRedisPassword.get()) if override_redis_password is None
and RayRedisPassword.get_value_source() == ValueSource.DEFAULT else
override_redis_password or RayRedisPassword.get())
if cluster:
# We only start ray in a cluster setting for the head node.
ray.init(
address=redis_address or "auto",
include_dashboard=False,
ignore_reinit_error=True,
_redis_password=redis_password,
)
else:
from modin.error_message import ErrorMessage
# This string is intentionally formatted this way. We want it indented in
# the warning message.
ErrorMessage.not_initialized(
"Ray",
"""
import ray
ray.init()
""",
)
object_store_memory = Memory.get()
# In case anything failed above, we can still improve the memory for Modin.
if object_store_memory is None:
virtual_memory = psutil.virtual_memory().total
if sys.platform.startswith("linux"):
shm_fd = os.open("/dev/shm", os.O_RDONLY)
try:
shm_stats = os.fstatvfs(shm_fd)
system_memory = shm_stats.f_bsize * shm_stats.f_bavail
if system_memory / (virtual_memory / 2) < 0.99:
warnings.warn(
f"The size of /dev/shm is too small ({system_memory} bytes). The required size "
+
f"at least half of RAM ({virtual_memory // 2} bytes). Please, delete files in /dev/shm or "
+
"increase size of /dev/shm with --shm-size in Docker. Also, you can set "
+
"the required memory size for each Ray worker in bytes to MODIN_MEMORY environment variable."
)
finally:
os.close(shm_fd)
else:
system_memory = virtual_memory
object_store_memory = int(0.6 * system_memory // 1e9 * 1e9)
# If the memory pool is smaller than 2GB, just use the default in ray.
if object_store_memory == 0:
object_store_memory = None
else:
object_store_memory = int(object_store_memory)
mac_size_limit = getattr(ray.ray_constants,
"MAC_DEGRADED_PERF_MMAP_SIZE_LIMIT", None)
if (sys.platform == "darwin" and mac_size_limit is not None
and object_store_memory > mac_size_limit):
warnings.warn(
"On Macs, Ray's performance is known to degrade with " +
"object store size greater than " +
f"{mac_size_limit / 2 ** 30:.4} GiB. Ray by default does "
+ "not allow setting an object store size greater than " +
"that. Modin is overriding that default limit because " +
"it would rather have a larger, slower object store " +
"than spill to disk more often. To override Modin's " +
"behavior, you can initialize Ray yourself.")
os.environ["RAY_ENABLE_MAC_LARGE_OBJECT_STORE"] = "1"
ray_init_kwargs = {
"num_cpus": CpuCount.get(),
"num_gpus": GpuCount.get(),
"include_dashboard": False,
"ignore_reinit_error": True,
"object_store_memory": object_store_memory,
"_redis_password": redis_password,
"_memory": object_store_memory,
}
ray.init(**ray_init_kwargs)
if StorageFormat.get() == "Cudf":
from modin.core.execution.ray.implementations.cudf_on_ray.partitioning import (
GPUManager,
GPU_MANAGERS,
)
# Check that GPU_MANAGERS is empty because _update_engine can be called multiple times
if not GPU_MANAGERS:
for i in range(GpuCount.get()):
GPU_MANAGERS.append(GPUManager.remote(i))
_move_stdlib_ahead_of_site_packages()
ray.worker.global_worker.run_function_on_all_workers(
_move_stdlib_ahead_of_site_packages)
ray.worker.global_worker.run_function_on_all_workers(_import_pandas)
num_cpus = int(ray.cluster_resources()["CPU"])
num_gpus = int(ray.cluster_resources().get("GPU", 0))
if StorageFormat.get() == "Cudf":
NPartitions._put(num_gpus)
else:
NPartitions._put(num_cpus)