def get_num_workers(parallelism: Union[int, float], total_lines: int, chunk_size: int = 5) -> int: """ Given a parallelism setting and a number of total lines, compute the number of parallel workers to be used. Args: parallelism: The number of concurrent workers to use. ``1`` (the default) disables parallelization, while a non-positive value means "number of CPUs + x" (i.e., use ``0`` for using as many workers as there are CPUs). A floating-point value is interpreted as a fraction of the available CPUs, rounded down. total_lines: The total number of lines to generate. chunk_size: the size of an individual unit of work to be distributed to workers. Returns: The number of required workers. """ num_chunks = (total_lines - 1) // chunk_size + 1 non_positive = False if parallelism <= 0: parallelism = -parallelism non_positive = True if isinstance(parallelism, float): num_workers = int(loky.cpu_count() * parallelism) else: num_workers = parallelism if non_positive: num_workers = loky.cpu_count() - num_workers num_workers = min(max(num_workers, 1), num_chunks) return num_workers
def test_only_physical_cores_with_user_limitation(): # Check that user limitation for the available number of cores is # respected even if only_physical_cores == True cpu_count_mp = mp.cpu_count() cpu_count_user = _cpu_count_user(cpu_count_mp) if cpu_count_user < cpu_count_mp: assert cpu_count() == cpu_count_user assert cpu_count(only_physical_cores=True) == cpu_count_user
def test_cpu_count(): cpus = cpu_count() assert type(cpus) is int assert cpus >= 1 cpus_physical = cpu_count(only_physical_cores=True) assert type(cpus_physical) is int assert 1 <= cpus_physical <= cpus # again to check that it's correctly cached cpus_physical = cpu_count(only_physical_cores=True) assert type(cpus_physical) is int assert 1 <= cpus_physical <= cpus
def test_only_physical_cores_error(): # Check the warning issued by cpu_count(only_physical_cores=True) when # unable to retrieve the number of physical cores. if sys.platform != "linux": pytest.skip() # if number of available cpus is already restricted, cpu_count will return # that value and no warning is issued even if only_physical_cores == True. # (tested in another test: test_only_physical_cores_with_user_limitation cpu_count_mp = mp.cpu_count() if _cpu_count_user(cpu_count_mp) < cpu_count_mp: pytest.skip() start_dir = os.path.abspath('.') with tempfile.TemporaryDirectory() as tmp_dir: # Write bad lscpu program lscpu_path = tmp_dir + '/lscpu' with open(lscpu_path, 'w') as f: f.write("#!/bin/sh\n" "exit(1)") os.chmod(lscpu_path, 0o777) try: old_path = os.environ['PATH'] os.environ['PATH'] = tmp_dir + ":" + old_path # clear the cache otherwise the warning is not triggered import loky.backend.context loky.backend.context.physical_cores_cache = None with pytest.warns(UserWarning, match="Could not find the number of" " physical cores"): cpu_count(only_physical_cores=True) # Should not warn the second time with pytest.warns(None) as record: cpu_count(only_physical_cores=True) assert not record finally: os.environ['PATH'] = old_path
def test_reusable_executor_reuse_true(self): executor = get_reusable_executor(max_workers=3, timeout=42) executor.submit(id, 42).result() assert len(executor._processes) == 3 assert executor._timeout == 42 executor2 = get_reusable_executor(reuse=True) executor2.submit(id, 42).result() assert len(executor2._processes) == 3 assert executor2._timeout == 42 assert executor2 is executor executor3 = get_reusable_executor() executor3.submit(id, 42).result() assert len(executor3._processes) == cpu_count() assert executor3._timeout == 10 assert executor3 is not executor executor4 = get_reusable_executor() assert executor4 is executor3
def test_cpu_count(): cpus = cpu_count() assert type(cpus) is int assert cpus >= 1
from __future__ import print_function import os import sys import time import math import psutil import pytest import threading from loky._base import TimeoutError from loky import get_reusable_executor, cpu_count # Compat Travis CPU_COUNT = cpu_count() if os.environ.get("TRAVIS_OS_NAME") is not None and sys.version_info < (3, 4): # Hard code number of cpu in travis as cpu_count return 32 whereas we # only access 2 cores. # This is done automatically by cpu_count for Python >= 3.4 CPU_COUNT = 2 # Compat windows try: # Increase time if the test is perform on a slow machine TIMEOUT = max(20 / CPU_COUNT, 5) except ImportError: TIMEOUT = 20 _test_event = None
def default_cpus(): if "NEWS_PLEASE_WORKERS" in os.environ: return int(os.environ["NEWS_PLEASE_WORKERS"]) else: return cpu_count()