def get_num_workers(parallelism: Union[int, float],
total_lines: int,
chunk_size: int = 5) -> int:
"""
Given a parallelism setting and a number of total lines, compute the number of parallel workers to be used.
Args:
parallelism: The number of concurrent workers to use. ``1`` (the default) disables parallelization,
while a non-positive value means "number of CPUs + x" (i.e., use ``0`` for using as many workers
as there are CPUs). A floating-point value is interpreted as a fraction of the available CPUs,
rounded down.
total_lines: The total number of lines to generate.
chunk_size: the size of an individual unit of work to be distributed to workers.
Returns:
The number of required workers.
"""
num_chunks = (total_lines - 1) // chunk_size + 1
non_positive = False
if parallelism <= 0:
parallelism = -parallelism
non_positive = True
if isinstance(parallelism, float):
num_workers = int(loky.cpu_count() * parallelism)
else:
num_workers = parallelism
if non_positive:
num_workers = loky.cpu_count() - num_workers
num_workers = min(max(num_workers, 1), num_chunks)
return num_workers
def test_only_physical_cores_with_user_limitation():
# Check that user limitation for the available number of cores is
# respected even if only_physical_cores == True
cpu_count_mp = mp.cpu_count()
cpu_count_user = _cpu_count_user(cpu_count_mp)
if cpu_count_user < cpu_count_mp:
assert cpu_count() == cpu_count_user
assert cpu_count(only_physical_cores=True) == cpu_count_user
def test_cpu_count():
cpus = cpu_count()
assert type(cpus) is int
assert cpus >= 1
cpus_physical = cpu_count(only_physical_cores=True)
assert type(cpus_physical) is int
assert 1 <= cpus_physical <= cpus
# again to check that it's correctly cached
cpus_physical = cpu_count(only_physical_cores=True)
assert type(cpus_physical) is int
assert 1 <= cpus_physical <= cpus
def test_only_physical_cores_error():
# Check the warning issued by cpu_count(only_physical_cores=True) when
# unable to retrieve the number of physical cores.
if sys.platform != "linux":
pytest.skip()
# if number of available cpus is already restricted, cpu_count will return
# that value and no warning is issued even if only_physical_cores == True.
# (tested in another test: test_only_physical_cores_with_user_limitation
cpu_count_mp = mp.cpu_count()
if _cpu_count_user(cpu_count_mp) < cpu_count_mp:
pytest.skip()
start_dir = os.path.abspath('.')
with tempfile.TemporaryDirectory() as tmp_dir:
# Write bad lscpu program
lscpu_path = tmp_dir + '/lscpu'
with open(lscpu_path, 'w') as f:
f.write("#!/bin/sh\n" "exit(1)")
os.chmod(lscpu_path, 0o777)
try:
old_path = os.environ['PATH']
os.environ['PATH'] = tmp_dir + ":" + old_path
# clear the cache otherwise the warning is not triggered
import loky.backend.context
loky.backend.context.physical_cores_cache = None
with pytest.warns(UserWarning,
match="Could not find the number of"
" physical cores"):
cpu_count(only_physical_cores=True)
# Should not warn the second time
with pytest.warns(None) as record:
cpu_count(only_physical_cores=True)
assert not record
finally:
os.environ['PATH'] = old_path
def test_reusable_executor_reuse_true(self):
executor = get_reusable_executor(max_workers=3, timeout=42)
executor.submit(id, 42).result()
assert len(executor._processes) == 3
assert executor._timeout == 42
executor2 = get_reusable_executor(reuse=True)
executor2.submit(id, 42).result()
assert len(executor2._processes) == 3
assert executor2._timeout == 42
assert executor2 is executor
executor3 = get_reusable_executor()
executor3.submit(id, 42).result()
assert len(executor3._processes) == cpu_count()
assert executor3._timeout == 10
assert executor3 is not executor
executor4 = get_reusable_executor()
assert executor4 is executor3
def test_cpu_count():
cpus = cpu_count()
assert type(cpus) is int
assert cpus >= 1
from __future__ import print_function
import os
import sys
import time
import math
import psutil
import pytest
import threading
from loky._base import TimeoutError
from loky import get_reusable_executor, cpu_count
# Compat Travis
CPU_COUNT = cpu_count()
if os.environ.get("TRAVIS_OS_NAME") is not None and sys.version_info < (3, 4):
# Hard code number of cpu in travis as cpu_count return 32 whereas we
# only access 2 cores.
# This is done automatically by cpu_count for Python >= 3.4
CPU_COUNT = 2
# Compat windows
try:
# Increase time if the test is perform on a slow machine
TIMEOUT = max(20 / CPU_COUNT, 5)
except ImportError:
TIMEOUT = 20
_test_event = None
def default_cpus():
if "NEWS_PLEASE_WORKERS" in os.environ:
return int(os.environ["NEWS_PLEASE_WORKERS"])
else:
return cpu_count()
