def __init__(self,
hyperopt_sampler: HyperoptSampler,
output_feature: str,
metric: str,
split: str,
num_workers: int = 2,
num_cpus_per_worker: int = -1,
num_gpus_per_worker: int = -1,
fiber_backend: str = "local",
**kwargs) -> None:
import fiber
HyperoptExecutor.__init__(self, hyperopt_sampler, output_feature,
metric, split)
fiber.init(backend=fiber_backend)
self.fiber_meta = fiber.meta
self.num_cpus_per_worker = num_cpus_per_worker
self.num_gpus_per_worker = num_gpus_per_worker
self.resource_limits = {}
if num_cpus_per_worker != -1:
self.resource_limits["cpu"] = num_cpus_per_worker
if num_gpus_per_worker != -1:
self.resource_limits["gpu"] = num_gpus_per_worker
self.num_workers = num_workers
self.pool = fiber.Pool(num_workers)
def test_chunk_size(self):
log_file = os.path.join(os.path.dirname(__file__), "..", "logs",
"chunk.log")
try:
fiber.reset()
log_file = fiber_config.log_file
log_level = fiber_config.log_level
fiber.init(cpu_per_job=8, log_file=log_file, log_level="debug")
n_envs = 9
queues = [(SimpleQueue(), SimpleQueue()) for _ in range(n_envs)]
pool = Pool(n_envs)
# explicitly start workers instead of lazy start
pool.start_workers()
print("waiting for all workers to be up")
# wait some time for workers to start
pool.wait_until_workers_up()
#time.sleep(20)
print("all workers are up")
def run_map():
print('[master]RUN MAP')
# Not setting chunk size 1 here, if chunk size is calculated
# wrong, map will get stuck
pool.map(double_queue_worker, enumerate(queues), chunksize=1)
print('[master]RUN MAP DONE')
td = threading.Thread(target=run_map, daemon=True)
td.start()
print('Checking...')
for i, (_, returns) in enumerate(queues):
print('[master]Checking queue', i, n_envs)
assert 'READY' in returns.get()
print(f'[master]Checking queue {i} done')
print('All workers are ready, put HELLOs')
for i, (instruction, _) in enumerate(queues):
instruction.put("HELLO")
print(f'[master]PUT HELLO {i}')
print('All HELLOs sent, waiting for ACKs')
for i, (_, returns) in enumerate(queues):
assert 'ACK' in returns.get()
print(f'[master]GOT ACK {i}')
print('All ACKs sent, send QUIT to workers')
for i, (instruction, _) in enumerate(queues):
instruction.put("QUIT")
print(f'[master]PUT QUIT {i}, {n_envs}')
pool.terminate()
pool.join()
finally:
fiber.init(log_file=log_file, log_level=log_level)
def test_config_env(self):
import os
old_val = os.environ.get("FIBER_IMAGE", None)
os.environ["FIBER_IMAGE"] = "image-from-env:latest"
try:
fiber.init()
assert fiber_config.image == "image-from-env:latest"
finally:
if old_val is None:
del os.environ["FIBER_IMAGE"]
else:
os.environ["FIBER_IMAGE"] = old_val
fiber.init()
def test_image_not_found(self):
if fiber.config.default_backend != "docker":
pytest.skip("skipped because current backend is not docker")
try:
with pytest.raises(multiprocessing.ProcessError):
fiber.init(
image='this-image-does-not-exist-and-is-only-used-for-testing'
)
p = fiber.Process(name="test_image_not_found")
p.start()
finally:
fiber.reset()
def test_command_line(self):
fiber.init(use_bash=False)
try:
p = fiber.Process(name="test_command_line")
popen = fspawn.Popen(p, backend="docker", launch=False)
cmd = popen.get_command_line(host="127.0.0.1", port="8000", id=1)
finally:
fiber.init(use_bash=True)
assert cmd == [
"/usr/local/bin/python", "-c",
'import sys;import os;import socket;import struct;import fiber;import fiber.util;from multiprocessing import spawn, reduction;sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM);sock.connect(("127.0.0.1", 8000));conn = sock;conn.send(struct.pack("<I", 1));fd = conn.fileno();exitcode = fiber.util.spawn_prepare(fd);sys.exit(exitcode)',
'--multiprocessing-fork'
]
def test_init_image(self, mock_create_job):
if fiber.config.default_backend != "docker":
pytest.skip("skipped because current backend is not docker")
try:
fiber.init(image='python:3.6-alpine', backend='docker')
p = fiber.Process(name="test_init_image")
p.start()
except NotImplementedError:
pass
finally:
fiber.reset()
args = mock_create_job.call_args
# https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock.call_args # noqa E501
job_spec_arg = args[0][0]
assert job_spec_arg.image == "python:3.6-alpine"
def test_no_python3_inside_image(self):
if fiber.config.default_backend != "docker":
pytest.skip("skipped because current backend is not docker")
'''
fp = io.StringIO()
handler = logging.StreamHandler(fp)
logger = logging.getLogger("fiber")
logger.setLevel(level=logging.DEBUG)
logger.addHandler(handler)
'''
with pytest.raises(multiprocessing.ProcessError):
try:
fiber.init(image='ubuntu:18.04')
p = fiber.Process(name="test_no_python3_inside_image")
p.start()
finally:
fiber.reset()
#logger.removeHandler(handler)
'''
def test_log_file_path(self):
if fiber.config.default_backend != "docker":
pytest.skip("skipped because current backend is not docker")
# make sure workers and master doesn't write to the same log file
try:
log_file = fiber_config.log_file
log_level = fiber_config.log_level
# clean up
if os.path.isdir("tests/logs"):
files = glob.glob("tests/logs/*")
for f in files:
os.remove(f)
files = glob.glob("tests/logs/*")
assert files == []
fiber.init(log_file="tests/logs/fiber.log", log_level="DEBUG")
q = fiber.SimpleQueue()
p = fiber.Process(target=dummy_q_worker, args=(q,))
p.start()
msg = q.get()
assert msg == "DUMMY_WORKER_DONE"
files = glob.glob("tests/logs/*")
files.sort()
assert len(files) == 2
assert files[0] == "tests/logs/fiber.log.MainProcess"
assert "tests/logs/fiber.log.Process-" in files[1]
regex = r'\w+:Process-\d+\(\d+\):'
with open("tests/logs/fiber.log.MainProcess") as f:
content = f.read()
assert re.search(regex, content) is None, (
"Fiber subprocess log found in master log file")
p.join()
finally:
fiber.init(log_file=log_file, log_level=log_level)
def test_no_python3_inside_image(self):
if fiber.config.default_backend != "docker":
pytest.skip("skipped because current backend is not docker")
fp = io.StringIO()
handler = logging.StreamHandler(fp)
logger = logging.getLogger("fiber")
logger.setLevel(level=logging.DEBUG)
logger.addHandler(handler)
try:
fiber.init(image='ubuntu:18.04')
p = fiber.Process(name="test_no_python3_inside_image")
p.start()
logs = fp.getvalue()
times = len(list(re.finditer('Failed to start Fiber process',
logs)))
assert times == 1
finally:
fiber.reset()
logger.removeHandler(handler)
