def __init__(self, hyperopt_sampler: HyperoptSampler, output_feature: str, metric: str, split: str, num_workers: int = 2, num_cpus_per_worker: int = -1, num_gpus_per_worker: int = -1, fiber_backend: str = "local", **kwargs) -> None: import fiber HyperoptExecutor.__init__(self, hyperopt_sampler, output_feature, metric, split) fiber.init(backend=fiber_backend) self.fiber_meta = fiber.meta self.num_cpus_per_worker = num_cpus_per_worker self.num_gpus_per_worker = num_gpus_per_worker self.resource_limits = {} if num_cpus_per_worker != -1: self.resource_limits["cpu"] = num_cpus_per_worker if num_gpus_per_worker != -1: self.resource_limits["gpu"] = num_gpus_per_worker self.num_workers = num_workers self.pool = fiber.Pool(num_workers)
def test_chunk_size(self): log_file = os.path.join(os.path.dirname(__file__), "..", "logs", "chunk.log") try: fiber.reset() log_file = fiber_config.log_file log_level = fiber_config.log_level fiber.init(cpu_per_job=8, log_file=log_file, log_level="debug") n_envs = 9 queues = [(SimpleQueue(), SimpleQueue()) for _ in range(n_envs)] pool = Pool(n_envs) # explicitly start workers instead of lazy start pool.start_workers() print("waiting for all workers to be up") # wait some time for workers to start pool.wait_until_workers_up() #time.sleep(20) print("all workers are up") def run_map(): print('[master]RUN MAP') # Not setting chunk size 1 here, if chunk size is calculated # wrong, map will get stuck pool.map(double_queue_worker, enumerate(queues), chunksize=1) print('[master]RUN MAP DONE') td = threading.Thread(target=run_map, daemon=True) td.start() print('Checking...') for i, (_, returns) in enumerate(queues): print('[master]Checking queue', i, n_envs) assert 'READY' in returns.get() print(f'[master]Checking queue {i} done') print('All workers are ready, put HELLOs') for i, (instruction, _) in enumerate(queues): instruction.put("HELLO") print(f'[master]PUT HELLO {i}') print('All HELLOs sent, waiting for ACKs') for i, (_, returns) in enumerate(queues): assert 'ACK' in returns.get() print(f'[master]GOT ACK {i}') print('All ACKs sent, send QUIT to workers') for i, (instruction, _) in enumerate(queues): instruction.put("QUIT") print(f'[master]PUT QUIT {i}, {n_envs}') pool.terminate() pool.join() finally: fiber.init(log_file=log_file, log_level=log_level)
def test_config_env(self): import os old_val = os.environ.get("FIBER_IMAGE", None) os.environ["FIBER_IMAGE"] = "image-from-env:latest" try: fiber.init() assert fiber_config.image == "image-from-env:latest" finally: if old_val is None: del os.environ["FIBER_IMAGE"] else: os.environ["FIBER_IMAGE"] = old_val fiber.init()
def test_image_not_found(self): if fiber.config.default_backend != "docker": pytest.skip("skipped because current backend is not docker") try: with pytest.raises(multiprocessing.ProcessError): fiber.init( image='this-image-does-not-exist-and-is-only-used-for-testing' ) p = fiber.Process(name="test_image_not_found") p.start() finally: fiber.reset()
def test_command_line(self): fiber.init(use_bash=False) try: p = fiber.Process(name="test_command_line") popen = fspawn.Popen(p, backend="docker", launch=False) cmd = popen.get_command_line(host="127.0.0.1", port="8000", id=1) finally: fiber.init(use_bash=True) assert cmd == [ "/usr/local/bin/python", "-c", 'import sys;import os;import socket;import struct;import fiber;import fiber.util;from multiprocessing import spawn, reduction;sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM);sock.connect(("127.0.0.1", 8000));conn = sock;conn.send(struct.pack("<I", 1));fd = conn.fileno();exitcode = fiber.util.spawn_prepare(fd);sys.exit(exitcode)', '--multiprocessing-fork' ]
def test_init_image(self, mock_create_job): if fiber.config.default_backend != "docker": pytest.skip("skipped because current backend is not docker") try: fiber.init(image='python:3.6-alpine', backend='docker') p = fiber.Process(name="test_init_image") p.start() except NotImplementedError: pass finally: fiber.reset() args = mock_create_job.call_args # https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock.call_args # noqa E501 job_spec_arg = args[0][0] assert job_spec_arg.image == "python:3.6-alpine"
def test_no_python3_inside_image(self): if fiber.config.default_backend != "docker": pytest.skip("skipped because current backend is not docker") ''' fp = io.StringIO() handler = logging.StreamHandler(fp) logger = logging.getLogger("fiber") logger.setLevel(level=logging.DEBUG) logger.addHandler(handler) ''' with pytest.raises(multiprocessing.ProcessError): try: fiber.init(image='ubuntu:18.04') p = fiber.Process(name="test_no_python3_inside_image") p.start() finally: fiber.reset() #logger.removeHandler(handler) '''
def test_log_file_path(self): if fiber.config.default_backend != "docker": pytest.skip("skipped because current backend is not docker") # make sure workers and master doesn't write to the same log file try: log_file = fiber_config.log_file log_level = fiber_config.log_level # clean up if os.path.isdir("tests/logs"): files = glob.glob("tests/logs/*") for f in files: os.remove(f) files = glob.glob("tests/logs/*") assert files == [] fiber.init(log_file="tests/logs/fiber.log", log_level="DEBUG") q = fiber.SimpleQueue() p = fiber.Process(target=dummy_q_worker, args=(q,)) p.start() msg = q.get() assert msg == "DUMMY_WORKER_DONE" files = glob.glob("tests/logs/*") files.sort() assert len(files) == 2 assert files[0] == "tests/logs/fiber.log.MainProcess" assert "tests/logs/fiber.log.Process-" in files[1] regex = r'\w+:Process-\d+\(\d+\):' with open("tests/logs/fiber.log.MainProcess") as f: content = f.read() assert re.search(regex, content) is None, ( "Fiber subprocess log found in master log file") p.join() finally: fiber.init(log_file=log_file, log_level=log_level)
def test_no_python3_inside_image(self): if fiber.config.default_backend != "docker": pytest.skip("skipped because current backend is not docker") fp = io.StringIO() handler = logging.StreamHandler(fp) logger = logging.getLogger("fiber") logger.setLevel(level=logging.DEBUG) logger.addHandler(handler) try: fiber.init(image='ubuntu:18.04') p = fiber.Process(name="test_no_python3_inside_image") p.start() logs = fp.getvalue() times = len(list(re.finditer('Failed to start Fiber process', logs))) assert times == 1 finally: fiber.reset() logger.removeHandler(handler)