def test_simple_queue_read_write_from_different_proc(self):
# One process writew, another process reads and then writes to q_out.
# Then master process reads from q_out.
n = 10
q = SimpleQueue()
q_out = SimpleQueue()
p1 = fiber.Process(target=put_queue, args=(q, [i for i in range(n)]))
p2 = fiber.Process(target=get_queue, args=(q, q_out, n))
p1.start()
p2.start()
for i in range(n):
data = q_out.get()
assert data == i
p1.join()
p2.join()
def test_queue_balance(self):
# We only test SimpleQueuePush because SimpleQueuePull doesn't gurantee
# balance.
inqueue = fiber.queues.SimpleQueuePush()
outqueue = fiber.queues.SimpleQueuePush()
num_workers = 4
multiplier = 600
workers = []
results = []
for i in range(num_workers):
print("create worker", i)
p = fiber.Process(target=worker, args=(inqueue, outqueue, i), daemon=True)
workers.append(p)
for i in range(num_workers):
workers[i].start()
# wait for all workers to connect
time.sleep(1)
for i in range(num_workers * multiplier):
inqueue.put("work")
for i in range(num_workers * multiplier):
results.append(outqueue.get())
stats = collections.Counter(results)
total = num_workers * multiplier
# send singals to all workers
for i in range(num_workers * multiplier):
inqueue.put("quit")
for i in range(num_workers):
workers[i].join()
for i in range(num_workers):
#print("{}: {} {:.2f}".format(i, stats[i], stats[i] / float(total)))
# data should be fairly queued
assert stats[i] == 600
def test_simple_queue_fiber(self):
q = SimpleQueue()
p = fiber.Process(target=put_queue, args=(q, 10))
p.start()
p.join()
data = q.get()
assert data == 10
def test_launch_fiber_background_thread(self):
# reset global variable
fiber.popen_fiber_spawn._fiber_background_thread = None
fp = io.StringIO()
handler = logging.StreamHandler(fp)
logger = logging.getLogger("fiber")
logger.setLevel(level=logging.DEBUG)
logger.addHandler(handler)
try:
procs = []
for i in range(4):
p = fiber.Process(
target=time.sleep,
args=(1, ),
name="test_launch_fiber_background_thread_{}".format(i))
procs.append(p)
for p in procs:
p.start()
logs = fp.getvalue()
times = len(list(re.finditer('creating background thread', logs)))
assert (times == 1)
finally:
for p in procs:
p.join()
logger.removeHandler(handler)
def test_subprocess_with_pipe(self):
reader, writer = Pipe()
p = fiber.Process(target=write_pipe, args=(writer, b"fiber pipe"))
p.start()
msg = reader.recv()
p.join()
assert msg == b"fiber pipe"
def run(self):
"""
Start this Ring. This will start the ring 0 process on the same machine
and start all the other ring nodes with Fiber processes.
"""
if self.size <= 0:
return
procs = []
rank = self.rank
# Start process rank 0
self.rank = 0
ctx = mp.get_context("spawn")
p = ctx.Process(target=self._target)
p.start()
procs.append(p)
for i in range(1, self.size):
self.rank = i
p = fiber.Process(target=self._target)
p.start()
procs.append(p)
self.rank = rank
# wait for all processes to finish
for i in range(self.size):
procs[i].join()
def test_managers_basic(self):
manager = fiber.Manager()
ns = manager.Namespace()
ns.x = 1
ns.y = [1]
ns.z = [1]
ls = manager.list([1, [1], [1], manager.list([1])])
di = manager.dict({0: 1, 1: [1], 2: [1], 3: manager.list([1])})
p = fiber.Process(target=f,
args=(ns, ls, di),
name="test_managers_basic")
p.start()
p.join()
assert ns.x == 2
assert ns.y == [1]
assert ns.z == [2]
assert ls[0] == 2
assert ls[1] == [1]
assert ls[2] == [2]
assert ls[3][0] == 2
assert len(ls) == 4
assert di[0] == 2
assert di[1] == [1]
assert di[2] == [2]
assert di[3][0] == 2
def test_sharing_pipe_with_fiber_and_multiprocessing(self):
reader, writer = Pipe()
p = fiber.Process(target=pipe_mp_worker, args=(writer,))
p.start()
msg = reader.recv()
p.join()
assert msg == b"hello"
def test_pipe_duplex_over_fiber_process(self):
conn1, conn2 = Pipe(duplex=True)
p = fiber.Process(target=pipe_worker, args=(conn2,))
p.start()
conn1.send(b"hello")
data = conn1.recv()
p.join()
assert data == b"ack"
def test_simple_queue_fiber_multi(self):
# Read and write multiple objects
n = 10
q = SimpleQueue()
p = fiber.Process(target=put_queue, args=(q, [i for i in range(n)]))
p.start()
p.join()
for i in range(n):
data = q.get()
assert data == i
def test_get_job(self):
p = fiber.Process(name="test_get_job")
popen = fspawn.Popen(p, launch=False)
job = popen._get_job(["echo", "hello fiber"])
assert job.command == ["echo", "hello fiber"]
#assert job.image == "fiber-test:latest"
# by default, do image should be provided. Backend should decide if
# to use default image or current container image.
assert job.image is None
def test_popen_method(self):
# make sure that fiber works with multiprocessing
p = fiber.Process(name="test_popen_method")
popen = p._Popen(p)
assert isinstance(popen, fiber.popen_fiber_spawn.Popen)
p = mp.Process(name="test_popen_method2")
popen = p._Popen(p)
assert popen.method == "fork"
assert isinstance(popen, mp.popen_fork.Popen)
def test_config_sync(self):
# child process should have the same config as parent process
r, w = fiber.Pipe()
p = fiber.Process(target=config_worker, args=(w,))
p.start()
got = r.recv()
expected = fiber.config.get_dict()
p.join()
assert got == expected
def test_interactive_shell_cloudpickle(self):
def add(a, b):
return a + b
# This will fail: "AttributeError: Can't pickle local object"
# default pickler can pickle local object
with pytest.raises(AttributeError):
p = fiber.Process(target=add, args=(1, 2),
name="test_interactive_shell_cloudpickle")
p.start()
# This will work. When using interactive console, pickler
# is set to cloudpickle
with mock.patch("fiber.util.is_in_interactive_console") as func:
func.return_value = True
p = fiber.Process(target=add, args=(1, 2),
name="test_interactive_shell_cloudpickle2")
p.start()
p.join()
def run_manager_standard():
manager = fiber.Manager()
ns = manager.Namespace()
ns.x = 1
ns.y = [1]
ns.z = [1]
ls = manager.list([1, [1], [1], manager.list([1])])
di = manager.dict({0: 1, 1: [1], 2: [1], 3: manager.list([1])})
print('before', ns, ls, ls[2], di, di[2], sep='\n')
p = fiber.Process(target=f, args=(ns, ls, di))
p.start()
p.join()
print('after', ns, ls, ls[2], di, di[2], sep='\n')
def test_image_not_found(self):
if fiber.config.default_backend != "docker":
pytest.skip("skipped because current backend is not docker")
try:
with pytest.raises(multiprocessing.ProcessError):
fiber.init(
image='this-image-does-not-exist-and-is-only-used-for-testing'
)
p = fiber.Process(name="test_image_not_found")
p.start()
finally:
fiber.reset()
def test_command_line(self):
fiber.init(use_bash=False)
try:
p = fiber.Process(name="test_command_line")
popen = fspawn.Popen(p, backend="docker", launch=False)
cmd = popen.get_command_line(host="127.0.0.1", port="8000", id=1)
finally:
fiber.init(use_bash=True)
assert cmd == [
"/usr/local/bin/python", "-c",
'import sys;import os;import socket;import struct;import fiber;import fiber.util;from multiprocessing import spawn, reduction;sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM);sock.connect(("127.0.0.1", 8000));conn = sock;conn.send(struct.pack("<I", 1));fd = conn.fileno();exitcode = fiber.util.spawn_prepare(fd);sys.exit(exitcode)',
'--multiprocessing-fork'
]
def test_get_listen_addr_fp(self):
ifce = get_ifce("docker0")
network = ipaddress.IPv4Network((ifce.address, ifce.netmask),
strict=False)
q = fiber.SimpleQueue()
p = fiber.Process(target=wrap_result,
args=(q, ),
name="test_get_listen_addr")
p.start()
res = q.get()
ip = res[0]
assert ipaddress.IPv4Address(ip) in network
p.join()
assert p.exitcode == 0
def test_init_image(self, mock_create_job):
if fiber.config.default_backend != "docker":
pytest.skip("skipped because current backend is not docker")
try:
fiber.init(image='python:3.6-alpine', backend='docker')
p = fiber.Process(name="test_init_image")
p.start()
except NotImplementedError:
pass
finally:
fiber.reset()
args = mock_create_job.call_args
# https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock.call_args # noqa E501
job_spec_arg = args[0][0]
assert job_spec_arg.image == "python:3.6-alpine"
def test_simple_queue_fiber2(self):
q = SimpleQueue()
n = 3
procs = []
try:
for i in range(n):
procs.append(fiber.Process(target=put_queue, args=(q, 10)))
for p in procs:
p.start()
for p in procs:
p.join()
for p in procs:
data = q.get()
assert data == 10
finally:
for p in procs:
p.join()
def test_no_python3_inside_image(self):
if fiber.config.default_backend != "docker":
pytest.skip("skipped because current backend is not docker")
'''
fp = io.StringIO()
handler = logging.StreamHandler(fp)
logger = logging.getLogger("fiber")
logger.setLevel(level=logging.DEBUG)
logger.addHandler(handler)
'''
with pytest.raises(multiprocessing.ProcessError):
try:
fiber.init(image='ubuntu:18.04')
p = fiber.Process(name="test_no_python3_inside_image")
p.start()
finally:
fiber.reset()
#logger.removeHandler(handler)
'''
def test_fiber_with_more_than_1024_fds(self):
# select.select can't handle fd >= 1024
# https://stackoverflow.com/questions/7328165/how-do-you-compile-python-with-1024-file-descriptors
import resource
resource.setrlimit(resource.RLIMIT_OFILE, (8192, 8192))
socks = []
for i in range(1023):
s = socket.socket(socket.AF_INET)
socks.append(s)
p = fiber.Process(target=time.sleep,
args=(10, ),
name="test_fiber_with_more_than_1024_fds")
p.start()
# if select is used, we should see errors like:
# `ValueError: filedescriptor out of range in select()`
print(p.exitcode)
p.terminate()
def test_log_file_path(self):
if fiber.config.default_backend != "docker":
pytest.skip("skipped because current backend is not docker")
# make sure workers and master doesn't write to the same log file
try:
log_file = fiber_config.log_file
log_level = fiber_config.log_level
# clean up
if os.path.isdir("tests/logs"):
files = glob.glob("tests/logs/*")
for f in files:
os.remove(f)
files = glob.glob("tests/logs/*")
assert files == []
fiber.init(log_file="tests/logs/fiber.log", log_level="DEBUG")
q = fiber.SimpleQueue()
p = fiber.Process(target=dummy_q_worker, args=(q,))
p.start()
msg = q.get()
assert msg == "DUMMY_WORKER_DONE"
files = glob.glob("tests/logs/*")
files.sort()
assert len(files) == 2
assert files[0] == "tests/logs/fiber.log.MainProcess"
assert "tests/logs/fiber.log.Process-" in files[1]
regex = r'\w+:Process-\d+\(\d+\):'
with open("tests/logs/fiber.log.MainProcess") as f:
content = f.read()
assert re.search(regex, content) is None, (
"Fiber subprocess log found in master log file")
p.join()
finally:
fiber.init(log_file=log_file, log_level=log_level)
def test_no_python3_inside_image(self):
if fiber.config.default_backend != "docker":
pytest.skip("skipped because current backend is not docker")
fp = io.StringIO()
handler = logging.StreamHandler(fp)
logger = logging.getLogger("fiber")
logger.setLevel(level=logging.DEBUG)
logger.addHandler(handler)
try:
fiber.init(image='ubuntu:18.04')
p = fiber.Process(name="test_no_python3_inside_image")
p.start()
logs = fp.getvalue()
times = len(list(re.finditer('Failed to start Fiber process',
logs)))
assert times == 1
finally:
fiber.reset()
logger.removeHandler(handler)
