def run(): with executor: input_endpoint = executor.create_endpoint() output_endpoint = executor.create_endpoint() executor.spawn(worker, input_endpoint=input_endpoint, output_endpoint=output_endpoint) ipc = minitask.IPC(communication=namedpipe) def notify(): with ipc.connect(output_endpoint) as x: while True: res = x.recv() if res is None: break print("*********************") print(res.serialize()) print("*********************") threading.Thread(target=notify).start() with ipc.serve(input_endpoint) as x: x.send(x.serialization.create_message("add", args=[1, 2])) x.send(x.serialization.create_message("add", args=[1, 2])) x.send(x.serialization.create_message("add", args=[1, 2])) x.send(x.serialization.create_message("add", args=[1, 2])) x.send(x.serialization.create_message("add", args=[1, 2])) x.send(x.serialization.create_message("add", args=[1, 2])) x.send(x.serialization.create_message("add", args=[1, 2])) x.send(x.serialization.create_message("add", args=[1, 2]))
def writer(*, endpoint: str): ipc = minitask.IPC(port=namedpipe) pid = os.getpid() with ipc.serve(endpoint) as x: for i in range(5): print(pid, "send", i) x.send(x.serialization.create_message("add", args=[i, i]))
def writer(*, endpoint: str): ipc = minitask.IPC(port=namedpipe) pid = os.getpid() with ipc.serve(endpoint) as x: for i in range(5): print(pid, "send", i) x.send("hello", kwargs=dict(i=i)) # write?
def run(*, keep: bool = False): import pathlib import tempfile ipc = minitask.IPC() with tempfile.TemporaryDirectory() as dirpath: n = 1 pairs = [] for uid in range(n): endpoint = pathlib.Path(dirpath) / f"worker.{uid}.fifo" ipc.create_endpoint(endpoint) sp = subprocess.Popen( [sys.executable, __file__, "producer", "--endpoint", endpoint] ) cp = subprocess.Popen( [sys.executable, __file__, "consumer", "--endpoint", endpoint] ) pairs.append((sp, cp)) for sp, cp in pairs: sp.wait() cp.terminate()
def reader(*, endpoint: str): ipc = minitask.IPC(port=namedpipe) pid = os.getpid() with ipc.connect(endpoint) as x: for req in x: print(pid, "got", req.serialize()) res = dispatcher.dispatch(req) print(pid, "res", res.serialize())
def consumer(*, endpoint: str): ipc = minitask.IPC() with ipc.connect(endpoint) as x: while True: msg = x.recv() if msg is None: break print("got", msg.unique_id, msg.method, msg.args, msg.kwargs)
def worker(*, endpoint: str): ipc = minitask.IPC(communication=namedpipe) for item in ipc.connect(endpoint): print("<-", item) import time time.sleep(0.1) print("->", sum(item["args"]))
def producer(*, endpoint: str): import os pid = os.getpid() ipc = minitask.IPC() with ipc.serve(endpoint) as x: for i in range(5): x.send("say", kwargs=dict(message="hello", sender=[pid])) time.sleep(0.1)
def run(): with executor: endpoint = executor.create_endpoint() executor.spawn(worker, endpoint=endpoint) ipc = minitask.IPC(communication=namedpipe) with ipc.serve(endpoint) as x: x.send({"method": "add", "args": [1, 2]}) x.send({"method": "add", "args": [1, 2]}) x.send({"method": "add", "args": [1, 2]})
def worker(*, input_endpoint: str, output_endpoint: str): ipc = minitask.IPC(communication=namedpipe) with ipc.serve(output_endpoint) as x: for req in ipc.connect(input_endpoint): print("req", req.serialize()) import time time.sleep(0.5) res = dispatcher.dispatch(req) print("res", res.serialize()) x.send(res)
def transformer(*, input_endpoint: str, output_endpoint): import os pid = os.getpid() ipc = minitask.IPC() with ipc.connect(input_endpoint) as c: with ipc.serve(output_endpoint) as s: while True: req = c.recv() if req is None: break req.kwargs["message"] = f"@@ {req.kwargs['message']}@@" req.kwargs["sender"].append(pid) s.send("say", kwargs=req.kwargs)
def worker(*, input_endpoint: str, output_endpoint: str): ipc = minitask.IPC(communication=namedpipe) wq = WorkerQueue(ipc, input_endpoint=input_endpoint) with ipc.serve(output_endpoint) as x: for req in wq: print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%") print("req", req.serialize()) print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%") import time time.sleep(0.1) res = dispatcher.dispatch(req) print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%") print("res", res.serialize()) print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%") x.send(res)
def run(*, keep: bool = False, n: int = 1): import pathlib import tempfile ipc = minitask.IPC() with tempfile.TemporaryDirectory() as dirpath: pairs = [] for uid in range(n): endpoint = ipc.create_endpoint( pathlib.Path(dirpath) / f"worker.{uid}.fifo") endpoint2 = ipc.create_endpoint( pathlib.Path(dirpath) / f"worker.{uid}.fifo2") sp = subprocess.Popen([ sys.executable, __file__, "producer", "--endpoint", endpoint, ]) tp = subprocess.Popen([ sys.executable, __file__, "transformer", "--input-endpoint", endpoint, "--output-endpoint", endpoint2, ]) cp = subprocess.Popen([ sys.executable, __file__, "consumer", "--endpoint", endpoint2, ]) pairs.append((sp, cp)) for sp, cp in pairs: sp.wait() tp.wait() cp.terminate()
def worker(*, input_endpoint: str, output_endpoint: str): ipc = minitask.IPC(communication=namedpipe) pid = os.getpid() q = queue.Queue() def peek(): with ipc.connect(input_endpoint) as x: for req in x: print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@") print("PUT", req.serialize()) print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@") q.put(req) q.put(None) th = threading.Thread(target=peek) th.start() with ipc.serve(output_endpoint) as x: while True: req = q.get() if req is None: q.task_done() break print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%") print("req", req.serialize()) print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%") import time time.sleep(0.1) res = dispatcher.dispatch(req) print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%") print("res", res.serialize()) print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%") x.send(res) q.task_done() q.join()
from minitask.communication import fake import minitask ipc = minitask.IPC(communication=fake) with ipc.serve("xxx") as x: x.send("hello") x.send("hello") x.send("hello") with ipc.connect("xxx") as x: for i, item in enumerate(x): if i == 2: break print(i, item) print("end")
import time import os import minitask from minitask.executor.namedpipe import Executor from handofcats import as_subcommand executor = Executor() ipc = minitask.IPC() @executor.register def watch(*, endpoint: str): pid = os.getpid() with ipc.connect(endpoint) as x: for line in x: print("hoi", line, pid) time.sleep(0.5) @as_subcommand def run(): with executor: endpoint = executor.create_endpoint(uid="xxx") executor.spawn(watch, endpoint=endpoint) with ipc.serve(endpoint) as x: for i in range(10): x.send(i) as_subcommand.run()
def reader(*, endpoint: str): ipc = minitask.IPC(port=namedpipe) pid = os.getpid() with ipc.connect(endpoint) as x: for msg in x: print(pid, "got", msg.serialize())