def run():
with executor:
input_endpoint = executor.create_endpoint()
output_endpoint = executor.create_endpoint()
executor.spawn(worker,
input_endpoint=input_endpoint,
output_endpoint=output_endpoint)
ipc = minitask.IPC(communication=namedpipe)
def notify():
with ipc.connect(output_endpoint) as x:
while True:
res = x.recv()
if res is None:
break
print("*********************")
print(res.serialize())
print("*********************")
threading.Thread(target=notify).start()
with ipc.serve(input_endpoint) as x:
x.send(x.serialization.create_message("add", args=[1, 2]))
x.send(x.serialization.create_message("add", args=[1, 2]))
x.send(x.serialization.create_message("add", args=[1, 2]))
x.send(x.serialization.create_message("add", args=[1, 2]))
x.send(x.serialization.create_message("add", args=[1, 2]))
x.send(x.serialization.create_message("add", args=[1, 2]))
x.send(x.serialization.create_message("add", args=[1, 2]))
x.send(x.serialization.create_message("add", args=[1, 2]))
def writer(*, endpoint: str):
ipc = minitask.IPC(port=namedpipe)
pid = os.getpid()
with ipc.serve(endpoint) as x:
for i in range(5):
print(pid, "send", i)
x.send(x.serialization.create_message("add", args=[i, i]))
def writer(*, endpoint: str):
ipc = minitask.IPC(port=namedpipe)
pid = os.getpid()
with ipc.serve(endpoint) as x:
for i in range(5):
print(pid, "send", i)
x.send("hello", kwargs=dict(i=i)) # write?
def run(*, keep: bool = False):
import pathlib
import tempfile
ipc = minitask.IPC()
with tempfile.TemporaryDirectory() as dirpath:
n = 1
pairs = []
for uid in range(n):
endpoint = pathlib.Path(dirpath) / f"worker.{uid}.fifo"
ipc.create_endpoint(endpoint)
sp = subprocess.Popen(
[sys.executable, __file__, "producer", "--endpoint", endpoint]
)
cp = subprocess.Popen(
[sys.executable, __file__, "consumer", "--endpoint", endpoint]
)
pairs.append((sp, cp))
for sp, cp in pairs:
sp.wait()
cp.terminate()
def reader(*, endpoint: str):
ipc = minitask.IPC(port=namedpipe)
pid = os.getpid()
with ipc.connect(endpoint) as x:
for req in x:
print(pid, "got", req.serialize())
res = dispatcher.dispatch(req)
print(pid, "res", res.serialize())
def consumer(*, endpoint: str):
ipc = minitask.IPC()
with ipc.connect(endpoint) as x:
while True:
msg = x.recv()
if msg is None:
break
print("got", msg.unique_id, msg.method, msg.args, msg.kwargs)
def worker(*, endpoint: str):
ipc = minitask.IPC(communication=namedpipe)
for item in ipc.connect(endpoint):
print("<-", item)
import time
time.sleep(0.1)
print("->", sum(item["args"]))
def producer(*, endpoint: str):
import os
pid = os.getpid()
ipc = minitask.IPC()
with ipc.serve(endpoint) as x:
for i in range(5):
x.send("say", kwargs=dict(message="hello", sender=[pid]))
time.sleep(0.1)
def run():
with executor:
endpoint = executor.create_endpoint()
executor.spawn(worker, endpoint=endpoint)
ipc = minitask.IPC(communication=namedpipe)
with ipc.serve(endpoint) as x:
x.send({"method": "add", "args": [1, 2]})
x.send({"method": "add", "args": [1, 2]})
x.send({"method": "add", "args": [1, 2]})
def worker(*, input_endpoint: str, output_endpoint: str):
ipc = minitask.IPC(communication=namedpipe)
with ipc.serve(output_endpoint) as x:
for req in ipc.connect(input_endpoint):
print("req", req.serialize())
import time
time.sleep(0.5)
res = dispatcher.dispatch(req)
print("res", res.serialize())
x.send(res)
def transformer(*, input_endpoint: str, output_endpoint):
import os
pid = os.getpid()
ipc = minitask.IPC()
with ipc.connect(input_endpoint) as c:
with ipc.serve(output_endpoint) as s:
while True:
req = c.recv()
if req is None:
break
req.kwargs["message"] = f"@@ {req.kwargs['message']}@@"
req.kwargs["sender"].append(pid)
s.send("say", kwargs=req.kwargs)
def worker(*, input_endpoint: str, output_endpoint: str):
ipc = minitask.IPC(communication=namedpipe)
wq = WorkerQueue(ipc, input_endpoint=input_endpoint)
with ipc.serve(output_endpoint) as x:
for req in wq:
print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
print("req", req.serialize())
print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
import time
time.sleep(0.1)
res = dispatcher.dispatch(req)
print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
print("res", res.serialize())
print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
x.send(res)
def run(*, keep: bool = False, n: int = 1):
import pathlib
import tempfile
ipc = minitask.IPC()
with tempfile.TemporaryDirectory() as dirpath:
pairs = []
for uid in range(n):
endpoint = ipc.create_endpoint(
pathlib.Path(dirpath) / f"worker.{uid}.fifo")
endpoint2 = ipc.create_endpoint(
pathlib.Path(dirpath) / f"worker.{uid}.fifo2")
sp = subprocess.Popen([
sys.executable,
__file__,
"producer",
"--endpoint",
endpoint,
])
tp = subprocess.Popen([
sys.executable,
__file__,
"transformer",
"--input-endpoint",
endpoint,
"--output-endpoint",
endpoint2,
])
cp = subprocess.Popen([
sys.executable,
__file__,
"consumer",
"--endpoint",
endpoint2,
])
pairs.append((sp, cp))
for sp, cp in pairs:
sp.wait()
tp.wait()
cp.terminate()
def worker(*, input_endpoint: str, output_endpoint: str):
ipc = minitask.IPC(communication=namedpipe)
pid = os.getpid()
q = queue.Queue()
def peek():
with ipc.connect(input_endpoint) as x:
for req in x:
print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
print("PUT", req.serialize())
print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
q.put(req)
q.put(None)
th = threading.Thread(target=peek)
th.start()
with ipc.serve(output_endpoint) as x:
while True:
req = q.get()
if req is None:
q.task_done()
break
print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
print("req", req.serialize())
print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
import time
time.sleep(0.1)
res = dispatcher.dispatch(req)
print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
print("res", res.serialize())
print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
x.send(res)
q.task_done()
q.join()
from minitask.communication import fake
import minitask
ipc = minitask.IPC(communication=fake)
with ipc.serve("xxx") as x:
x.send("hello")
x.send("hello")
x.send("hello")
with ipc.connect("xxx") as x:
for i, item in enumerate(x):
if i == 2:
break
print(i, item)
print("end")
import time
import os
import minitask
from minitask.executor.namedpipe import Executor
from handofcats import as_subcommand
executor = Executor()
ipc = minitask.IPC()
@executor.register
def watch(*, endpoint: str):
pid = os.getpid()
with ipc.connect(endpoint) as x:
for line in x:
print("hoi", line, pid)
time.sleep(0.5)
@as_subcommand
def run():
with executor:
endpoint = executor.create_endpoint(uid="xxx")
executor.spawn(watch, endpoint=endpoint)
with ipc.serve(endpoint) as x:
for i in range(10):
x.send(i)
as_subcommand.run()
def reader(*, endpoint: str):
ipc = minitask.IPC(port=namedpipe)
pid = os.getpid()
with ipc.connect(endpoint) as x:
for msg in x:
print(pid, "got", msg.serialize())
