def open_writer_queue(self,
uid: str,
*,
force: bool = False) -> t.Iterator[Q[T]]:
try:
with namedpipe.create_writer_port(uid, force=force) as wf:
try:
sem = pathlib.Path(uid).with_suffix(".lock")
sem.touch()
q: Q[T] = Q(wf,
format_protocol=PickleMessageFormat(),
adapter=_QueueAdapter)
yield q
finally:
# INFO: this is work-around code, when opened by multiple readers
time.sleep(_MINI_WAIT_TIME)
with open(sem) as rf:
for _ in rf:
q.put(None) # type: ignore
# logger.debug("... SEND sem")
time.sleep(_MINI_WAIT_TIME)
# logger.debug("..., DELETE sem")
sem.unlink()
except BrokenPipeError as e:
logger.info("broken type: %s", e)
except Exception as e:
if self.config.sensitive:
raise
logger.warning("error occured: %s", e, exc_info=True)
def run():
q = Q(queue.Queue(), format_protcol=PickleFormat())
for i in range(5):
q.put(i)
q.put(None)
for i in consume(q):
print(i)
def open_writer_queue(self,
uid: str,
*,
force: bool = False) -> t.Iterator[Q[T]]:
from minitask.transport import fake
q = fake.create_writer_port(uid).q
yield Q(q)
for _ in range(self.reader_count):
q.put(None)
def run():
endpoint = str((pathlib.Path(__file__).parent / "x.fifo").absolute())
ex = Executor()
ex.spawn(worker, endpoint=endpoint)
with open_port(endpoint, "w") as wf:
q = Q(QueueLike(wf), format_protocol=PickleFormat())
for i in range(5):
q.put(i)
time.sleep(0.05)
q.put(None)
ex.wait()
print("ok")
def run():
q = Q(queue.Queue())
for i in range(5):
q.put(i)
q.put(None)
while True:
m, task_done = q.get()
if m is None:
task_done()
break
print(m)
task_done()
def run():
q = Q(queue.Queue(), format_protcol=PickleFormat())
for i in range(5):
q.put({"message": f"hello {i}"})
q.put(None)
while True:
m, task_done = q.get()
if m is None:
task_done()
break
print(m)
task_done()
def run():
q = Q(queue.Queue(), format_protcol=PickleFormat())
for i in range(5):
q.put(i)
q.put(None)
# ここでrefにする必要がある
def worker():
for i in consume(q):
print(i)
th = threading.Thread(target=worker)
th.start()
q.join()
print("ok")
def run():
import sys
import subprocess
endpoint = "x.fifo"
subprocess.Popen(
[sys.executable, __file__, "worker", "--endpoint", endpoint])
with open_port(endpoint, "w") as wf:
q = Q(QueueLike(wf), format_protcol=PickleFormat())
for i in range(5):
q.put(i)
time.sleep(0.05)
q.put(None)
print("ok")
def open_reader_queue(self, uid: str) -> t.Iterator[Q[T]]:
try:
with namedpipe.create_reader_port(uid) as rf:
threading.Thread(target=partial(self._handshake, uid),
daemon=True).start()
yield Q(rf,
format_protocol=PickleMessageFormat(),
adapter=_QueueAdapter)
# logger.debug("... END")
except BrokenPipeError as e:
logger.info("broken type: %s", e)
except Exception as e:
if self.config.sensitive:
raise
logger.warning("error occured: %s", e, exc_info=True)
def run():
q = Q(queue.Queue(), format_protcol=PickleFormat())
ex = Executor()
def producer():
for i in range(5):
q.put(i)
time.sleep(0.05)
q.put(None)
def worker():
for i in consume(q):
print(i)
ex.spawn(producer)
ex.spawn(worker)
ex.wait()
print("ok")
def run():
q = Q(queue.Queue(), format_protcol=PickleFormat())
ev = threading.Event()
def producer():
for i in range(5):
q.put(i)
time.sleep(0.05)
q.put(None)
ev.set()
def worker():
for i in consume(q):
print(i)
threading.Thread(target=producer).start()
threading.Thread(target=worker).start()
ev.wait()
q.join()
print("ok")
def open_writer_queue(self,
uid: str,
*,
force: bool = False) -> t.Iterator[Q[T]]:
internal = SQSQueue(uid, config=self.config)
yield Q(internal, adapter=None, format_protocol=JSONMessageFormat())
import typing as t
import queue
import random
import math
from minitask.q import Q, Message
random.seed(0)
class PriorityQueueFormat:
def encode(self, m: Message[t.Any]):
assert "priority" in m.metadata # TODO: type checking
return (m.metadata["priority"], m.body)
def decode(self, b: t.Tuple[t.Any, t.Any]) -> Message[t.Any]:
priority, body = b
return Message(body=body, metadata={"priority": priority})
q = Q(queue.PriorityQueue(), format_protocol=PriorityQueueFormat())
q.put(None, priority=math.inf)
for i in range(5):
print("<-", i)
q.put(i, priority=random.random())
for item in q:
print("->", item)
import queue
from minitask.q import Q
q = Q(queue.LifoQueue())
q.put(None)
for i in range(5):
print("<-", i)
q.put(i)
for item in q:
print("->", item)
def open_reader_queue(self, uid: str) -> t.Iterator[Q[T]]:
from minitask.transport import fake
self.reader_count += 1
q = fake.create_reader_port(uid).q
yield Q(q)
import queue
from minitask.q import Q
q = Q(queue.Queue())
for i in range(5):
print("<-", i)
q.put(i)
q.put(None)
for item in q:
print("->", item)
def worker(*, endpoint: str):
with open_port(endpoint, "r") as rf:
q = Q(QueueLike(rf), format_protcol=PickleFormat())
for item in consume(q):
print(item)
print("end")
def on_message(ev: events.MessageEvent[t.Any], q: Q) -> None:
print(os.getpid(), "!")
q.put(ev.content)
def open_reader_queue(self, uid: str):
from minitask.q import Q
internal = SQSQueue(self.config)
yield Q(internal, adapter=None)
