def open_writer_queue(self, uid: str, *, force: bool = False) -> t.Iterator[Q[T]]: try: with namedpipe.create_writer_port(uid, force=force) as wf: try: sem = pathlib.Path(uid).with_suffix(".lock") sem.touch() q: Q[T] = Q(wf, format_protocol=PickleMessageFormat(), adapter=_QueueAdapter) yield q finally: # INFO: this is work-around code, when opened by multiple readers time.sleep(_MINI_WAIT_TIME) with open(sem) as rf: for _ in rf: q.put(None) # type: ignore # logger.debug("... SEND sem") time.sleep(_MINI_WAIT_TIME) # logger.debug("..., DELETE sem") sem.unlink() except BrokenPipeError as e: logger.info("broken type: %s", e) except Exception as e: if self.config.sensitive: raise logger.warning("error occured: %s", e, exc_info=True)
def run(): q = Q(queue.Queue(), format_protcol=PickleFormat()) for i in range(5): q.put(i) q.put(None) for i in consume(q): print(i)
def open_writer_queue(self, uid: str, *, force: bool = False) -> t.Iterator[Q[T]]: from minitask.transport import fake q = fake.create_writer_port(uid).q yield Q(q) for _ in range(self.reader_count): q.put(None)
def run(): endpoint = str((pathlib.Path(__file__).parent / "x.fifo").absolute()) ex = Executor() ex.spawn(worker, endpoint=endpoint) with open_port(endpoint, "w") as wf: q = Q(QueueLike(wf), format_protocol=PickleFormat()) for i in range(5): q.put(i) time.sleep(0.05) q.put(None) ex.wait() print("ok")
def run(): q = Q(queue.Queue()) for i in range(5): q.put(i) q.put(None) while True: m, task_done = q.get() if m is None: task_done() break print(m) task_done()
def run(): q = Q(queue.Queue(), format_protcol=PickleFormat()) for i in range(5): q.put({"message": f"hello {i}"}) q.put(None) while True: m, task_done = q.get() if m is None: task_done() break print(m) task_done()
def run(): q = Q(queue.Queue(), format_protcol=PickleFormat()) for i in range(5): q.put(i) q.put(None) # ここでrefにする必要がある def worker(): for i in consume(q): print(i) th = threading.Thread(target=worker) th.start() q.join() print("ok")
def run(): import sys import subprocess endpoint = "x.fifo" subprocess.Popen( [sys.executable, __file__, "worker", "--endpoint", endpoint]) with open_port(endpoint, "w") as wf: q = Q(QueueLike(wf), format_protcol=PickleFormat()) for i in range(5): q.put(i) time.sleep(0.05) q.put(None) print("ok")
def open_reader_queue(self, uid: str) -> t.Iterator[Q[T]]: try: with namedpipe.create_reader_port(uid) as rf: threading.Thread(target=partial(self._handshake, uid), daemon=True).start() yield Q(rf, format_protocol=PickleMessageFormat(), adapter=_QueueAdapter) # logger.debug("... END") except BrokenPipeError as e: logger.info("broken type: %s", e) except Exception as e: if self.config.sensitive: raise logger.warning("error occured: %s", e, exc_info=True)
def run(): q = Q(queue.Queue(), format_protcol=PickleFormat()) ex = Executor() def producer(): for i in range(5): q.put(i) time.sleep(0.05) q.put(None) def worker(): for i in consume(q): print(i) ex.spawn(producer) ex.spawn(worker) ex.wait() print("ok")
def run(): q = Q(queue.Queue(), format_protcol=PickleFormat()) ev = threading.Event() def producer(): for i in range(5): q.put(i) time.sleep(0.05) q.put(None) ev.set() def worker(): for i in consume(q): print(i) threading.Thread(target=producer).start() threading.Thread(target=worker).start() ev.wait() q.join() print("ok")
def open_writer_queue(self, uid: str, *, force: bool = False) -> t.Iterator[Q[T]]: internal = SQSQueue(uid, config=self.config) yield Q(internal, adapter=None, format_protocol=JSONMessageFormat())
import typing as t import queue import random import math from minitask.q import Q, Message random.seed(0) class PriorityQueueFormat: def encode(self, m: Message[t.Any]): assert "priority" in m.metadata # TODO: type checking return (m.metadata["priority"], m.body) def decode(self, b: t.Tuple[t.Any, t.Any]) -> Message[t.Any]: priority, body = b return Message(body=body, metadata={"priority": priority}) q = Q(queue.PriorityQueue(), format_protocol=PriorityQueueFormat()) q.put(None, priority=math.inf) for i in range(5): print("<-", i) q.put(i, priority=random.random()) for item in q: print("->", item)
import queue from minitask.q import Q q = Q(queue.LifoQueue()) q.put(None) for i in range(5): print("<-", i) q.put(i) for item in q: print("->", item)
def open_reader_queue(self, uid: str) -> t.Iterator[Q[T]]: from minitask.transport import fake self.reader_count += 1 q = fake.create_reader_port(uid).q yield Q(q)
import queue from minitask.q import Q q = Q(queue.Queue()) for i in range(5): print("<-", i) q.put(i) q.put(None) for item in q: print("->", item)
def worker(*, endpoint: str): with open_port(endpoint, "r") as rf: q = Q(QueueLike(rf), format_protcol=PickleFormat()) for item in consume(q): print(item) print("end")
def on_message(ev: events.MessageEvent[t.Any], q: Q) -> None: print(os.getpid(), "!") q.put(ev.content)
def open_reader_queue(self, uid: str): from minitask.q import Q internal = SQSQueue(self.config) yield Q(internal, adapter=None)