def test_lock_and_till(self): locker = Lock("prime lock") got_signal = Signal() a_is_ready = Signal("a lock") b_is_ready = Signal("b lock") Log.note("begin") def loop(is_ready, please_stop): with locker: while not got_signal: locker.wait(till=Till(seconds=0.01)) is_ready.go() Log.note("{{thread}} is ready", thread=Thread.current().name) Log.note("outside loop") locker.wait() Log.note("thread is expected to get here") thread_a = Thread.run("a", loop, a_is_ready).release() thread_b = Thread.run("b", loop, b_is_ready).release() a_is_ready.wait() b_is_ready.wait() timeout = Till(seconds=1) with locker: got_signal.go() while not thread_a.stopped: # WE MUST CONTINUE TO USE THE locker TO ENSURE THE OTHER THREADS ARE NOT ORPHANED IN THERE locker.wait(till=Till(seconds=0.1)) Log.note("wait for a thread") while not thread_b.stopped: # WE MUST CONTINUE TO USE THE locker TO ENSURE THE OTHER THREADS ARE NOT ORPHANED IN THERE locker.wait(till=Till(seconds=0.1)) Log.note("wait for b thread") thread_a.join() thread_b.join() if timeout: Log.error("Took too long") self.assertTrue(bool(thread_a.stopped), "Thread should be done by now") self.assertTrue(bool(thread_b.stopped), "Thread should be done by now")
class PersistentQueue(object): """ THREAD-SAFE, PERSISTENT QUEUE CAN HANDLE MANY PRODUCERS, BUT THE pop(), commit() IDIOM CAN HANDLE ONLY ONE CONSUMER. IT IS IMPORTANT YOU commit() or close(), OTHERWISE NOTHING COMES OFF THE QUEUE """ def __init__(self, _file): """ file - USES FILE FOR PERSISTENCE """ self.file = File.new_instance(_file) self.lock = Lock("lock for persistent queue using file " + self.file.name) self.please_stop = Signal() self.db = Data() self.pending = [] if self.file.exists: for line in self.file: with suppress_exception: delta = mo_json.json2value(line) apply_delta(self.db, delta) if self.db.status.start == None: # HAPPENS WHEN ONLY ADDED TO QUEUE, THEN CRASH self.db.status.start = 0 self.start = self.db.status.start # SCRUB LOST VALUES lost = 0 for k in self.db.keys(): with suppress_exception: if k!="status" and int(k) < self.start: self.db[k] = None lost += 1 # HAPPENS FOR self.db.status, BUT MAYBE OTHER PROPERTIES TOO if lost: Log.warning("queue file had {{num}} items lost", num= lost) DEBUG and Log.note("Persistent queue {{name}} found with {{num}} items", name=self.file.abspath, num=len(self)) else: self.db.status = Data( start=0, end=0 ) self.start = self.db.status.start DEBUG and Log.note("New persistent queue {{name}}", name=self.file.abspath) def _add_pending(self, delta): delta = wrap(delta) self.pending.append(delta) def _apply_pending(self): for delta in self.pending: apply_delta(self.db, delta) self.pending = [] def __iter__(self): """ BLOCKING ITERATOR """ while not self.please_stop: try: value = self.pop() if value is not THREAD_STOP: yield value except Exception as e: Log.warning("Tell me about what happened here", cause=e) def add(self, value): with self.lock: if self.closed: Log.error("Queue is closed") if value is THREAD_STOP: DEBUG and Log.note("Stop is seen in persistent queue") self.please_stop.go() return self._add_pending({"add": {str(self.db.status.end): value}}) self.db.status.end += 1 self._add_pending({"add": {"status.end": self.db.status.end}}) self._commit() return self def __len__(self): with self.lock: return self.db.status.end - self.start def __getitem__(self, item): return self.db[str(item + self.start)] def pop(self, timeout=None): """ :param timeout: OPTIONAL DURATION :return: None, IF timeout PASSES """ with self.lock: while not self.please_stop: if self.db.status.end > self.start: value = self.db[str(self.start)] self.start += 1 return value if timeout is not None: with suppress_exception: self.lock.wait(timeout=timeout) if self.db.status.end <= self.start: return None else: self.lock.wait() DEBUG and Log.note("persistent queue already stopped") return THREAD_STOP def pop_all(self): """ NON-BLOCKING POP ALL IN QUEUE, IF ANY """ with self.lock: if self.please_stop: return [THREAD_STOP] if self.db.status.end == self.start: return [] output = [] for i in range(self.start, self.db.status.end): output.append(self.db[str(i)]) self.start = self.db.status.end return output def rollback(self): with self.lock: if self.closed: return self.start = self.db.status.start self.pending = [] def commit(self): with self.lock: if self.closed: Log.error("Queue is closed, commit not allowed") try: self._add_pending({"add": {"status.start": self.start}}) for i in range(self.db.status.start, self.start): self._add_pending({"remove": str(i)}) if self.db.status.end - self.start < 10 or Random.range(0, 1000) == 0: # FORCE RE-WRITE TO LIMIT FILE SIZE # SIMPLY RE-WRITE FILE if DEBUG: Log.note("Re-write {{num_keys}} keys to persistent queue", num_keys=self.db.status.end - self.start) for k in self.db.keys(): if k == "status" or int(k) >= self.db.status.start: continue Log.error("Not expecting {{key}}", key=k) self._commit() self.file.write(mo_json.value2json({"add": self.db}) + "\n") else: self._commit() except Exception as e: raise e def _commit(self): self.file.append("\n".join(mo_json.value2json(p) for p in self.pending)) self._apply_pending() def close(self): self.please_stop.go() with self.lock: if self.db is None: return self.add(THREAD_STOP) if self.db.status.end == self.start: DEBUG and Log.note("persistent queue clear and closed") self.file.delete() else: DEBUG and Log.note("persistent queue closed with {{num}} items left", num=len(self)) try: self._add_pending({"add": {"status.start": self.start}}) for i in range(self.db.status.start, self.start): self._add_pending({"remove": str(i)}) self.file.write(mo_json.value2json({"add": self.db}) + "\n" + ("\n".join(mo_json.value2json(p) for p in self.pending)) + "\n") self._apply_pending() except Exception as e: raise e self.db = None @property def closed(self): with self.lock: return self.db is None
class PersistentQueue(object): """ THREAD-SAFE, PERSISTENT QUEUE CAN HANDLE MANY PRODUCERS, BUT THE pop(), commit() IDIOM CAN HANDLE ONLY ONE CONSUMER. IT IS IMPORTANT YOU commit() or close(), OTHERWISE NOTHING COMES OFF THE QUEUE """ def __init__(self, _file): """ file - USES FILE FOR PERSISTENCE """ self.file = File.new_instance(_file) self.lock = Lock("lock for persistent queue using file " + self.file.name) self.please_stop = Signal() self.db = Data() self.pending = [] if self.file.exists: for line in self.file: with suppress_exception: delta = json2value(line) apply_delta(self.db, delta) if self.db.status.start == None: # HAPPENS WHEN ONLY ADDED TO QUEUE, THEN CRASH self.db.status.start = 0 self.start = self.db.status.start # SCRUB LOST VALUES lost = 0 for k in self.db.keys(): with suppress_exception: if k != "status" and int(k) < self.start: self.db[k] = None lost += 1 # HAPPENS FOR self.db.status, BUT MAYBE OTHER PROPERTIES TOO if lost: Log.warning("queue file had {{num}} items lost", num=lost) DEBUG and Log.note( "Persistent queue {{name}} found with {{num}} items", name=self.file.abspath, num=len(self)) else: self.db.status = Data(start=0, end=0) self.start = self.db.status.start DEBUG and Log.note("New persistent queue {{name}}", name=self.file.abspath) def _add_pending(self, delta): delta = to_data(delta) self.pending.append(delta) def _apply_pending(self): for delta in self.pending: apply_delta(self.db, delta) self.pending = [] def __iter__(self): """ BLOCKING ITERATOR """ while not self.please_stop: try: value = self.pop() if value is not THREAD_STOP: yield value except Exception as e: Log.warning("Tell me about what happened here", cause=e) def add(self, value): with self.lock: if self.closed: Log.error("Queue is closed") if value is THREAD_STOP: DEBUG and Log.note("Stop is seen in persistent queue") self.please_stop.go() return self._add_pending({"add": {str(self.db.status.end): value}}) self.db.status.end += 1 self._add_pending({"add": {"status.end": self.db.status.end}}) self._commit() return self def __len__(self): with self.lock: return self.db.status.end - self.start def __getitem__(self, item): return self.db[str(item + self.start)] def pop(self, timeout=None): """ :param timeout: OPTIONAL DURATION :return: None, IF timeout PASSES """ with self.lock: while not self.please_stop: if self.db.status.end > self.start: value = self.db[str(self.start)] self.start += 1 return value if timeout is not None: with suppress_exception: self.lock.wait(timeout=timeout) if self.db.status.end <= self.start: return None else: self.lock.wait() DEBUG and Log.note("persistent queue already stopped") return THREAD_STOP def pop_all(self): """ NON-BLOCKING POP ALL IN QUEUE, IF ANY """ with self.lock: if self.please_stop: return [THREAD_STOP] if self.db.status.end == self.start: return [] output = [] for i in range(self.start, self.db.status.end): output.append(self.db[str(i)]) self.start = self.db.status.end return output def rollback(self): with self.lock: if self.closed: return self.start = self.db.status.start self.pending = [] def commit(self): with self.lock: if self.closed: Log.error("Queue is closed, commit not allowed") try: self._add_pending({"add": {"status.start": self.start}}) for i in range(self.db.status.start, self.start): self._add_pending({"remove": str(i)}) if self.db.status.end - self.start < 10 or randoms.range( 0, 1000) == 0: # FORCE RE-WRITE TO LIMIT FILE SIZE # SIMPLY RE-WRITE FILE if DEBUG: Log.note( "Re-write {{num_keys}} keys to persistent queue", num_keys=self.db.status.end - self.start) for k in self.db.keys(): if k == "status" or int(k) >= self.db.status.start: continue Log.error("Not expecting {{key}}", key=k) self._commit() self.file.write(value2json({"add": self.db}) + "\n") else: self._commit() except Exception as e: raise e def _commit(self): self.file.append("\n".join(value2json(p) for p in self.pending)) self._apply_pending() def close(self): self.please_stop.go() with self.lock: if self.db is None: return self.add(THREAD_STOP) if self.db.status.end == self.start: DEBUG and Log.note("persistent queue clear and closed") self.file.delete() else: DEBUG and Log.note( "persistent queue closed with {{num}} items left", num=len(self)) try: self._add_pending({"add": {"status.start": self.start}}) for i in range(self.db.status.start, self.start): self._add_pending({"remove": str(i)}) self.file.write( value2json({"add": self.db}) + "\n" + ("\n".join(value2json(p) for p in self.pending)) + "\n") self._apply_pending() except Exception as e: raise e self.db = None @property def closed(self): with self.lock: return self.db is None
class Queue(object): """ SIMPLE MESSAGE QUEUE, multiprocessing.Queue REQUIRES SERIALIZATION, WHICH IS DIFFICULT TO USE JUST BETWEEN THREADS (SERIALIZATION REQUIRED) """ def __init__(self, name, max=None, silent=False, unique=False, allow_add_after_close=False): """ max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK silent - COMPLAIN IF THE READERS ARE TOO SLOW unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME """ if not _Log: _late_import() self.name = name self.max = coalesce(max, 2**10) self.silent = silent self.allow_add_after_close = allow_add_after_close self.unique = unique self.please_stop = Signal("stop signal for " + name) self.lock = Lock("lock for queue " + name) self.queue = deque() self.next_warning = time() # FOR DEBUGGING def __iter__(self): try: while True: value = self.pop(self.please_stop) if value is THREAD_STOP: break if value is not None: yield value except Exception as e: _Log.warning("Tell me about what happened here", e) if not self.silent: _Log.note("queue iterator is done") def add(self, value, timeout=None): with self.lock: if value is THREAD_STOP: # INSIDE THE lock SO THAT EXITING WILL RELEASE wait() self.queue.append(value) self.please_stop.go() return self._wait_for_queue_space(timeout=timeout) if self.please_stop and not self.allow_add_after_close: _Log.error("Do not add to closed queue") else: if self.unique: if value not in self.queue: self.queue.append(value) else: self.queue.append(value) return self def push(self, value): """ SNEAK value TO FRONT OF THE QUEUE """ if self.please_stop and not self.allow_add_after_close: _Log.error("Do not push to closed queue") with self.lock: self._wait_for_queue_space() if not self.please_stop: self.queue.appendleft(value) return self def pop_message(self, till=None): """ RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE DUMMY IMPLEMENTATION FOR DEBUGGING """ if till is not None and not isinstance(till, Signal): _Log.error("Expecting a signal") return Null, self.pop(till=till) def extend(self, values): if self.please_stop and not self.allow_add_after_close: _Log.error("Do not push to closed queue") with self.lock: # ONCE THE queue IS BELOW LIMIT, ALLOW ADDING MORE self._wait_for_queue_space() if not self.please_stop: if self.unique: for v in values: if v is THREAD_STOP: self.please_stop.go() continue if v not in self.queue: self.queue.append(v) else: for v in values: if v is THREAD_STOP: self.please_stop.go() continue self.queue.append(v) return self def _wait_for_queue_space(self, timeout=DEFAULT_WAIT_TIME): """ EXPECT THE self.lock TO BE HAD, WAITS FOR self.queue TO HAVE A LITTLE SPACE """ wait_time = 5 if DEBUG and len(self.queue) > 1 * 1000 * 1000: Log.warning("Queue {{name}} has over a million items") now = time() if timeout != None: time_to_stop_waiting = now + timeout else: time_to_stop_waiting = Null if self.next_warning < now: self.next_warning = now + wait_time while not self.please_stop and len(self.queue) >= self.max: if now > time_to_stop_waiting: if not _Log: _late_import() _Log.error(THREAD_TIMEOUT) if self.silent: self.lock.wait(Till(till=time_to_stop_waiting)) else: self.lock.wait(Till(timeout=wait_time)) if len(self.queue) >= self.max: now = time() if self.next_warning < now: self.next_warning = now + wait_time _Log.alert( "Queue by name of {{name|quote}} is full with ({{num}} items), thread(s) have been waiting {{wait_time}} sec", name=self.name, num=len(self.queue), wait_time=wait_time) def __len__(self): with self.lock: return len(self.queue) def __nonzero__(self): with self.lock: return any(r != THREAD_STOP for r in self.queue) def pop(self, till=None): """ WAIT FOR NEXT ITEM ON THE QUEUE RETURN THREAD_STOP IF QUEUE IS CLOSED RETURN None IF till IS REACHED AND QUEUE IS STILL EMPTY :param till: A `Signal` to stop waiting and return None :return: A value, or a THREAD_STOP or None """ if till is not None and not isinstance(till, Signal): _Log.error("expecting a signal") with self.lock: while True: if self.queue: value = self.queue.popleft() return value if self.please_stop: break if not self.lock.wait(till=till | self.please_stop): if self.please_stop: break return None if DEBUG or not self.silent: _Log.note(self.name + " queue stopped") return THREAD_STOP def pop_all(self): """ NON-BLOCKING POP ALL IN QUEUE, IF ANY """ with self.lock: output = list(self.queue) self.queue.clear() return output def pop_one(self): """ NON-BLOCKING POP IN QUEUE, IF ANY """ with self.lock: if self.please_stop: return [THREAD_STOP] elif not self.queue: return None else: v = self.queue.pop() if v is THREAD_STOP: # SENDING A STOP INTO THE QUEUE IS ALSO AN OPTION self.please_stop.go() return v def close(self): with self.lock: self.please_stop.go() def commit(self): pass def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close()