def test_lock_and_till(self):
locker = Lock("prime lock")
got_signal = Signal()
a_is_ready = Signal("a lock")
b_is_ready = Signal("b lock")
Log.note("begin")
def loop(is_ready, please_stop):
with locker:
while not got_signal:
locker.wait(till=Till(seconds=0.01))
is_ready.go()
Log.note("{{thread}} is ready",
thread=Thread.current().name)
Log.note("outside loop")
locker.wait()
Log.note("thread is expected to get here")
thread_a = Thread.run("a", loop, a_is_ready).release()
thread_b = Thread.run("b", loop, b_is_ready).release()
a_is_ready.wait()
b_is_ready.wait()
timeout = Till(seconds=1)
with locker:
got_signal.go()
while not thread_a.stopped:
# WE MUST CONTINUE TO USE THE locker TO ENSURE THE OTHER THREADS ARE NOT ORPHANED IN THERE
locker.wait(till=Till(seconds=0.1))
Log.note("wait for a thread")
while not thread_b.stopped:
# WE MUST CONTINUE TO USE THE locker TO ENSURE THE OTHER THREADS ARE NOT ORPHANED IN THERE
locker.wait(till=Till(seconds=0.1))
Log.note("wait for b thread")
thread_a.join()
thread_b.join()
if timeout:
Log.error("Took too long")
self.assertTrue(bool(thread_a.stopped), "Thread should be done by now")
self.assertTrue(bool(thread_b.stopped), "Thread should be done by now")
class PersistentQueue(object):
"""
THREAD-SAFE, PERSISTENT QUEUE
CAN HANDLE MANY PRODUCERS, BUT THE pop(), commit() IDIOM CAN HANDLE ONLY
ONE CONSUMER.
IT IS IMPORTANT YOU commit() or close(), OTHERWISE NOTHING COMES OFF THE QUEUE
"""
def __init__(self, _file):
"""
file - USES FILE FOR PERSISTENCE
"""
self.file = File.new_instance(_file)
self.lock = Lock("lock for persistent queue using file " + self.file.name)
self.please_stop = Signal()
self.db = Data()
self.pending = []
if self.file.exists:
for line in self.file:
with suppress_exception:
delta = mo_json.json2value(line)
apply_delta(self.db, delta)
if self.db.status.start == None: # HAPPENS WHEN ONLY ADDED TO QUEUE, THEN CRASH
self.db.status.start = 0
self.start = self.db.status.start
# SCRUB LOST VALUES
lost = 0
for k in self.db.keys():
with suppress_exception:
if k!="status" and int(k) < self.start:
self.db[k] = None
lost += 1
# HAPPENS FOR self.db.status, BUT MAYBE OTHER PROPERTIES TOO
if lost:
Log.warning("queue file had {{num}} items lost", num= lost)
DEBUG and Log.note("Persistent queue {{name}} found with {{num}} items", name=self.file.abspath, num=len(self))
else:
self.db.status = Data(
start=0,
end=0
)
self.start = self.db.status.start
DEBUG and Log.note("New persistent queue {{name}}", name=self.file.abspath)
def _add_pending(self, delta):
delta = wrap(delta)
self.pending.append(delta)
def _apply_pending(self):
for delta in self.pending:
apply_delta(self.db, delta)
self.pending = []
def __iter__(self):
"""
BLOCKING ITERATOR
"""
while not self.please_stop:
try:
value = self.pop()
if value is not THREAD_STOP:
yield value
except Exception as e:
Log.warning("Tell me about what happened here", cause=e)
def add(self, value):
with self.lock:
if self.closed:
Log.error("Queue is closed")
if value is THREAD_STOP:
DEBUG and Log.note("Stop is seen in persistent queue")
self.please_stop.go()
return
self._add_pending({"add": {str(self.db.status.end): value}})
self.db.status.end += 1
self._add_pending({"add": {"status.end": self.db.status.end}})
self._commit()
return self
def __len__(self):
with self.lock:
return self.db.status.end - self.start
def __getitem__(self, item):
return self.db[str(item + self.start)]
def pop(self, timeout=None):
"""
:param timeout: OPTIONAL DURATION
:return: None, IF timeout PASSES
"""
with self.lock:
while not self.please_stop:
if self.db.status.end > self.start:
value = self.db[str(self.start)]
self.start += 1
return value
if timeout is not None:
with suppress_exception:
self.lock.wait(timeout=timeout)
if self.db.status.end <= self.start:
return None
else:
self.lock.wait()
DEBUG and Log.note("persistent queue already stopped")
return THREAD_STOP
def pop_all(self):
"""
NON-BLOCKING POP ALL IN QUEUE, IF ANY
"""
with self.lock:
if self.please_stop:
return [THREAD_STOP]
if self.db.status.end == self.start:
return []
output = []
for i in range(self.start, self.db.status.end):
output.append(self.db[str(i)])
self.start = self.db.status.end
return output
def rollback(self):
with self.lock:
if self.closed:
return
self.start = self.db.status.start
self.pending = []
def commit(self):
with self.lock:
if self.closed:
Log.error("Queue is closed, commit not allowed")
try:
self._add_pending({"add": {"status.start": self.start}})
for i in range(self.db.status.start, self.start):
self._add_pending({"remove": str(i)})
if self.db.status.end - self.start < 10 or Random.range(0, 1000) == 0: # FORCE RE-WRITE TO LIMIT FILE SIZE
# SIMPLY RE-WRITE FILE
if DEBUG:
Log.note("Re-write {{num_keys}} keys to persistent queue", num_keys=self.db.status.end - self.start)
for k in self.db.keys():
if k == "status" or int(k) >= self.db.status.start:
continue
Log.error("Not expecting {{key}}", key=k)
self._commit()
self.file.write(mo_json.value2json({"add": self.db}) + "\n")
else:
self._commit()
except Exception as e:
raise e
def _commit(self):
self.file.append("\n".join(mo_json.value2json(p) for p in self.pending))
self._apply_pending()
def close(self):
self.please_stop.go()
with self.lock:
if self.db is None:
return
self.add(THREAD_STOP)
if self.db.status.end == self.start:
DEBUG and Log.note("persistent queue clear and closed")
self.file.delete()
else:
DEBUG and Log.note("persistent queue closed with {{num}} items left", num=len(self))
try:
self._add_pending({"add": {"status.start": self.start}})
for i in range(self.db.status.start, self.start):
self._add_pending({"remove": str(i)})
self.file.write(mo_json.value2json({"add": self.db}) + "\n" + ("\n".join(mo_json.value2json(p) for p in self.pending)) + "\n")
self._apply_pending()
except Exception as e:
raise e
self.db = None
@property
def closed(self):
with self.lock:
return self.db is None
class PersistentQueue(object):
"""
THREAD-SAFE, PERSISTENT QUEUE
CAN HANDLE MANY PRODUCERS, BUT THE pop(), commit() IDIOM CAN HANDLE ONLY
ONE CONSUMER.
IT IS IMPORTANT YOU commit() or close(), OTHERWISE NOTHING COMES OFF THE QUEUE
"""
def __init__(self, _file):
"""
file - USES FILE FOR PERSISTENCE
"""
self.file = File.new_instance(_file)
self.lock = Lock("lock for persistent queue using file " +
self.file.name)
self.please_stop = Signal()
self.db = Data()
self.pending = []
if self.file.exists:
for line in self.file:
with suppress_exception:
delta = json2value(line)
apply_delta(self.db, delta)
if self.db.status.start == None: # HAPPENS WHEN ONLY ADDED TO QUEUE, THEN CRASH
self.db.status.start = 0
self.start = self.db.status.start
# SCRUB LOST VALUES
lost = 0
for k in self.db.keys():
with suppress_exception:
if k != "status" and int(k) < self.start:
self.db[k] = None
lost += 1
# HAPPENS FOR self.db.status, BUT MAYBE OTHER PROPERTIES TOO
if lost:
Log.warning("queue file had {{num}} items lost", num=lost)
DEBUG and Log.note(
"Persistent queue {{name}} found with {{num}} items",
name=self.file.abspath,
num=len(self))
else:
self.db.status = Data(start=0, end=0)
self.start = self.db.status.start
DEBUG and Log.note("New persistent queue {{name}}",
name=self.file.abspath)
def _add_pending(self, delta):
delta = to_data(delta)
self.pending.append(delta)
def _apply_pending(self):
for delta in self.pending:
apply_delta(self.db, delta)
self.pending = []
def __iter__(self):
"""
BLOCKING ITERATOR
"""
while not self.please_stop:
try:
value = self.pop()
if value is not THREAD_STOP:
yield value
except Exception as e:
Log.warning("Tell me about what happened here", cause=e)
def add(self, value):
with self.lock:
if self.closed:
Log.error("Queue is closed")
if value is THREAD_STOP:
DEBUG and Log.note("Stop is seen in persistent queue")
self.please_stop.go()
return
self._add_pending({"add": {str(self.db.status.end): value}})
self.db.status.end += 1
self._add_pending({"add": {"status.end": self.db.status.end}})
self._commit()
return self
def __len__(self):
with self.lock:
return self.db.status.end - self.start
def __getitem__(self, item):
return self.db[str(item + self.start)]
def pop(self, timeout=None):
"""
:param timeout: OPTIONAL DURATION
:return: None, IF timeout PASSES
"""
with self.lock:
while not self.please_stop:
if self.db.status.end > self.start:
value = self.db[str(self.start)]
self.start += 1
return value
if timeout is not None:
with suppress_exception:
self.lock.wait(timeout=timeout)
if self.db.status.end <= self.start:
return None
else:
self.lock.wait()
DEBUG and Log.note("persistent queue already stopped")
return THREAD_STOP
def pop_all(self):
"""
NON-BLOCKING POP ALL IN QUEUE, IF ANY
"""
with self.lock:
if self.please_stop:
return [THREAD_STOP]
if self.db.status.end == self.start:
return []
output = []
for i in range(self.start, self.db.status.end):
output.append(self.db[str(i)])
self.start = self.db.status.end
return output
def rollback(self):
with self.lock:
if self.closed:
return
self.start = self.db.status.start
self.pending = []
def commit(self):
with self.lock:
if self.closed:
Log.error("Queue is closed, commit not allowed")
try:
self._add_pending({"add": {"status.start": self.start}})
for i in range(self.db.status.start, self.start):
self._add_pending({"remove": str(i)})
if self.db.status.end - self.start < 10 or randoms.range(
0, 1000) == 0: # FORCE RE-WRITE TO LIMIT FILE SIZE
# SIMPLY RE-WRITE FILE
if DEBUG:
Log.note(
"Re-write {{num_keys}} keys to persistent queue",
num_keys=self.db.status.end - self.start)
for k in self.db.keys():
if k == "status" or int(k) >= self.db.status.start:
continue
Log.error("Not expecting {{key}}", key=k)
self._commit()
self.file.write(value2json({"add": self.db}) + "\n")
else:
self._commit()
except Exception as e:
raise e
def _commit(self):
self.file.append("\n".join(value2json(p) for p in self.pending))
self._apply_pending()
def close(self):
self.please_stop.go()
with self.lock:
if self.db is None:
return
self.add(THREAD_STOP)
if self.db.status.end == self.start:
DEBUG and Log.note("persistent queue clear and closed")
self.file.delete()
else:
DEBUG and Log.note(
"persistent queue closed with {{num}} items left",
num=len(self))
try:
self._add_pending({"add": {"status.start": self.start}})
for i in range(self.db.status.start, self.start):
self._add_pending({"remove": str(i)})
self.file.write(
value2json({"add": self.db}) + "\n" +
("\n".join(value2json(p)
for p in self.pending)) + "\n")
self._apply_pending()
except Exception as e:
raise e
self.db = None
@property
def closed(self):
with self.lock:
return self.db is None
class Queue(object):
"""
SIMPLE MESSAGE QUEUE, multiprocessing.Queue REQUIRES SERIALIZATION, WHICH
IS DIFFICULT TO USE JUST BETWEEN THREADS (SERIALIZATION REQUIRED)
"""
def __init__(self,
name,
max=None,
silent=False,
unique=False,
allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
if not _Log:
_late_import()
self.name = name
self.max = coalesce(max, 2**10)
self.silent = silent
self.allow_add_after_close = allow_add_after_close
self.unique = unique
self.please_stop = Signal("stop signal for " + name)
self.lock = Lock("lock for queue " + name)
self.queue = deque()
self.next_warning = time() # FOR DEBUGGING
def __iter__(self):
try:
while True:
value = self.pop(self.please_stop)
if value is THREAD_STOP:
break
if value is not None:
yield value
except Exception as e:
_Log.warning("Tell me about what happened here", e)
if not self.silent:
_Log.note("queue iterator is done")
def add(self, value, timeout=None):
with self.lock:
if value is THREAD_STOP:
# INSIDE THE lock SO THAT EXITING WILL RELEASE wait()
self.queue.append(value)
self.please_stop.go()
return
self._wait_for_queue_space(timeout=timeout)
if self.please_stop and not self.allow_add_after_close:
_Log.error("Do not add to closed queue")
else:
if self.unique:
if value not in self.queue:
self.queue.append(value)
else:
self.queue.append(value)
return self
def push(self, value):
"""
SNEAK value TO FRONT OF THE QUEUE
"""
if self.please_stop and not self.allow_add_after_close:
_Log.error("Do not push to closed queue")
with self.lock:
self._wait_for_queue_space()
if not self.please_stop:
self.queue.appendleft(value)
return self
def pop_message(self, till=None):
"""
RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE
DUMMY IMPLEMENTATION FOR DEBUGGING
"""
if till is not None and not isinstance(till, Signal):
_Log.error("Expecting a signal")
return Null, self.pop(till=till)
def extend(self, values):
if self.please_stop and not self.allow_add_after_close:
_Log.error("Do not push to closed queue")
with self.lock:
# ONCE THE queue IS BELOW LIMIT, ALLOW ADDING MORE
self._wait_for_queue_space()
if not self.please_stop:
if self.unique:
for v in values:
if v is THREAD_STOP:
self.please_stop.go()
continue
if v not in self.queue:
self.queue.append(v)
else:
for v in values:
if v is THREAD_STOP:
self.please_stop.go()
continue
self.queue.append(v)
return self
def _wait_for_queue_space(self, timeout=DEFAULT_WAIT_TIME):
"""
EXPECT THE self.lock TO BE HAD, WAITS FOR self.queue TO HAVE A LITTLE SPACE
"""
wait_time = 5
if DEBUG and len(self.queue) > 1 * 1000 * 1000:
Log.warning("Queue {{name}} has over a million items")
now = time()
if timeout != None:
time_to_stop_waiting = now + timeout
else:
time_to_stop_waiting = Null
if self.next_warning < now:
self.next_warning = now + wait_time
while not self.please_stop and len(self.queue) >= self.max:
if now > time_to_stop_waiting:
if not _Log:
_late_import()
_Log.error(THREAD_TIMEOUT)
if self.silent:
self.lock.wait(Till(till=time_to_stop_waiting))
else:
self.lock.wait(Till(timeout=wait_time))
if len(self.queue) >= self.max:
now = time()
if self.next_warning < now:
self.next_warning = now + wait_time
_Log.alert(
"Queue by name of {{name|quote}} is full with ({{num}} items), thread(s) have been waiting {{wait_time}} sec",
name=self.name,
num=len(self.queue),
wait_time=wait_time)
def __len__(self):
with self.lock:
return len(self.queue)
def __nonzero__(self):
with self.lock:
return any(r != THREAD_STOP for r in self.queue)
def pop(self, till=None):
"""
WAIT FOR NEXT ITEM ON THE QUEUE
RETURN THREAD_STOP IF QUEUE IS CLOSED
RETURN None IF till IS REACHED AND QUEUE IS STILL EMPTY
:param till: A `Signal` to stop waiting and return None
:return: A value, or a THREAD_STOP or None
"""
if till is not None and not isinstance(till, Signal):
_Log.error("expecting a signal")
with self.lock:
while True:
if self.queue:
value = self.queue.popleft()
return value
if self.please_stop:
break
if not self.lock.wait(till=till | self.please_stop):
if self.please_stop:
break
return None
if DEBUG or not self.silent:
_Log.note(self.name + " queue stopped")
return THREAD_STOP
def pop_all(self):
"""
NON-BLOCKING POP ALL IN QUEUE, IF ANY
"""
with self.lock:
output = list(self.queue)
self.queue.clear()
return output
def pop_one(self):
"""
NON-BLOCKING POP IN QUEUE, IF ANY
"""
with self.lock:
if self.please_stop:
return [THREAD_STOP]
elif not self.queue:
return None
else:
v = self.queue.pop()
if v is THREAD_STOP: # SENDING A STOP INTO THE QUEUE IS ALSO AN OPTION
self.please_stop.go()
return v
def close(self):
with self.lock:
self.please_stop.go()
def commit(self):
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
