class Queue(object):
"""
SIMPLE MULTI-THREADED QUEUE
(multiprocessing.Queue REQUIRES SERIALIZATION, WHICH
IS DIFFICULT TO USE JUST BETWEEN THREADS)
"""
def __init__(self,
name,
max=None,
silent=False,
unique=False,
allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
self.name = name
self.max = coalesce(max, 2**10)
self.silent = silent
self.allow_add_after_close = allow_add_after_close
self.unique = unique
self.closed = Signal(
"stop adding signal for " +
name) # INDICATE THE PRODUCER IS DONE GENERATING ITEMS TO QUEUE
self.lock = Lock("lock for queue " + name)
self.queue = deque()
def __iter__(self):
try:
while True:
value = self.pop()
if value is THREAD_STOP:
break
if value is not None:
yield value
except Exception as e:
Log.warning("Tell me about what happened here", e)
def add(self, value, timeout=None, force=False):
"""
:param value: ADDED THE THE QUEUE
:param timeout: HOW LONG TO WAIT FOR QUEUE TO NOT BE FULL
:param force: ADD TO QUEUE, EVEN IF FULL (USE ONLY WHEN CONSUMER IS RETURNING WORK TO THE QUEUE)
:return: self
"""
with self.lock:
if value is THREAD_STOP:
# INSIDE THE lock SO THAT EXITING WILL RELEASE wait()
self.queue.append(value)
self.closed.go()
return
if not force:
self._wait_for_queue_space(timeout=timeout)
if self.closed and not self.allow_add_after_close:
Log.error("Do not add to closed queue")
if self.unique:
if value not in self.queue:
self.queue.append(value)
else:
self.queue.append(value)
return self
def push(self, value):
"""
SNEAK value TO FRONT OF THE QUEUE
"""
if self.closed and not self.allow_add_after_close:
Log.error("Do not push to closed queue")
with self.lock:
self._wait_for_queue_space()
if not self.closed:
self.queue.appendleft(value)
return self
def push_all(self, values):
"""
SNEAK values TO FRONT OF THE QUEUE
"""
if self.closed and not self.allow_add_after_close:
Log.error("Do not push to closed queue")
with self.lock:
self._wait_for_queue_space()
if not self.closed:
self.queue.extendleft(values)
return self
def pop_message(self, till=None):
"""
RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE
DUMMY IMPLEMENTATION FOR DEBUGGING
"""
if till is not None and not isinstance(till, Signal):
Log.error("Expecting a signal")
return Null, self.pop(till=till)
def extend(self, values):
if self.closed and not self.allow_add_after_close:
Log.error("Do not push to closed queue")
with self.lock:
# ONCE THE queue IS BELOW LIMIT, ALLOW ADDING MORE
self._wait_for_queue_space()
if not self.closed:
if self.unique:
for v in values:
if v is THREAD_STOP:
self.closed.go()
continue
if v not in self.queue:
self.queue.append(v)
else:
for v in values:
if v is THREAD_STOP:
self.closed.go()
continue
self.queue.append(v)
return self
def _wait_for_queue_space(self, timeout=None):
"""
EXPECT THE self.lock TO BE HAD, WAITS FOR self.queue TO HAVE A LITTLE SPACE
:param timeout: IN SECONDS
"""
wait_time = 5
(DEBUG and len(self.queue) > 1 * 1000 * 1000
) and Log.warning("Queue {{name}} has over a million items")
start = time()
stop_waiting = Till(till=start + coalesce(timeout, DEFAULT_WAIT_TIME))
while not self.closed and len(self.queue) >= self.max:
if stop_waiting:
Log.error(THREAD_TIMEOUT)
if self.silent:
self.lock.wait(stop_waiting)
else:
self.lock.wait(Till(seconds=wait_time))
if not stop_waiting and len(self.queue) >= self.max:
now = time()
Log.alert(
"Queue with name {{name|quote}} is full with ({{num}} items), thread(s) have been waiting {{wait_time}} sec",
name=self.name,
num=len(self.queue),
wait_time=now - start)
def __len__(self):
with self.lock:
return len(self.queue)
def __nonzero__(self):
with self.lock:
return any(r != THREAD_STOP for r in self.queue)
def pop(self, till=None):
"""
WAIT FOR NEXT ITEM ON THE QUEUE
RETURN THREAD_STOP IF QUEUE IS CLOSED
RETURN None IF till IS REACHED AND QUEUE IS STILL EMPTY
:param till: A `Signal` to stop waiting and return None
:return: A value, or a THREAD_STOP or None
"""
if till is not None and not isinstance(till, Signal):
Log.error("expecting a signal")
with self.lock:
while True:
if self.queue:
return self.queue.popleft()
if self.closed:
break
if not self.lock.wait(till=self.closed | till):
if self.closed:
break
return None
(DEBUG or not self.silent) and Log.note(self.name + " queue closed")
return THREAD_STOP
def pop_all(self):
"""
NON-BLOCKING POP ALL IN QUEUE, IF ANY
"""
with self.lock:
output = list(self.queue)
self.queue.clear()
return output
def pop_one(self):
"""
NON-BLOCKING POP IN QUEUE, IF ANY
"""
with self.lock:
if self.closed:
return THREAD_STOP
elif not self.queue:
return None
else:
v = self.queue.popleft()
if v is THREAD_STOP: # SENDING A STOP INTO THE QUEUE IS ALSO AN OPTION
self.closed.go()
return v
def close(self):
self.closed.go()
def commit(self):
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
class Queue(object):
"""
SIMPLE MESSAGE QUEUE, multiprocessing.Queue REQUIRES SERIALIZATION, WHICH
IS DIFFICULT TO USE JUST BETWEEN THREADS (SERIALIZATION REQUIRED)
"""
def __init__(self,
name,
max=None,
silent=False,
unique=False,
allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
if not _Log:
_late_import()
self.name = name
self.max = coalesce(max, 2**10)
self.silent = silent
self.allow_add_after_close = allow_add_after_close
self.unique = unique
self.please_stop = Signal("stop signal for " + name)
self.lock = Lock("lock for queue " + name)
self.queue = deque()
self.next_warning = time() # FOR DEBUGGING
def __iter__(self):
try:
while True:
value = self.pop(self.please_stop)
if value is THREAD_STOP:
break
if value is not None:
yield value
except Exception as e:
_Log.warning("Tell me about what happened here", e)
if not self.silent:
_Log.note("queue iterator is done")
def add(self, value, timeout=None):
with self.lock:
if value is THREAD_STOP:
# INSIDE THE lock SO THAT EXITING WILL RELEASE wait()
self.queue.append(value)
self.please_stop.go()
return
self._wait_for_queue_space(timeout=timeout)
if self.please_stop and not self.allow_add_after_close:
_Log.error("Do not add to closed queue")
else:
if self.unique:
if value not in self.queue:
self.queue.append(value)
else:
self.queue.append(value)
return self
def push(self, value):
"""
SNEAK value TO FRONT OF THE QUEUE
"""
if self.please_stop and not self.allow_add_after_close:
_Log.error("Do not push to closed queue")
with self.lock:
self._wait_for_queue_space()
if not self.please_stop:
self.queue.appendleft(value)
return self
def pop_message(self, till=None):
"""
RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE
DUMMY IMPLEMENTATION FOR DEBUGGING
"""
if till is not None and not isinstance(till, Signal):
_Log.error("Expecting a signal")
return Null, self.pop(till=till)
def extend(self, values):
if self.please_stop and not self.allow_add_after_close:
_Log.error("Do not push to closed queue")
with self.lock:
# ONCE THE queue IS BELOW LIMIT, ALLOW ADDING MORE
self._wait_for_queue_space()
if not self.please_stop:
if self.unique:
for v in values:
if v is THREAD_STOP:
self.please_stop.go()
continue
if v not in self.queue:
self.queue.append(v)
else:
for v in values:
if v is THREAD_STOP:
self.please_stop.go()
continue
self.queue.append(v)
return self
def _wait_for_queue_space(self, timeout=DEFAULT_WAIT_TIME):
"""
EXPECT THE self.lock TO BE HAD, WAITS FOR self.queue TO HAVE A LITTLE SPACE
"""
wait_time = 5
now = time()
if timeout != None:
time_to_stop_waiting = now + timeout
else:
time_to_stop_waiting = Null
if self.next_warning < now:
self.next_warning = now + wait_time
while not self.please_stop and len(self.queue) >= self.max:
if now > time_to_stop_waiting:
if not _Log:
_late_import()
_Log.error(THREAD_TIMEOUT)
if self.silent:
self.lock.wait(Till(till=time_to_stop_waiting))
else:
self.lock.wait(Till(timeout=wait_time))
if len(self.queue) > self.max:
now = time()
if self.next_warning < now:
self.next_warning = now + wait_time
_Log.alert(
"Queue by name of {{name|quote}} is full with ({{num}} items), thread(s) have been waiting {{wait_time}} sec",
name=self.name,
num=len(self.queue),
wait_time=wait_time)
def __len__(self):
with self.lock:
return len(self.queue)
def __nonzero__(self):
with self.lock:
return any(r != THREAD_STOP for r in self.queue)
def pop(self, till=None):
"""
WAIT FOR NEXT ITEM ON THE QUEUE
RETURN THREAD_STOP IF QUEUE IS CLOSED
RETURN None IF till IS REACHED AND QUEUE IS STILL EMPTY
:param till: A `Signal` to stop waiting and return None
:return: A value, or a THREAD_STOP or None
"""
if till is not None and not isinstance(till, Signal):
_Log.error("expecting a signal")
with self.lock:
while True:
if self.queue:
value = self.queue.popleft()
return value
if self.please_stop:
break
if not self.lock.wait(till=till | self.please_stop):
if self.please_stop:
break
return None
if DEBUG or not self.silent:
_Log.note(self.name + " queue stopped")
return THREAD_STOP
def pop_all(self):
"""
NON-BLOCKING POP ALL IN QUEUE, IF ANY
"""
with self.lock:
output = list(self.queue)
self.queue.clear()
return output
def pop_one(self):
"""
NON-BLOCKING POP IN QUEUE, IF ANY
"""
with self.lock:
if self.please_stop:
return [THREAD_STOP]
elif not self.queue:
return None
else:
v = self.queue.pop()
if v is THREAD_STOP: # SENDING A STOP INTO THE QUEUE IS ALSO AN OPTION
self.please_stop.go()
return v
def close(self):
with self.lock:
self.please_stop.go()
def commit(self):
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
