def _put_many(self, side: Side, objs: Iterable, block, timeout):
objs = [self._serializer.dumps(obj) for obj in objs]
if side == Side.LEFT:
objs = objs[::-1]
maxsize = self.maxsize or float("inf")
push_method = getattr(self._redis, side + "push")
if not block:
if len(objs) > maxsize - len(self):
raise Full()
else:
push_method(self.name, *objs)
return
started = datetime.utcnow()
length = len(self)
while len(objs) > maxsize - length:
self.log.tick(
"Would be overfull (%s of %s), sleeping for %s",
length,
maxsize,
POLL_INTERVAL,
)
time.sleep(POLL_INTERVAL)
if timeout is not None:
elapsed = datetime.utcnow() - started
if elapsed.total_seconds() >= timeout:
raise Full()
length = len(self)
push_method(self.name, *objs)
self.log.debug("Put %s element(s) on %s of queue", len(objs), side)
def test_raiseException_when_queuesIsFull(self):
self.queue_save_data.put_nowait = MagicMock(side_effect=Full())
self.queue_send_data.put_nowait = MagicMock(side_effect=Full())
item = BaseItem()
self.thread.put_in_queues(item)
eq_(self.queue_save_data.qsize(), 0)
eq_(self.queue_send_data.qsize(), 0)
def append(self, obj):
# nop if closed (should it raise?)
if self._put_closed:
raise Full()
# extract flow
try:
flow, obj = obj
except Exception:
if obj is not _sentinel:
raise
# Queue has passed us _sentinel as obj, close the workers!
self._put_closed = True
return
# get queue for flow
try:
flow_queue = self._flow_queues[flow]
except KeyError:
# add it to the queue
flow_queue = self._flow_queues[flow] = SimpleQueue()
self.unlock_flow(flow)
# add obj to flow queue
flow_queue.put(obj)
def add_first(self, e):
if self._size == len(self._data):
raise Full("Full Queue Exception")
#avail = self._front - 1
self._front = (self._front-1) % len(self._data)
self._data[self._front] = e
self._size += 1
def add_last(self, e):
if self._size == len(self._data):
raise Full("Full Queue Exception")
''' Insert e at the end of the queue '''
avail = (self._front+self._size) % len(self._data)
self._data[avail] = e
self._size += 1
def enqueue(self, e): #O(1)
if self._size == len(self._data):
raise Full("Full Queue Exception") #ERROR
''' Insert e at the end of the queue '''
avail = (self._front + self._size) % len(self._data) #N #!!!!!!
self._data[avail] = e
self._size += 1
def add_first(self, e):
if self._size == len(self._data):
raise Full("Full Deque Exception")
loc = (self._front - 1) % len(self._data)
self._data[loc] = e
self._front = (self._front - 1) % len(self._data)
self._size += 1
def enqueue(self, e):
"""Add an element to the back of queue."""
if self._size == len(self._data):
raise Full('Queue is Full')
#self._resize(2 * len(self.data)) # double the array size
avail = (self._front + self._size) % len(self._data)
self._data[avail] = e
self._size += 1
def __push(self, coefficients: ndarray) -> None:
try:
self.__params.coeff_queue.put(coefficients, timeout=TIMEOUT)
except AssertionError:
err_msg = ('Coefficient queue is already closed. Instantiate'
' a new <Parallel> object to get going again!')
raise AssertionError(err_msg)
except Full:
raise Full('Coefficient queue is full!')
def __push(self, array: ndarray) -> None:
try:
self.__params.point_queue.put(array, timeout=TIMEOUT)
except AssertionError:
err_msg = ('Point queue is already closed. Instantiate a'
' new <Parallel> object to start all over!')
raise AssertionError(err_msg)
except Full:
raise Full('Point queue is full!')
def check_full(self):
while True:
try:
self.last_item = self.read_queue.get(timeout=0.000001)
except Empty:
break
if self.view.i_item == self.last_item:
raise Full("Queue of length {} full when trying to insert {}, last item read was {}".format(self.view.n_items,
self.view.i_item, self.last_item))
def wait_write(fd, block=True, timeout=0):
if block:
timeout = None
r, w, x = select([], [fd], [], timeout)
if not w:
if not block:
raise Full()
else:
raise EOF("Write Error")
def put(self, item: QueueMessage, block=True, timeout=None):
"""Overwrite ``DeltaQueue.put``."""
if self._saved_value is None:
to_put = self._index_and_put(item, block=block, timeout=timeout)
# same rule as in _index_and_put: only non-None is saved
if to_put.msg is not None:
self._saved_value = to_put
else:
raise Full("Put to already populated ConstQueue.")
def __next__(self):
# type: () -> T
if self.iterator is None:
raise TypeError
if not self.semaphore.acquire(timeout=self.timeout):
raise Full("BoundedQueue semaphore acquisition timed out")
return next(self.iterator)
def put(self, item: QueueMessage, block=True, timeout=None):
"""Overwrite ``DeltaQueue.put``."""
if self._saved_value is None:
self._delta_put(item, block, timeout)
# only non-None is saved
if item.msg is not None:
self._saved_value = item
else:
raise Full("Put to already populated ConstQueue.")
def test_noPutItemInQueue_when_queueIsFullInLoop(self):
queue_notice = Queue()
queue_send_data = Queue()
queue_send_data.put_nowait = MagicMock(side_effect=Full())
apply_sql_file(path.join(self.path_sql, 'data_example.sql'))
thread = DBToSendThread(queue_send_data=queue_send_data,
db=self.dev_db,
queue_notice=queue_notice)
thread.activity()
eq_(queue_send_data.qsize(), 0)
def put(self, delay_message):
"""Adds message to queue.
:param delay_message: DelayMessage instance.
:type delay_message: DelayMessage
"""
assert isinstance(delay_message, DelayMessage)
if self.full():
raise Full("Delay queue is full")
self._queue.put((
delay_message.eta.timestamp(),
delay_message.delivery_tag,
delay_message,
))
def append(self, obj):
# nop if closed (should it raise?)
if self._put_closed:
raise Full()
# extract flow
try:
priority, obj = obj
except Exception:
if obj is not _sentinel:
raise
# Queue has passed us _sentinel as obj, close the workers!
self._put_closed = True
return
# finally put item!
self._priority_queue.put(PrioritizedItem(priority, obj))
def push(self, priority, value):
"""Add an item to the queue with priority.
Args:
priority: An integer that determines the placement of `value` in
the queue. Must be unique.
value: The value to queue.
Raises:
Full if queue is full.
"""
if self.full:
raise Full()
if priority in self.queue:
raise ValueError("Duplicate priority value: {}".format(priority))
self.queue[priority] = value
if self.min_priority is None or priority < self.min_priority:
self.min_priority = priority
def enqueue(self, e):
if (self.is_full()):
raise Full()
self._data[(self._front + self._size) % len(self._data)] = e
self._size += 1
async def put_nowait(self, item):
if self.full():
raise Full()
self._put(item)
self._task_count += 1
await self._get_waiting.notify()
def save(self, data):
try:
self.q.put_nowait(data)
except Full:
raise Full('dataset buffer overflow')
def push(self, priority, value):
if self.full:
raise Full()
self.queue[priority] = value
if self.min_priority is None or priority < self.min_priority:
self.min_priority = priority
def add_last(self, e):
if self._size == len(self._data):
raise Full("Full Queue Exception")
''' Insert e at the end of the queue '''
pass
def put(self, item, **kwargs):
"""Put an item into the queue."""
if self.full():
raise Full()
self._append(item)
