def _get(self, queue):
"""Try to retrieve a single message off ``queue``."""
q_url = self._new_queue(queue)
resp = self.sqs(queue=queue).receive_message(
QueueUrl=q_url, MaxNumberOfMessages=1,
WaitTimeSeconds=self.wait_time_seconds)
if resp.get('Messages'):
body = AsyncMessage(body=resp['Messages'][0]['Body']).decode()
resp['Messages'][0]['Body'] = body
return self._messages_to_python(resp['Messages'], queue)[0]
raise Empty()
def dequeue(self):
"""Remove and return the first element of the queue (i.e., FIFO).
Raise Empty exception if the queue is empty.
"""
if self.is_empty():
raise Empty('Queue is empty')
answer = self._data[self._front]
self._data[self._front] = None # help garbage collection
self._front = (self._front + 1) % len(self._data)
self._size -= 1
return answer
def test_poll_readable_raises_Empty(self):
x = X(self.app)
reader = Mock(name='reader')
x.hub.add_reader(6, reader, 6)
x.hub.on_tick.add(x.close_then_error(Mock(name='tick'), 2))
poller = x.hub.poller
poller.poll.return_value = [(6, READ)]
reader.side_effect = Empty()
with pytest.raises(socket.error):
asynloop(*x.args)
reader.assert_called_with(6)
poller.poll.assert_called()
def get(self, nowait=False):
if self._queue.empty():
s = input()
if s == '':
if nowait:
raise Empty()
else:
raise MachineBlockedException()
for c in s:
self._queue.put(ord(c))
self._queue.put(10)
return self._queue.get()
def drain_events(self, timeout=None, callback=None, **kwargs):
"""Return a single payload message from one of our queues.
Raises:
Queue.Empty: if no messages available.
"""
# If we're not allowed to consume or have no consumers, raise Empty
if not self._consumers or not self.qos.can_consume():
raise Empty()
# At this point, go and get more messages from SQS
self._poll(self.cycle, callback, timeout=timeout)
def dequeue(self):
"""Remove and return the first element of the queue (i.e., FIFO).
Raise Empty exception if the queue is empty.
"""
if self.is_empty():
raise Empty('Queue is empty')
answer = self._head._element
self._head = self._head._next
self._size -= 1
if self.is_empty(): # special case as queue is empty
self._tail = None # removed head had been the tail
return answer
def pop(self):
"""Remove and return the item in the queue with lowest priority.
Raises:
Empty if the queue is emtpy.
"""
if self.empty:
raise Empty()
value = self.queue.pop(self.min_priority)
if self.empty:
self.min_priority = None
else:
self.min_priority = min(self.queue.keys())
return value
def boost(self, k): # O(k)
# moves the element from the back of the queue k steps forward.
# If the queue is empty an exception is raised.
# If k is too big (greater or equal to the number of elements in the
# queue) the last element will become the first.
# No return value.
if self.is_empty():
raise Empty()
if k >= self._size:
self._data = [self._data[-1]] + self._data[:-1]
else:
self._data = self._data[:self._front - k - 1] + [
self._data[-1]
] + self._data[self._front - k - 1:-1]
def dequeue(self):
"""Remove and return the first element of the queue (i.e., FIFO).
Raise Empty exception if the queue is empty.
"""
if self.is_empty():
raise Empty('Queue is empty')
oldhead = self._tail._next
if self._size == 1: # removing only element
self._tail = None # queue becomes empty
else:
self._tail._next = oldhead._next # bypass the old head
self._size -= 1
return oldhead._element
def test_deadline_exec_reply(self, preprocessor, cell_mock, message_mock):
# exec_reply is never received, so we expect to hit the timeout.
preprocessor.kc.shell_channel.get_msg = MagicMock(side_effect=Empty())
preprocessor.timeout = 1
with pytest.raises(TimeoutError):
preprocessor.run_cell(cell_mock)
assert message_mock.call_count == 3
# Ensure the output was captured
self.assertListEqual(cell_mock.outputs, [
{'output_type': 'stream', 'name': 'stdout', 'text': 'foo'},
{'output_type': 'stream', 'name': 'stderr', 'text': 'bar'}
])
def _get(self, queue):
"""Try to retrieve a single message off ``queue``."""
q = self._new_queue(queue)
rs = q.pop(1)
if rs['items']:
m = rs['items'][0]
payload = loads(bytes_to_str(m['body']))
if queue in self._noack_queues:
q.message(m['id']).delete()
else:
payload['properties']['delivery_info'].update({
'slmq_message_id': m['id'], 'slmq_queue_name': q.name})
return payload
raise Empty()
def _get(self, queue):
"""Get next message from `queue`."""
queue_folder = os.path.join(self.data_folder_in, queue)
if (not os.path.exists(queue_folder)):
raise Empty()
messages = os.listdir(queue_folder)
messages = sorted(messages)
while len(messages) > 0:
filename = messages.pop(0)
if self.store_processed:
processed_folder = os.path.join(self.processed_folder, queue)
os.makedirs(processed_folder, exist_ok = True)
else:
processed_folder = tempfile.gettempdir()
try:
# move the file to the tmp/processed folder
shutil.move(os.path.join(queue_folder, filename),
processed_folder)
except OSError:
pass # file could be locked, or removed in meantime so ignore
filename = os.path.join(processed_folder, filename)
try:
f = open(filename, 'rb')
payload = f.read()
f.close()
if not self.store_processed:
os.remove(filename)
except OSError:
raise ChannelError(
f'Cannot read file {filename!r} from queue.')
return loads(bytes_to_str(payload))
raise Empty()
def wait_read(fd, block=True, timeout=0):
if block:
timeout = None
while 1:
r, w, x = select([fd], [], [], timeout)
if not r:
if not block:
raise Empty()
else:
raise EOF("Select Error")
if block:
st = fstat(fd)
if not st.st_size:
sleep(0.01)
continue
def _get(self, queue, timeout=None):
"""Try to retrieve a single message off ``queue``."""
q = self._ensure_queue(queue)
messages = q.receive_messages(messages_per_page=1, timeout=timeout)
try:
message = next(messages)
except StopIteration:
raise Empty()
content = loads(message.content)
q.delete_message(message=message)
return content
def get(self):
"""
Get the next pending request to be dispatched.
Blocks until a request is available.
:return: The next pending request.
:rtype: Document
:raise Empty: on thread aborted.
"""
while not Thread.aborted():
try:
return self.queue.get(timeout=3)
except Empty:
pass
# aborted
raise Empty()
def execute(self):
# parsing the arguments
jumpFlag = True
opcode = str(self.instructs[self.pointer])
argc = self.argc(opcode)
argv = self.instructs[self.pointer + 1: self.pointer + argc]
args = [(k, v) for (k, v) in zip(argv, opcode.zfill(argc + 1)[:-2][::-1])]
if opcode.endswith('99'):
self.running = False
elif opcode.endswith('1'): # addition
self.instructs[self.get_position(args[2])] = self.get_value(args[0]) + self.get_value(args[1])
elif opcode.endswith('2'): # multiplication
self.instructs[self.get_position(args[2])] = self.get_value(args[0]) * self.get_value(args[1])
elif opcode.endswith('3'): # input
if self.inputs.empty():
raise Empty("something went wrong: no inputs")
self.instructs[self.get_position(args[0])] = self.inputs.get()
elif opcode.endswith('4'): # output
self.inputs.put(self.get_value(args[0]))
self.paused = True # pause the execution
elif opcode.endswith('5'): # jump if true
if self.get_value(args[0]) != 0:
jumpFlag = False
self.pointer = self.get_value(args[1])
elif opcode.endswith('6'): # jump if false
if self.get_value(args[0]) == 0 :
jumpFlag = False
self.pointer = self.get_value(args[1])
elif opcode.endswith('7'): # less than
self.instructs[self.get_position(args[2])] = 1 if self.get_value(args[0]) < self.get_value(args[1]) else 0
elif opcode.endswith('8'): # equal
self.instructs[self.get_position(args[2])] = 1 if self.get_value(args[0]) == self.get_value(args[1]) else 0
elif opcode.endswith('9'): # add to the relative position
self.relative_base += self.get_value(args[0])
if jumpFlag:
self.pointer += argc
def _get(self, queue, timeout=None):
"""Try to retrieve a single message off ``queue``."""
q = self._ensure_queue(queue)
messages = self.queue_service.get_messages(q, num_messages=1,
timeout=timeout)
if not messages:
raise Empty()
message = messages[0]
raw_content = self.queue_service.decode_function(message.content)
content = loads(raw_content)
self.queue_service.delete_message(q, message.id, message.pop_receipt)
return content
def test_eventual_deadline_iopub(self, preprocessor, cell_mock, message_mock):
# Process a few messages before raising a timeout from iopub
message_mock.side_effect = list(message_mock.side_effect)[:-1] + [Empty()]
preprocessor.kc.shell_channel.get_msg = MagicMock(
return_value={'parent_header': {'msg_id': preprocessor.parent_id}})
preprocessor.raise_on_iopub_timeout = True
with pytest.raises(TimeoutError):
preprocessor.run_cell(cell_mock)
assert message_mock.call_count == 3
# Ensure the output was captured
self.assertListEqual(cell_mock.outputs, [
{'output_type': 'stream', 'name': 'stdout', 'text': 'foo'},
{'output_type': 'stream', 'name': 'stderr', 'text': 'bar'}
])
def get_nowait(self):
with self.lock:
if len(self.keys) == 0:
self.keys = self.queues.keys()
while (len(self.keys) != 0):
k = self.keys.pop()
if k not in self.queues:
continue
q = self.queues[k]
f = self.funcs[k]
try:
v = q.get_nowait()
except Empty:
continue
#IPython.embed()
return (v, f, k)
raise Empty()
def test_run_once_empty_queue():
channel = MagicMock()
publisher = MagicMock()
queue = MagicMock()
logger = MagicMock()
queue.get = MagicMock()
queue.get.side_effect = Empty()
publisher.publish = MagicMock()
channel.complete = MagicMock()
nanny_thread = kadabra.agent.NannyThread(channel, publisher, queue, logger)
nanny_thread._run_once()
queue.get.assert_called_with(timeout=10)
publisher.publish.assert_has_calls([])
channel.complete.assert_has_calls([])
def _get(self, side: Side, block=True, timeout=None) -> object:
if block:
msg = getattr(self._redis, "b{}pop".format(side))(
self.name, timeout=self._int_timeout(timeout))
if msg is not None:
msg = msg[1]
else:
msg = getattr(self._redis, side + "pop")(self.name)
if msg is None:
raise Empty()
if self._serializer:
msg = self._serializer.loads(msg)
self.log.debug("Got item from %s", side)
return msg
def wrapper(*args, **kwargs):
device_instance = args[0]
if threading.get_ident() == device_instance.current_thread:
command_set = [func, args, kwargs]
device_instance.command_queue.put(command_set)
while True:
try:
return device_instance.reply_queue.get(timeout=10)
# Setting timeout to 10 secs is a temporary change to fix the issue when library hangs
# if exception is thrown from the function wrapped with @command decorator and get_return=True
# Without timeout here's what happens:
# wrapper puts the wrapped function in the queue with device_instance.command_queue.put(command_set) and then main thread blocks on reply_queue.get()
# Then a command execution thread fetches & executes the function where exception occurs, so potentially no command is sent to serial port & no reply can be expected
# But main thread keeps blocked forever on reply_queue.get() because it has no clue that command has not been actually sent
except Empty:
raise Empty("Reply queue timeout!") from None
else:
return func(*args, **kwargs)
def _get(self, queue):
obj = self._get_or_create(queue)
if self.session.bind.name == 'sqlite':
self.session.execute('BEGIN IMMEDIATE TRANSACTION')
try:
msg = self.session.query(self.message_cls) \
.with_for_update() \
.filter(self.message_cls.queue_id == obj.id) \
.filter(self.message_cls.visible != False) \
.order_by(self.message_cls.sent_at) \
.order_by(self.message_cls.id) \
.limit(1) \
.first()
if msg:
msg.visible = False
return loads(bytes_to_str(msg.payload))
raise Empty()
finally:
self.session.commit()
def _get_bulk(self,
queue,
max_if_unlimited=SQS_MAX_MESSAGES,
callback=None):
"""Try to retrieve multiple messages off ``queue``.
Where :meth:`_get` returns a single Payload object, this method
returns a list of Payload objects. The number of objects returned
is determined by the total number of messages available in the queue
and the number of messages the QoS object allows (based on the
prefetch_count).
Note:
Ignores QoS limits so caller is responsible for checking
that we are allowed to consume at least one message from the
queue. get_bulk will then ask QoS for an estimate of
the number of extra messages that we can consume.
Arguments:
queue (str): The queue name to pull from.
Returns:
List[Message]
"""
# drain_events calls `can_consume` first, consuming
# a token, so we know that we are allowed to consume at least
# one message.
# Note: ignoring max_messages for SQS with boto3
max_count = self._get_message_estimate()
if max_count:
q_url = self._new_queue(queue)
resp = self.sqs(queue=queue).receive_message(
QueueUrl=q_url,
MaxNumberOfMessages=max_count,
WaitTimeSeconds=self.wait_time_seconds)
if resp.get('Messages'):
for m in resp['Messages']:
m['Body'] = AsyncMessage(body=m['Body']).decode()
for msg in self._messages_to_python(resp['Messages'], queue):
self.connection._deliver(msg, queue)
return
raise Empty()
def _get(self, queue):
if queue in self._fanout_queues:
try:
msg = next(self._get_broadcast_cursor(queue))
except StopIteration:
msg = None
else:
msg = self.messages.find_one_and_delete(
{'queue': queue},
sort=[('priority', pymongo.ASCENDING)],
)
if self.ttl:
self._update_queues_expire(queue)
if msg is None:
raise Empty()
return loads(bytes_to_str(msg['payload']))
def runCC(this, self): #this es CC, self es mainClass
# print("Buscando el termino '" + this.aBuscar + "' en " + this.parametro_url) #debug
done = False
# Resetear index para limpiar busquedas pasadas
self.model.df = self.model.df.iloc[0:0]
while not done:
try:
# Limitar coincidencias
if self.ui.cBLimitarCoincidencias.isChecked():
if len(this.enlacesEncontrados
) - 1 == self.ui.sBCoincidencias.value():
raise Empty()
# Sacar enlace de la cola
enlace = this.colaCrawl.get(timeout=5)
# Comprobar paths de las urls, no podemos comprobar urls ya que podrian tener netlocs diferentes con paths iguales
if any(
urlparse(x.url).path == urlparse(enlace.url).path
for x in this.enlacesEncontrados):
continue
# Meter en el dataframe del tableview en enlace
if len(this.enlacesEncontrados) > 0:
df = pd.DataFrame([[enlace.texto, enlace.url]],
columns=('Enlace', 'URL'))
self.updateProgress.emit(df)
# Meter enlace a la cola de enlaces vistados
this.enlacesEncontrados.append(enlace)
# Scrap workers
worker = this.workers.submit(this.scrapWeb, enlace.url)
worker.add_done_callback(this.scrapeCallback)
time.sleep(0.5)
except Empty:
self.ui.lEstadoActual.setText('Busqueda finalizada')
done = True
return
except Exception as e:
print(e)
continue
def Receive(self, Wait=False, Timeout=120):
logger.debug("In DIMSEprovider.Receive")
if self.message is None:
self.message = DIMSEMessage()
if Wait:
# loop until complete DIMSE message is received
logger.debug('Entering loop for receiving DIMSE message')
# If connection fails, the peek loop can iterate forever, as the
# DUL Receive never happens. Approximate a timeout to abort
itrs = 0
delay = 0.001
while 1:
time.sleep(delay)
nxt = self.DUL.Peek()
if nxt is None:
itrs += 1
if Timeout and itrs > Timeout / float(delay):
# just like the DUL.Receive would on timeout
raise Empty('Timeout waiting for DIMSE message')
else:
continue
if nxt.__class__ is not P_DATA_ServiceParameters:
return None, None
if self.message.Decode(self.DUL.Receive(Wait, Timeout)):
tmp = self.message
self.message = None
logger.debug('Decoded DIMSE message: %s', str(tmp))
return tmp.ToParams(), tmp.ID
else:
cls = self.DUL.Peek().__class__
if cls not in (type(None), P_DATA_ServiceParameters):
logger.debug('Waiting for P-DATA but received %s', cls)
return None, None
if self.message.Decode(self.DUL.Receive(Wait, Timeout)):
tmp = self.message
self.message = None
logger.debug('Received DIMSE message: %s', tmp)
return tmp.ToParams(), tmp.ID
else:
return None, None
def _brpop_read(self, **options):
try:
try:
dest__item = self.client.parse_response(
self.client.connection, 'BRPOP', **options)
except self.connection_errors:
# if there's a ConnectionError, disconnect so the next
# iteration will reconnect automatically.
self.client.connection.disconnect()
raise
if dest__item:
dest, item = dest__item
dest = bytes_to_str(dest).rsplit(self.sep, 1)[0]
self._queue_cycle.rotate(dest)
self.connection._deliver(loads(bytes_to_str(item)), dest)
return True
else:
raise Empty()
finally:
self._in_poll = None
def get(
self,
block: bool = True,
timeout: float = float("inf")
) -> Generator[ServiceInstance, None, None]:
if not block:
timeout = 0.0
timeout_timestamp = time.time() + timeout
entry = None
with self.local_state_condition:
while True:
first_available_entry_node = self._get_first_available_entry_node(
)
if first_available_entry_node is not None:
entry = self._lock_and_get_entry(
first_available_entry_node)
if entry is not None:
break
next_upcoming_wait_until = self._get_next_upcoming_wait_until()
cond_wait_until = min(
timeout_timestamp,
next_upcoming_wait_until,
time.time() + MAX_SLEEP_TIME,
)
hit_timeout = not self.local_state_condition.wait(
timeout=cond_wait_until - time.time())
if hit_timeout and time.time() >= timeout_timestamp:
raise Empty()
entry_data, entry_stat = entry
try:
yield self._parse_data(entry_data)
except Exception:
self._release(first_available_entry_node)
raise
else:
self._consume(first_available_entry_node)
def get_random_image(self):
"""
TODO
Returns:
"""
with self.loading_pool.thread_lock:
self._warn_access_limit()
timeout_s = 5
try:
image_pair, n_accesses = self.loaded_queue.get(timeout=timeout_s)
except Empty as e:
raise Empty("Could not get ImagePair from dataset {} with "
"timeout of {} seconds. Consider increasing the "
"number of load threads / max loaded per dataset /"
" access threshold".format(self.dataset.identifier,
timeout_s)) from e
try:
yield image_pair
finally:
self._release_image(image_pair, n_accesses)
