def get_action(self):
"""
get waiting action from the policy's action queue. if there is no action
in the queue, pick 'F' and log the unresponsiveness error.
:return: action from {'R','L','F'}.
"""
try:
round, action = self.aq.get_nowait()
if round != self.round:
raise queue.Empty()
elif action not in base_policy.Policy.ACTIONS:
self.logq.put((str(self.id), "ERROR", ILLEGAL_MOVE + str(action)))
raise queue.Empty()
else:
self.too_slow = False
self.unresponsive_count = 0
except queue.Empty:
self.unresponsive_count += 1
action = base_policy.Policy.DEFAULT_ACTION
if self.unresponsive_count <= UNRESPONSIVE_THRESHOLD:
self.logq.put((str(self.id), "ERROR", NO_RESPONSE + str(self.unresponsive_count) + " in a row!"))
else:
self.logq.put((str(self.id), "ERROR", UNRESPONSIVE_PLAYER))
self.unresponsive_count = TOO_SLOW_THRESHOLD
if self.unresponsive_count > TOO_SLOW_THRESHOLD:
self.too_slow = True
clear_q(self.aq) # clear the queue from unhandled actions
return action
def _GetWorkLoad(self, workerCtxGroup):
'''
Retrieves a workload of the given context group.
:param workerCtxGroup:
'''
jcGroup = self.__jobCtxGroups[workerCtxGroup]
try:
with jcGroup:
while jcGroup.Active() is None:
jcIdle = jcGroup.Get()
if jcIdle is None:
raise WorkerAbortSignal()
if self.__StartCtx(jcIdle):
jcGroup.SetActive(jcIdle)
if self.__destroying:
# if in destroying state raise Queue.Empty() to enter
# the except section and get FinishCtx() called
raise queue.Empty()
jobCtxActive = jcGroup.Active()
workLoad = jobCtxActive.GetWorkLoad()
return jobCtxActive, workLoad # FIXME: no tuple
except queue.Empty:
# no more workloads, job done, only __FinishCtx() needs to be done
# wait for all workers to be done
with jcGroup:
jcGroup.IncDoneCount()
if jcGroup.CheckBusy():
self.__logger.debug("<%s> block until ready... %s",
threading.currentThread().getName(),
jcGroup.DoneCount())
jcGroup.WaitDoneEvent()
self.__logger.debug("<%s> block released continuing... %s",
threading.currentThread().getName(),
jcGroup.DoneCount())
else:
with jcGroup:
jobCtxActive = jcGroup.Active()
if jobCtxActive is not None:
jcGroup.SetActive(None)
self.__FinishCtx(jobCtxActive)
self.__logger.debug("<%s> set done... %s",
threading.currentThread().getName(),
jcGroup.DoneCount())
jcGroup.SetDoneEvent()
if self.__destroying:
raise WorkerAbortSignal()
else:
raise queue.Empty()
def get(self, timeout: tp.Optional[float] = None) -> Message:
if timeout is None:
return self.recv()
ready = self.poll(timeout)
if not ready:
raise queue.Empty()
return self.recv()
def _grab_task(self) -> WorkerTask:
"""Will initiate that a task is gotten from the queue and that it
spawns its worker process.
Returns:
WorkerTask: The WorkerTask grabbed from the queue.
Raises:
queue.Empty: If the task queue was empty
"""
# Get a task from the queue
try:
task = self.task_queue.get_nowait()
except queue.Empty as err:
raise queue.Empty(
"No more tasks available in tasks queue.") from err
else:
log.debug("Got task %s from queue. (Priority: %s)", task.uid,
task.priority)
# Let it spawn its own worker
task.spawn_worker()
# Now return the task
return task
def test_basics():
# setup
incoming = Mock()
incoming.get.side_effect = [(MessageType.ADD_SOURCE, "src1"),
queue.Empty()]
shared = Mock()
shared.queues.get_messages_to_controller.return_value = incoming
shared.queues.MessageType = MessageType
interrupt = Mock()
interrupt.is_set.return_value = False
ctr = BaseController.BaseController("test", shared)
ctr.add_interrupt(interrupt)
# test run
with pytest.raises(NotImplementedError):
ctr.run()
# test fetch_one_incoming
item = ctr.fetch_one_incoming()
assert item == (MessageType.ADD_SOURCE, "src1")
item = ctr.fetch_one_incoming()
assert item == (None, None)
# send_outgoing
outgoing_item = BaseSource(key="src2", rule="r2")
ctr.send_outgoing(outgoing_item)
item = shared.queues.run_expressions_in_rule.call_args[0]
assert item == (outgoing_item, )
def pop(self, index, timeout=None):
if not len(self._list):
with self.condition:
self.condition.wait(timeout)
if len(self._list) < index + 1:
raise queue.Empty()
return self._list.pop(index)
def get(self, timeout=None):
try:
return self.items.pop()
except IndexError:
if timeout:
raise queue.Empty()
raise
def get(self):
with self._rlock:
if self._size.value:
self._size.value -= 1
return _read_object(self._pipe_r)
else:
raise queue.Empty()
def test_basics():
# setup
incoming = Mock()
incoming.get.side_effect = [(MessageType.ADD_SOURCE, "src1"),
queue.Empty()]
shared = Mock()
shared.queues.get_messages_to_controller.return_value = incoming
shared.queues.MessageType = MessageType
interrupt = Mock()
interrupt.is_set.return_value = False
ctr = BaseController.BaseController("test", shared)
ctr.add_interrupt(interrupt)
# test run
with pytest.raises(NotImplementedError):
ctr.run()
# test fetch_one_incoming
item = ctr.fetch_one_incoming()
assert item == (MessageType.ADD_SOURCE, "src1")
item = ctr.fetch_one_incoming()
assert item is None
# send_outgoing
ctr.send_outgoing(BaseSource(key="src2", rule="r2"))
msg_t, rule2, src2 = shared.queues.send_message_to_rule.call_args[0]
assert msg_t == MessageType.RUN_EXPRESSION
assert rule2 == "r2"
assert src2.key == "src2"
def get_action(self, board_state):
"""
get waiting action from the policy's action queue. if there is no action
in the queue, pick a random action and log the unresponsiveness error.
:return: action (column to insert game piece).
"""
try:
round, action = self.aq.get_nowait()
if round != self.round:
raise queue.Empty()
else:
self.too_slow = False
self.unresponsive_count = 0
except queue.Empty:
self.unresponsive_count += 1
action = np.random.choice(get_legal_moves(board_state))
if self.unresponsive_count <= UNRESPONSIVE_THRESHOLD:
self.logq.put((str(self.id), "ERROR", NO_RESPONSE +
str(self.unresponsive_count) + " in a row!"))
if self.unresponsive_count > TOO_SLOW_THRESHOLD:
self.too_slow = True
clear_q(self.aq) # clear the queue from unhandled actions
return action
def worker_run(self):
timeout = 0
while not self._worker_stop.wait(timeout=timeout):
try:
delay_abs, method, task, args, kwargs = self.task_queue.get_nowait(
)
if delay_abs > self.timefunc():
self.task_queue.put_nowait(
(delay_abs, method, task, args, kwargs))
raise queue.Empty()
except queue.Empty:
timeout = 0.25
else:
timeout = 0
try:
task.completed = method(*args, **kwargs)
except KeyboardInterrupt:
break
except Exception: # catch all
task.completed = False
import traceback
traceback.print_exc()
# print('Process background task:', method, task)
self.task_queue.task_done()
def scheduler(outgoing, incoming, bounds):
schedules = collections.deque()
# Process incoming packets
while True:
# Calculate the timeout to the next schedule
try:
# Calculate when is the next timeout
next_timeout = None
if schedules:
next_timeout = max(0, schedules[0]-time.time())
if not next_timeout:
# Fake timeout to give priority to the tick
raise queue.Empty()
data = incoming.get(timeout=next_timeout)
except queue.Empty:
outgoing.put_nowait((BusType.TICK, None))
schedules.popleft()
continue
# Handle stop
if data is None:
return
# Add to the schedulers
if data:
logging.info("Scheduler will tick in %d seconds", data[0])
schedules.append(time.time() + data[0])
def wait_for_event(self,
event_name,
predicate,
timeout=DEFAULT_TIMEOUT,
*args,
**kwargs):
"""Wait for an event that satisfies a predicate to appear.
Continuously pop events of a particular name and check against the
predicate until an event that satisfies the predicate is popped or
timed out. Note this will remove all the events of the same name that
do not satisfy the predicate in the process.
Args:
event_name: Name of the event to be popped.
predicate: A function that takes an event and returns True if the
predicate is satisfied, False otherwise.
timeout: Number of seconds to wait.
*args: Optional positional args passed to predicate().
**kwargs: Optional keyword args passed to predicate().
consume_ignored_events: Whether or not to consume events while
searching for the desired event. Defaults to True if unset.
Returns:
The event that satisfies the predicate.
Raises:
queue.Empty: Raised if no event that satisfies the predicate was
found before time out.
"""
deadline = time.time() + timeout
ignored_events = []
consume_events = kwargs.pop('consume_ignored_events', True)
while True:
event = None
try:
event = self.pop_event(event_name, 1)
if consume_events:
self.log.debug('Consuming event: %r' % event)
else:
self.log.debug('Peeking at event: %r' % event)
ignored_events.append(event)
except queue.Empty:
pass
if event and predicate(event, *args, **kwargs):
for ignored_event in ignored_events:
self.get_event_q(event_name).put(ignored_event)
self.log.debug('Matched event: %r with %s' %
(event, predicate.__name__))
return event
if time.time() > deadline:
for ignored_event in ignored_events:
self.get_event_q(event_name).put(ignored_event)
msg = 'Timeout after {}s waiting for event: {}'.format(
timeout, event_name)
self.log.info(msg)
raise queue.Empty(msg)
def get(self, **kwargs):
import queue
self._got = kwargs
try:
return self._items.pop()
except IndexError:
raise queue.Empty()
def get(self, timeout=None) -> None:
"""Get an item."""
ret = self._count.acquire(timeout=timeout)
if not ret:
raise queue.Empty()
with self._lock:
return self._queue.pop(0)
def test_make_request(self):
"""Test that _make_request exits when the queue is Empty."""
job_queue = next(_make_mocks())
job_queue.get_nowait.side_effect = queue.Empty()
st, _, _ = _initialize_a_session_thread(job_queue=job_queue)
st._make_request()
job_queue.get_nowait.assert_called_once_with()
def get(self, block=True, timeout=None):
'get an item from the queue'
if self.d:
rv = self.d.popleft()
else:
raise queue.Empty()
return rv
def put(self, item):
with self.not_full:
if self.maxsize > 0:
while not self.__is_exhausted and self._qsize() == self.maxsize:
self.not_full.wait()
if self.__is_exhausted:
raise queue.Empty()
self._put(item)
self.not_empty.notify()
def get_new_logs(self):
logs = []
while not self.log_queue.empty():
try:
log = self.log_queue.get(block=False)
logs.append(log)
except queue.Empty():
return logs
return logs
def get_new_msgs(self):
msgs = []
while not self._msg_queue.empty():
try:
msg = self._msg_queue.get(block=False)
msgs.append(msg)
except queue.Empty():
return msgs
return msgs
def get(self):
with self.not_empty:
while not self.__is_exhausted and not self._qsize():
self.not_empty.wait()
if self.__is_exhausted and not self._qsize():
raise queue.Empty()
item = self._get()
self.not_full.notify()
return item
def get(self, block=True):
if self.obj is None:
raise queue.Empty()
elif self.obj is QuitSentinel:
self.obj = None
return QuitSentinel
else:
tmp = self.obj
self.obj = QuitSentinel
return tmp
def queue_func(name, number, filepath1, filepath2, consumer_key,
consumer_secret, access_key, access_secret):
q = queue.Queue()
q.put(
tweetvideo.googlevision(name, number, filepath1, filepath2,
consumer_key, consumer_secret, access_key,
access_secret))
q.put(tweetvideo.Image2Video(name, filepath1))
while not queue.Empty():
q.get()
def getJob(self):
try:
if self.output.empty():
raise queue.Empty()
job = self.output.get(True, 1)
self.output.task_done()
except queue.Empty as e:
job = {'waiting': True}
finally:
return job
def test_exit_when_inactive_empty(self):
q = mock.create_autospec(queue.Queue, instance=True)
q.get.side_effect = queue.Empty()
call = mock.create_autospec(grpc.Call, instance=True)
call.is_active.return_value = False
generator = bidi._RequestQueueGenerator(q)
generator.call = call
items = list(generator)
assert items == []
def test_worker_thread_disk():
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(),
mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.disk_threads = 1
d._disk_queue = mock.MagicMock()
d._disk_queue.get.side_effect = [
(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()),
]
d._process_data = mock.MagicMock()
patched_tc.side_effect = [False, True]
d._worker_thread_disk()
assert d._process_data.call_count == 1
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(),
mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.disk_threads = 1
d._disk_queue = mock.MagicMock()
d._disk_queue.get.side_effect = [
(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()),
]
d._process_data = mock.MagicMock()
d._process_data.side_effect = Exception()
patched_tc.side_effect = [False, True]
d._worker_thread_disk()
assert len(d._exceptions) == 1
with mock.patch(
'blobxfer.operations.download.Downloader.termination_check',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(),
mock.MagicMock())
d._general_options.dry_run = False
d._general_options.concurrency.disk_threads = 1
d._disk_queue = mock.MagicMock()
d._disk_queue.get.side_effect = queue.Empty()
d._process_data = mock.MagicMock()
patched_tc.side_effect = [False, True]
d._worker_thread_disk()
assert d._process_data.call_count == 0
def test_worker_run_empty_in_queue(self):
mocked_func = MagicMock()
mocked_in_queue = MagicMock()
mocked_in_queue.get.side_effect = queue.Empty()
mocked_out_queue = MagicMock()
mocked_stopper = MagicMock()
mocked_stopper.is_set.side_effect = [False, True]
worker = Worker(mocked_func, mocked_in_queue, mocked_out_queue, mocked_stopper)
worker.run()
mocked_in_queue.get.assert_called_with(timeout=5)
self.assertFalse(mocked_func.called)
self.assertFalse(mocked_out_queue.put.called)
def poll_log_queue(self):
print("checked")
# Check every 100ms if there is a new message in the queue to display
while True:
try:
print("OOOOOOOOOin", queue.Empty())
record = self.log_queue.get(block=False)
except queue.Empty:
break
else:
self.display()
self.frame.after(100, self.poll_log_queue)
def get(self, timeout=None):
"""Get the next tile coords `(tx, ty)` which needs fetching.
:param timeout: If not `None` then wait this many seconds for a job,
before raising :class:`queue.Empty`
"""
now = _time.clock()
while timeout is None or _time.clock() < now + timeout:
with self._lock:
if len(self._needed_tiles) > 0:
return self._needed_tiles.pop()
self._job_queue.get(timeout=timeout)
raise queue.Empty()
def test_yield_initial_callable_and_exit(self):
q = mock.create_autospec(queue.Queue, instance=True)
q.get.side_effect = queue.Empty()
call = mock.create_autospec(grpc.Call, instance=True)
call.is_active.return_value = False
generator = bidi._RequestQueueGenerator(
q, initial_request=lambda: mock.sentinel.A)
generator.call = call
items = list(generator)
assert items == [mock.sentinel.A]
