def test_getnowait(self):
with patch.object(self.m, "__len__") as mock_len:
mock_len.return_value = 1
q = iodict.DurableQueue(path="/not/a/path")
with patch.object(q._count, "acquire",
autospec=True) as mock_acquire:
with patch.object(q._queue, "popitem") as mock_popitem:
mock_popitem.return_value = "test"
self.assertEqual(q.get_nowait(), "test")
mock_acquire.assert_called_with(False, None)
def job_q_processor(self, q_processes, lock=None):
"""Process a given work queue.
The defined `queue` is processed. The `processes` arg allows this
method to spawn N processes.
:param q_processes: Queue object
:type q_processes: Object
:param lock: Locking object, used if a component requires it.
:type lock: Object
"""
def _get_pending_bypass_threads(parents):
"""Return the current non-bypass thread objects.
:param parents: Dictionary of all parent objects.
:type parents: Dictionary
:returns: List
"""
return [
i["t"].name
for i in parents.values()
if i["bypass"]
and not i["t"].is_alive()
and i["t"].ident is None
]
def _get_pending_threads(parents):
"""Return the current non-bypass thread objects.
:param parents: Dictionary of all parent objects.
:type parents: Dictionary
:returns: List
"""
return [
i["t"].name
for i in parents.values()
if not i["bypass"]
and not i["t"].is_alive()
and i["t"].ident is None
]
def _get_thread_count(parents):
"""Return the current non-bypass thread count.
:param parents: Dictionary of all parent objects.
:type parents: Dictionary
:returns: Integer
"""
return len(
[
True
for i in parents.values()
if not i["bypass"] and i["t"].is_alive()
]
)
if not lock:
lock = self.driver.get_lock()
parent_tracker_recover = iodict.DurableQueue(
path=os.path.join(self.args.cache_path, "parent_tracker"),
lock=lock,
)
parent_tracker = collections.OrderedDict()
for item in parent_tracker_recover.getter():
k, v = item
q = self.driver.get_queue(name=k)
parent_tracker[k] = dict(t=None, q=q, bypass=v)
parent_tracker[k]["t"] = self.driver.thread_processor(
target=self.q_processor,
kwargs=dict(
queue=parent_tracker[k]["q"],
lock=lock,
),
name=k,
daemon=True,
)
while not q_processes.empty() or parent_tracker:
try:
(
component_kwargs,
command,
info,
) = q_processes.get_nowait()
except ValueError as e:
self.log.critical("Queue object value error [ %s ]", str(e))
except Exception:
sleep_interval = 0.1
else:
sleep_interval = 0.001
lower_command = command.lower()
job = component_kwargs["job"]
self.log.debug("Received job_id [ %s ]", job["job_id"])
# NOTE(cloudnull): If the command is queuesentinel purge all
# queued items. This is on the ONE component
# where we intercept and react outside of
# the component structure.
if lower_command == "queuesentinel":
count = 0
for value in list(parent_tracker.values()):
count = 0
for item in value["q"].getter():
_kwargs, _command, _ = item
self.q_return.put(
(
None,
None,
False,
"Omitted due to sentinel from {}".format(
job["job_id"]
),
_kwargs["job"],
_command,
0,
None,
)
)
count += 1
self.log.info(
"Purged %s items from the work queues", count
)
if job.get("parent_async_bypass") is True:
_q_name = "q_bypass_{}".format(
job.get("parent_id", job["job_id"])
)
elif job.get("parent_async") is True:
_q_name = "q_async_{}".format(
job.get("parent_id", job["job_id"])
)
else:
_q_name = "q_general"
if _q_name not in parent_tracker:
parent_tracker.setdefault(
_q_name,
dict(
t=None,
q=self.driver.get_queue(name=_q_name),
bypass=job.get("parent_async_bypass", False),
),
)
parent_tracker[_q_name][
"t"
] = self.driver.thread_processor(
target=self.q_processor,
kwargs=dict(
queue=parent_tracker[_q_name]["q"],
lock=lock,
),
name=_q_name,
daemon=True,
)
self.log.info("Parent queue [ %s ] created.", _q_name)
parent_tracker[_q_name]["q"].put(
(component_kwargs, command, info)
)
for t in _get_pending_bypass_threads(parents=parent_tracker):
self.log.info("Starting bypass process [ %s ]", t)
parent_tracker[t]["t"].start()
while _get_thread_count(parents=parent_tracker) <= self.cpu_count:
for t in _get_pending_threads(parents=parent_tracker):
self.log.info("Starting process [ %s ]", t)
parent_tracker[t]["t"].start()
else:
break
for key, value in list(parent_tracker.items()):
if value["t"].is_alive() or value["t"].ident is None:
continue
else:
timestamp = time.time()
timeout = timestamp - value.get("timeout", timestamp) > 5
if value["q"].empty() and timeout:
self.terminate_process(process=value["t"])
parent_tracker.pop(key)
self.log.info("Pruned parent [ %s ]", key)
elif not value["q"].empty():
self.log.warning(
"Parent thread was terminated but the queue [ %s ]"
" had items in it, respawning",
key,
)
parent_tracker[key].pop("timeout", None)
parent_tracker[key][
"t"
] = self.driver.thread_processor(
target=self.q_processor,
kwargs=dict(
queue=parent_tracker[key]["q"],
lock=lock,
),
name=key,
daemon=True,
)
elif "timeout" not in parent_tracker[key]:
self.log.info("Starting to prune parent [ %s ]", key)
parent_tracker[key]["timeout"] = timestamp
if self.driver.event.is_set():
for k, v in parent_tracker.items():
v["q"].flush()
parent_tracker_recover.put((k, v["bypass"]))
return
time.sleep(sleep_interval)
def test_putnowait(self):
q = iodict.DurableQueue(path="/not/a/path")
with patch.object(self.m, "_queue") as mock__queue:
mock__queue.return_value = dict()
with patch("builtins.open", unittest.mock.mock_open()):
q.put_nowait("test")
def test_get_timeout(self):
q = iodict.DurableQueue(path="/not/a/path")
with self.assertRaises(queue.Empty):
q.get(timeout=0.1)
def test_get_negative_timeout(self):
q = iodict.DurableQueue(path="/not/a/path")
with self.assertRaises(ValueError):
q.get(timeout=-1)
def test_empty(self):
q = iodict.DurableQueue(path="/not/a/path")
self.assertEqual(q.empty(), True)
with patch.object(self.m, "__len__"):
self.assertEqual(q.empty(), False)
def test_close_missing(self):
q = iodict.DurableQueue(path="/not/a/path")
with patch("os.rmdir") as mock_rmdir:
mock_rmdir.side_effect = FileNotFoundError
q.close()
def test_close(self):
q = iodict.DurableQueue(path="/not/a/path")
with patch("os.rmdir") as mock_rmdir:
q.close()