def run(self):
"""Run infinite while receive a marker var or exec something"""
while self.is_alive():
task, = self.queue.get()
if task is None:
LOGGER.debug("%s> getting end-of-queue markers", self.name)
# Indicate that a formerly enqueued task is complete
self.queue.task_done()
# reached end of queue
break
LOGGER.debug("%s> begin(%s) task.tid=%s", self.name,\
utils.strdate(), task)
# run the task
with self.lock:
current_task = self.tasks[task]
current_task.state = Task.STATE_RUNNING
self.tasks[task] = current_task
try:
for retry in range_type(1, self.max_retry + 1):
LOGGER.debug("running retry=%d task state %d", \
retry, self.tasks[task].state)
result = current_task.run(self.builder, retry=retry)
if current_task.state == Task.STATE_SUCCESS:
LOGGER.debug("%s> end(%s) task.tid=%s results=%s",\
self.name, utils.strdate(), task, result)
break
except TaskDependenciesError as err:
current_task.state = Task.STATE_WRONG
LOGGER.error("%s> %s", self.name, err)
except Exception as err: # pylint: disable=broad-except
current_task.state = Task.STATE_ERROR
trb = traceback.format_exc()
LOGGER.error("%s> You will see this error in prod: %s",\
self.name, err)
LOGGER.error(trb)
finally:
# write proxydict content
with self.lock:
self.tasks[task] = current_task
LOGGER.debug("update childs of %s", task)
# Update tasks depends on this task
for updated in current_task.update_childs(self.tasks):
for tasku_id, tasku in iteritems(updated):
# LOGGER.debug("update task of %s", tasku)
self.tasks[tasku_id] = tasku
self.queue.task_done()
else:
self.queue.task_done()
def isolated_vertices(self):
"""Generate a list of isolated vertices (ie no dependency)
Yields:
str : isolated vertices.
Examples:
>>> graph
[
('for_test-tid1', 'for_test-tid2'),
('for_test-tid1', 'for_test-tid3'),
('for_test-tid1', 'for_test-tid4'),
('for_test-tid2', 'for_test-tid4'),
('for_test-tid3', 'for_test-tid3'),
('for_test-tid4', 'for_test-tid4')
]
>>> list(graph.isolated_vertices())
['for_test-tid4', 'for_test-tid3']
"""
for key, value in iteritems(self):
if not value:
yield key
def test_update_childs():
task1 = Task("tid1", {"foo": "bar1"}, "func")
task2 = Task("tid2", {"foo": "bar2"}, "func", require=["tid1", "tid3"])
task3 = Task("tid3", {"foo": "bar3"}, "func", require=["tid1"])
tasks = {
task1.tid: task1,
task2.tid: task2,
task3.tid: task3,
}
task1.state = Task.STATE_SUCCESS
items = list(task1.update_childs(tasks))
# Update tasks depends on this task
for updated in items:
for task_id, task in _py6.iteritems(updated):
tasks[task_id] = task
assert tasks[task3.tid].require == []
assert tasks[task2.tid].require == [task3.tid]
task3.state = Task.STATE_SUCCESS
tasks[task3.tid] = task3
def test_dict3():
_py6.PY2 = False
assert list(_py6.iterkeys({1: 2})) == [1]
assert list(_py6.itervalues({1: 2})) == [2]
assert list(_py6.iteritems({1: 2})) == [(1, 2)]